RUMM-655 / RUMM-689 Use NTP time for Logs, Spans and RUM events

Datadog/Datadog.xcodeproj/project.pbxproj

@@ -214,6 +214,7 @@
 		61940C7C25668EC600A20043 /* URLSessionInterceptionHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61940C7B25668EC600A20043 /* URLSessionInterceptionHandler.swift */; };
 		6198D27124C6E3B700493501 /* RUMViewScopeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6198D27024C6E3B700493501 /* RUMViewScopeTests.swift */; };
 		61A763DC252DB2B3005A23F2 /* NSURLSessionBridge.m in Sources */ = {isa = PBXBuildFile; fileRef = 61A763DB252DB2B3005A23F2 /* NSURLSessionBridge.m */; };
+		61A9238E256FCAA2009B9667 /* DateCorrectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61A9238D256FCAA2009B9667 /* DateCorrectionTests.swift */; };
 		61AD4E182451C7FF006E34EA /* TracingFeatureMocks.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61AD4E172451C7FF006E34EA /* TracingFeatureMocks.swift */; };
 		61AD4E3824531500006E34EA /* DataFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61AD4E3724531500006E34EA /* DataFormat.swift */; };
 		61AD4E3A24534075006E34EA /* TracingFeatureTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61AD4E3924534075006E34EA /* TracingFeatureTests.swift */; };
@@ -242,6 +243,7 @@
 		61BB2B1B244A185D009F3F56 /* PerformancePreset.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61BB2B1A244A185D009F3F56 /* PerformancePreset.swift */; };
 		61BBD19524ED4E9E0023E65F /* FeaturesConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61BBD19424ED4E9E0023E65F /* FeaturesConfiguration.swift */; };
 		61BBD19724ED50040023E65F /* FeaturesConfigurationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61BBD19624ED50040023E65F /* FeaturesConfigurationTests.swift */; };
+		61BCB81F256EB77F0039887B /* ServerDateProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61BCB81E256EB77F0039887B /* ServerDateProvider.swift */; };
 		61C2C20724C098FC00C0321C /* RUMSessionScope.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C2C20624C098FC00C0321C /* RUMSessionScope.swift */; };
 		61C2C20924C0C75500C0321C /* RUMSessionScopeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C2C20824C0C75500C0321C /* RUMSessionScopeTests.swift */; };
 		61C2C20B24C1045300C0321C /* SendRUMFixture1ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C2C20A24C1045300C0321C /* SendRUMFixture1ViewController.swift */; };
@@ -256,6 +258,7 @@
 		61C3E63924BF19B4008053F2 /* RUMContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C3E63824BF19B4008053F2 /* RUMContext.swift */; };
 		61C3E63B24BF1A4B008053F2 /* RUMCommand.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C3E63A24BF1A4B008053F2 /* RUMCommand.swift */; };
 		61C3E63E24BF1B91008053F2 /* RUMApplicationScope.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C3E63D24BF1B91008053F2 /* RUMApplicationScope.swift */; };
+		61C576C6256E65BD00295F7C /* DateCorrection.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C576C5256E65BD00295F7C /* DateCorrection.swift */; };
 		61C5A88424509A0C00DA608C /* DDSpan.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C5A87824509A0C00DA608C /* DDSpan.swift */; };
 		61C5A88524509A0C00DA608C /* DDNoOps.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C5A87924509A0C00DA608C /* DDNoOps.swift */; };
 		61C5A88624509A0C00DA608C /* TracingUUIDGenerator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C5A87B24509A0C00DA608C /* TracingUUIDGenerator.swift */; };
@@ -635,6 +638,7 @@
 		61A763D9252DB2B3005A23F2 /* DatadogTests-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "DatadogTests-Bridging-Header.h"; sourceTree = "<group>"; };
 		61A763DA252DB2B3005A23F2 /* NSURLSessionBridge.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NSURLSessionBridge.h; sourceTree = "<group>"; };
 		61A763DB252DB2B3005A23F2 /* NSURLSessionBridge.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NSURLSessionBridge.m; sourceTree = "<group>"; };
+		61A9238D256FCAA2009B9667 /* DateCorrectionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DateCorrectionTests.swift; sourceTree = "<group>"; };
 		61AD4E172451C7FF006E34EA /* TracingFeatureMocks.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TracingFeatureMocks.swift; sourceTree = "<group>"; };
 		61AD4E3724531500006E34EA /* DataFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataFormat.swift; sourceTree = "<group>"; };
 		61AD4E3924534075006E34EA /* TracingFeatureTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TracingFeatureTests.swift; sourceTree = "<group>"; };
@@ -656,6 +660,7 @@
 		61BB2B1A244A185D009F3F56 /* PerformancePreset.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerformancePreset.swift; sourceTree = "<group>"; };
 		61BBD19424ED4E9E0023E65F /* FeaturesConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeaturesConfiguration.swift; sourceTree = "<group>"; };
 		61BBD19624ED50040023E65F /* FeaturesConfigurationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeaturesConfigurationTests.swift; sourceTree = "<group>"; };
+		61BCB81E256EB77F0039887B /* ServerDateProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerDateProvider.swift; sourceTree = "<group>"; };
 		61C2C20624C098FC00C0321C /* RUMSessionScope.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMSessionScope.swift; sourceTree = "<group>"; };
 		61C2C20824C0C75500C0321C /* RUMSessionScopeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMSessionScopeTests.swift; sourceTree = "<group>"; };
 		61C2C20A24C1045300C0321C /* SendRUMFixture1ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SendRUMFixture1ViewController.swift; sourceTree = "<group>"; };
@@ -670,6 +675,7 @@
 		61C3E63824BF19B4008053F2 /* RUMContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMContext.swift; sourceTree = "<group>"; };
 		61C3E63A24BF1A4B008053F2 /* RUMCommand.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMCommand.swift; sourceTree = "<group>"; };
 		61C3E63D24BF1B91008053F2 /* RUMApplicationScope.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMApplicationScope.swift; sourceTree = "<group>"; };
+		61C576C5256E65BD00295F7C /* DateCorrection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DateCorrection.swift; sourceTree = "<group>"; };
 		61C5A87824509A0C00DA608C /* DDSpan.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DDSpan.swift; sourceTree = "<group>"; };
 		61C5A87924509A0C00DA608C /* DDNoOps.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DDNoOps.swift; sourceTree = "<group>"; };
 		61C5A87B24509A0C00DA608C /* TracingUUIDGenerator.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TracingUUIDGenerator.swift; sourceTree = "<group>"; };
@@ -923,7 +929,7 @@
 		61133BA12423979B00786299 /* System */ = {
 			isa = PBXGroup;
 			children = (
-				61133BA82423979B00786299 /* DateProvider.swift */,
+				61C576C4256E655600295F7C /* Time */,
 				61E36A10254B2280001AD6F2 /* LaunchTimeProvider.swift */,
 				61133BA22423979B00786299 /* CarrierInfoProvider.swift */,
 				61133BA32423979B00786299 /* MobileDevice.swift */,
@@ -1149,6 +1155,7 @@
 		61133C222423990D00786299 /* System */ = {
 			isa = PBXGroup;
 			children = (
+				61A9238C256FCA92009B9667 /* Time */,
 				614AD085254C3027004999A3 /* LaunchTimeProviderTests.swift */,
 				61133C232423990D00786299 /* MobileDeviceTests.swift */,
 				61133C242423990D00786299 /* NetworkConnectionInfoProviderTests.swift */,
@@ -1777,6 +1784,14 @@
 			path = TapActionAutoInstrumentation;
 			sourceTree = "<group>";
 		};
+		61A9238C256FCA92009B9667 /* Time */ = {
+			isa = PBXGroup;
+			children = (
+				61A9238D256FCAA2009B9667 /* DateCorrectionTests.swift */,
+			);
+			path = Time;
+			sourceTree = "<group>";
+		};
 		61B03872252724AB00518F3C /* URLSessionAutoInstrumentation */ = {
 			isa = PBXGroup;
 			children = (
@@ -1837,6 +1852,16 @@
 			path = Scopes;
 			sourceTree = "<group>";
 		};
+		61C576C4256E655600295F7C /* Time */ = {
+			isa = PBXGroup;
+			children = (
+				61133BA82423979B00786299 /* DateProvider.swift */,
+				61BCB81E256EB77F0039887B /* ServerDateProvider.swift */,
+				61C576C5256E65BD00295F7C /* DateCorrection.swift */,
+			);
+			path = Time;
+			sourceTree = "<group>";
+		};
 		61C5A87724509A0C00DA608C /* Tracing */ = {
 			isa = PBXGroup;
 			children = (
@@ -2490,6 +2515,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				61E917D12465423600E6C631 /* TracerConfiguration.swift in Sources */,
+				61C576C6256E65BD00295F7C /* DateCorrection.swift in Sources */,
 				61E909ED24A24DD3005EA2DE /* OTSpan.swift in Sources */,
 				61133BDE2423979B00786299 /* CompilationConditions.swift in Sources */,
 				61E909F324A24DD3005EA2DE /* OTSpanContext.swift in Sources */,
@@ -2524,6 +2550,7 @@
 				61494CB124C839460082C633 /* RUMResourceScope.swift in Sources */,
 				61C2C20724C098FC00C0321C /* RUMSessionScope.swift in Sources */,
 				617CEB392456BC3A00AD4669 /* TracingUUID.swift in Sources */,
+				61BCB81F256EB77F0039887B /* ServerDateProvider.swift in Sources */,
 				61FF282624B8A248000B3D9B /* RUMEventOutput.swift in Sources */,
 				618715F924DC13A100FC0F69 /* RUMDataModelsMapping.swift in Sources */,
 				61C3638524361E9200C4D4E6 /* Globals.swift in Sources */,
@@ -2693,6 +2720,7 @@
 				61C5A89624509BF600DA608C /* TracerTests.swift in Sources */,
 				61F1A61A2498A51700075390 /* CoreMocks.swift in Sources */,
 				61E45BD22450F65B00F2C652 /* SpanBuilderTests.swift in Sources */,
+				61A9238E256FCAA2009B9667 /* DateCorrectionTests.swift in Sources */,
 				61E45BCF2450A6EC00F2C652 /* TracingUUIDTests.swift in Sources */,
 				614AD086254C3027004999A3 /* LaunchTimeProviderTests.swift in Sources */,
 				61133C482423990D00786299 /* DDDatadogTests.swift in Sources */,

Datadog/Datadog.xcodeproj/xcshareddata/xcschemes/Example.xcscheme

@@ -58,7 +58,7 @@
          </EnvironmentVariable>
          <EnvironmentVariable
             key = "DD_TEST_SCENARIO_IDENTIFIER"
-            value = "TracingScenario"
+            value = "TracingManualInstrumentationScenario"
             isEnabled = "NO">
          </EnvironmentVariable>
          <EnvironmentVariable

Datadog/Example/Debugging/DebugTracingViewController.swift

@@ -117,5 +117,5 @@ class DebugTracingViewController: UIViewController {
 }
 
 private func wait(seconds: TimeInterval) {
-    Thread.sleep(forTimeInterval: 0.5)
+    Thread.sleep(forTimeInterval: seconds)
 }

Sources/Datadog/Core/Feature.swift

@@ -12,6 +12,7 @@ internal struct FeaturesCommonDependencies {
     let httpClient: HTTPClient
     let mobileDevice: MobileDevice
     let dateProvider: DateProvider
+    let dateCorrection: DateCorrectionType
     let userInfoProvider: UserInfoProvider
     let networkConnectionInfoProvider: NetworkConnectionInfoProviderType
     let carrierInfoProvider: CarrierInfoProviderType

Sources/Datadog/Core/System/Time/DateCorrection.swift

@@ -0,0 +1,63 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-2020 Datadog, Inc.
+ */
+
+import Foundation
+
+/// Adjusts device time to server time using the time difference calculated with NTP.
+internal protocol DateCorrectionType {
+    /// Corrects given device time to server time using the last known time difference between the two.
+    func toServerDate(deviceDate: Date) -> Date
+}
+
+internal class DateCorrection: DateCorrectionType {
+    static let datadogNTPServers = [
+        "0.datadog.pool.ntp.org",
+        "1.datadog.pool.ntp.org",
+        "2.datadog.pool.ntp.org",
+        "3.datadog.pool.ntp.org"
+    ]
+    private let deviceDateProvider: DateProvider
+    private let serverDateProvider: ServerDateProvider
+
+    init(deviceDateProvider: DateProvider, serverDateProvider: ServerDateProvider) {
+        self.deviceDateProvider = deviceDateProvider
+        self.serverDateProvider = serverDateProvider
+        // swiftlint:disable trailing_closure
+        serverDateProvider.synchronize(
+            with: DateCorrection.datadogNTPServers.randomElement()!, // swiftlint:disable:this force_unwrapping
+            completion: { serverTime in
+                let deviceTime = deviceDateProvider.currentDate()
+                if let serverTime = serverTime {
+                    let difference = (serverTime.timeIntervalSince(deviceTime) * 1_000).rounded() / 1_000
+                    userLogger.info(
+                        """
+                        NTP time synchronization completed.
+                        Server time will be used for signing events (current server time is \(serverTime); \(difference)s difference with device time).
+                        """
+                    )
+                } else {
+                    userLogger.warn(
+                        """
+                        NTP time synchronization failed.
+                        Device time will be used for signing events (current device time is \(deviceTime)).
+                        """
+                    )
+                }
+            }
+        )
+        // swiftlint:enable trailing_closure
+    }
+
+    func toServerDate(deviceDate: Date) -> Date {
+        if let serverTime = serverDateProvider.currentDate() {
+            let deviceTime = deviceDateProvider.currentDate()
+            let timeDifference = serverTime.timeIntervalSince(deviceTime)
+            return deviceDate.addingTimeInterval(timeDifference)
+        } else {
+            return deviceDate
+        }
+    }
+}

Sources/Datadog/Core/System/DateProvider.swift → Sources/Datadog/Core/System/Time/DateProvider.swift

@@ -6,8 +6,9 @@
 
 import Foundation
 
-/// Interface for date provider used for files orchestration.
+/// Provides current device time information.
 internal protocol DateProvider {
+    /// Current device time.
     func currentDate() -> Date
 }
 

Sources/Datadog/Core/System/Time/ServerDateProvider.swift

@@ -0,0 +1,32 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-2020 Datadog, Inc.
+ */
+
+import Foundation
+import Kronos
+
+/// Abstract the monotonic clock synchronized with the server using NTP.
+internal protocol ServerDateProvider {
+    /// Start the clock synchronisation with NTP server.
+    /// Calls the `completion` by passing it the server time when the synchronization succeeds or`nil` if it fails.
+    func synchronize(with ntpPool: String, completion: @escaping (Date?) -> Void)
+    /// Returns the server time or `nil` if not yet determined.
+    /// This time gets more precise while synchronization is pending.
+    func currentDate() -> Date?
+}
+
+internal class NTPServerDateProvider: ServerDateProvider {
+    func synchronize(with ntpPool: String, completion: @escaping (Date?) -> Void) {
+        // swiftlint:disable trailing_closure multiline_arguments_brackets
+        Clock.sync(from: ntpPool, completion: { serverTime, _ in
+            completion(serverTime)
+        })
+        // swiftlint:enable trailing_closure multiline_arguments_brackets
+    }
+
+    func currentDate() -> Date? {
+        return Clock.now
+    }
+}

Sources/Datadog/Core/Upload/DataUploader.swift

@@ -14,14 +14,6 @@ internal class UploadURLProvider {
     class QueryItemProvider {
         let value: () -> URLQueryItem
 
-        /// Creates `batch_time=...` query item adding current timestamp (in milliseconds) to the URL.
-        static func batchTime(using dateProvider: DateProvider) -> QueryItemProvider {
-            return QueryItemProvider {
-                let timestamp = dateProvider.currentDate().timeIntervalSince1970.toMilliseconds
-                return URLQueryItem(name: "batch_time", value: "\(timestamp)")
-            }
-        }
-
         /// Creates `ddsource=ios` query item.
         static func ddsource() -> QueryItemProvider {
             let queryItem = URLQueryItem(name: "ddsource", value: Datadog.Constants.ddsource)
@@ -41,7 +33,10 @@ internal class UploadURLProvider {
 
     var url: URL {
         var urlComponents = URLComponents(url: urlWithClientToken, resolvingAgainstBaseURL: false)
-        urlComponents?.queryItems = queryItemProviders.map { $0.value() }
+
+        if !queryItemProviders.isEmpty {
+            urlComponents?.queryItems = queryItemProviders.map { $0.value() }
+        }
 
         guard let url = urlComponents?.url else {
             userLogger.error("🔥 Failed to create URL from \(urlWithClientToken) with \(queryItemProviders)")

Sources/Datadog/Datadog.swift

@@ -5,7 +5,6 @@
  */
 
 import Foundation
-import Kronos
 
 /// Datadog SDK configuration object.
 public class Datadog {
@@ -48,10 +47,6 @@ public class Datadog {
     ///   - appContext: context passing information about the app.
     ///   - configuration: the SDK configuration obtained using `Datadog.Configuration.builderUsing(clientToken:)`.
     public static func initialize(appContext: AppContext, configuration: Configuration) {
-        // TODO: RUMM-655 This is only to check at runtime if `lyft/Kronos` is properly linked. Will be removed before
-        // merging to `master` branch.
-        _ = Clock.now
-
         // TODO: RUMM-511 remove this warning
         #if targetEnvironment(macCatalyst)
         consolePrint("⚠️ Catalyst is not officially supported by Datadog SDK: some features may NOT be functional!")
@@ -101,6 +96,10 @@ public class Datadog {
         }
 
         let dateProvider = SystemDateProvider()
+        let dateCorrection = DateCorrection(
+            deviceDateProvider: dateProvider,
+            serverDateProvider: NTPServerDateProvider()
+        )
         let userInfoProvider = UserInfoProvider()
         let networkConnectionInfoProvider = NetworkConnectionInfoProvider()
         let carrierInfoProvider = CarrierInfoProvider()
@@ -133,6 +132,7 @@ public class Datadog {
             httpClient: HTTPClient(),
             mobileDevice: MobileDevice.current,
             dateProvider: dateProvider,
+            dateCorrection: dateCorrection,
             userInfoProvider: userInfoProvider,
             networkConnectionInfoProvider: networkConnectionInfoProvider,
             carrierInfoProvider: carrierInfoProvider,

Sources/Datadog/FeaturesIntegration/LoggingForTracingAdapter.swift

@@ -26,7 +26,8 @@ internal struct LoggingForTracingAdapter {
                     loggerName: "trace",
                     userInfoProvider: tracingFeature.userInfoProvider,
                     networkConnectionInfoProvider: tracerConfiguration.sendNetworkInfo ? tracingFeature.networkConnectionInfoProvider : nil,
-                    carrierInfoProvider: tracerConfiguration.sendNetworkInfo ? tracingFeature.carrierInfoProvider : nil
+                    carrierInfoProvider: tracerConfiguration.sendNetworkInfo ? tracingFeature.carrierInfoProvider : nil,
+                    dateCorrection: loggingFeature.dateCorrection
                 ),
                 fileWriter: loggingFeature.storage.writer,
 

Sources/Datadog/Logger.swift

@@ -404,7 +404,8 @@ public class Logger {
                 loggerName: resolveLoggerName(for: loggingFeature),
                 userInfoProvider: loggingFeature.userInfoProvider,
                 networkConnectionInfoProvider: sendNetworkInfo ? loggingFeature.networkConnectionInfoProvider : nil,
-                carrierInfoProvider: sendNetworkInfo ? loggingFeature.carrierInfoProvider : nil
+                carrierInfoProvider: sendNetworkInfo ? loggingFeature.carrierInfoProvider : nil,
+                dateCorrection: loggingFeature.dateCorrection
             )
 
             switch (useFileOutput, useConsoleLogFormat) {