RUMM-3151 feat: reduce number of view updates by filtering events fro…

…m payload

DatadogCore/Sources/Core/Storage/DataBlock.swift

@@ -5,6 +5,7 @@
  */
 
 import Foundation
+import DatadogInternal
 
 /// Block size binary type
 internal typealias BlockSize = UInt32

DatadogCore/Sources/Core/Storage/EventGenerator.swift

@@ -0,0 +1,60 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-Present Datadog, Inc.
+ */
+
+import Foundation
+import DatadogInternal
+
+/// Event generator that generates events from the given data blocks.
+internal struct EventGenerator: Sequence, IteratorProtocol {
+    private let dataBlocks: [DataBlock]
+    private var index: Int
+
+    init(dataBlocks: [DataBlock], index: Int = 0) {
+        self.dataBlocks = dataBlocks
+        self.index = index
+    }
+
+    /// Returns the next event.
+    ///
+    /// Data format
+    /// ```
+    /// [EVENT 1 METADATA] [EVENT 1] [EVENT 2 METADATA] [EVENT 2] [EVENT 3]
+    /// ```
+    ///
+    /// - Returns: The next event or `nil` if there are no more events.
+    /// - Note: a `DataBlock` with `.event` type marks the beginning of the event.
+    ///         It is either followed by another `DataBlock` with `.event` type or
+    ///         by a `DataBlock` with `.metadata` type.
+    mutating func next() -> Event? {
+        guard index < dataBlocks.count else {
+            return nil
+        }
+
+        var metadata: DataBlock? = nil
+        // If the next block is an event metadata, read it.
+        if dataBlocks[index].type == .eventMetadata {
+            metadata = dataBlocks[index]
+            index += 1
+        }
+
+        // If this is the last block, return nil.
+        // there cannot be a metadata block without an event block.
+        guard index < dataBlocks.count else {
+            return nil
+        }
+
+        // If the next block is an event, read it.
+        guard dataBlocks[index].type == .event else {
+            // this is safeguard against corrupted data.
+            // if there was a metadata block, it will be skipped.
+            return next()
+        }
+        let event = dataBlocks[index]
+        index += 1
+
+        return Event(data: event.data, metadata: metadata?.data)
+    }
+}

DatadogCore/Sources/Core/Storage/Reading/Reader.swift

@@ -5,6 +5,7 @@
  */
 
 import Foundation
+import DatadogInternal
 
 internal struct Batch {
     /// Data blocks in the batch.

DatadogCore/Sources/Core/Storage/Writing/AsyncWriter.swift

@@ -7,24 +7,6 @@
 import Foundation
 import DatadogInternal
 
-/// A type, writing data.
-public protocol Writer {
-    /// Encodes given encodable value and metadata, and writes to the destination.
-    /// - Parameter value: Encodable value to write.
-    /// - Parameter metadata: Encodable metadata to write.
-    func write<T: Encodable, M: Encodable>(value: T, metadata: M?)
-}
-
-extension Writer {
-    /// Encodes given encodable value and writes to the destination.
-    /// Uses `write(value:metadata:)` with `nil` metadata.
-    /// - Parameter value: Encodable value to write.
-    public func write<T: Encodable>(value: T) {
-        let metadata: Data? = nil
-        write(value: value, metadata: metadata)
-    }
-}
-
 /// Writer performing writes asynchronously on a given queue.
 internal struct AsyncWriter: Writer {
     private let writer: Writer
@@ -41,5 +23,6 @@ internal struct AsyncWriter: Writer {
 }
 
 internal struct NOPWriter: Writer {
-    func write<T: Encodable, M: Encodable>(value: T, metadata: M?) { }
+    func write<T: Encodable, M: Encodable>(value: T, metadata: M?) {
+    }
 }

DatadogCore/Tests/Datadog/Core/Persistence/EventGeneratorTests.swift

@@ -5,7 +5,7 @@
  */
 
 import XCTest
-@testable import Datadog
+@testable import DatadogCore
 
 final class EventGeneratorTests: XCTestCase {
     func testEmpty() throws {

DatadogCore/Tests/Datadog/Core/Persistence/Reading/FileReaderTests.swift

@@ -30,15 +30,13 @@ class FileReaderTests: XCTestCase {
                 dateProvider: SystemDateProvider()
             )
         )
-
         let dataBlocks = [
             DataBlock(type: .eventMetadata, data: "EFGH".utf8Data),
             DataBlock(type: .event, data: "ABCD".utf8Data)
         ]
         let data = try dataBlocks
             .map { try $0.serialize() }
             .reduce(.init(), +)
-
         _ = try directory
             .createFile(named: Date.mockAny().toFileName)
             .append(data: data)
@@ -103,7 +101,6 @@ class FileReaderTests: XCTestCase {
         )
         let file1 = try directory.createFile(named: dateProvider.now.toFileName)
         try file1.append(data: DataBlock(type: .eventMetadata, data: "2".utf8Data).serialize())
-
         try file1.append(data: DataBlock(type: .event, data: "1".utf8Data).serialize())
 
         let file2 = try directory.createFile(named: dateProvider.now.toFileName)

DatadogCore/Tests/Datadog/Core/Persistence/Writing/FileWriterTests.swift

@@ -26,7 +26,7 @@ class FileWriterTests: XCTestCase {
     func testItWritesDataWithMetadataToSingleFileInTLVFormat() throws {
         let writer = FileWriter(
             orchestrator: FilesOrchestrator(
-                directory: temporaryDirectory,
+                directory: directory,
                 performance: PerformancePreset.mockAny(),
                 dateProvider: SystemDateProvider()
             ),
@@ -38,8 +38,8 @@ class FileWriterTests: XCTestCase {
         writer.write(value: ["key2": "value2"]) // skipped metadata here
         writer.write(value: ["key3": "value3"], metadata: ["meta3": "metaValue3"])
 
-        XCTAssertEqual(try temporaryDirectory.files().count, 1)
-        let stream = try temporaryDirectory.files()[0].stream()
+        XCTAssertEqual(try directory.files().count, 1)
+        let stream = try directory.files()[0].stream()
 
         let reader = DataBlockReader(input: stream)
         var block = try reader.next()
@@ -62,7 +62,7 @@ class FileWriterTests: XCTestCase {
     func testItWritesEncryptedDataWithMetadataToSingleFileInTLVFormat() throws {
         let writer = FileWriter(
             orchestrator: FilesOrchestrator(
-                directory: temporaryDirectory,
+                directory: directory,
                 performance: PerformancePreset.mockAny(),
                 dateProvider: SystemDateProvider()
             ),
@@ -78,8 +78,8 @@ class FileWriterTests: XCTestCase {
         writer.write(value: ["key2": "value2"]) // skipped metadata here
         writer.write(value: ["key3": "value3"], metadata: ["meta3": "metaValue3"])
 
-        XCTAssertEqual(try temporaryDirectory.files().count, 1)
-        let stream = try temporaryDirectory.files()[0].stream()
+        XCTAssertEqual(try directory.files().count, 1)
+        let stream = try directory.files()[0].stream()
 
         let reader = DataBlockReader(input: stream)
         var block = try reader.next()

DatadogCore/Tests/Datadog/Mocks/CoreMocks.swift

@@ -280,7 +280,7 @@ struct DataUploaderMock: DataUploaderType {
 
     var onUpload: (() throws -> Void)? = nil
 
-    func upload(events: [Data], context: DatadogContext) throws -> DataUploadStatus {
+    func upload(events: [Event], context: DatadogContext) throws -> DataUploadStatus {
         try onUpload?()
         return uploadStatus
     }

DatadogInternal/Sources/Storage/Writer.swift

@@ -8,11 +8,18 @@ import Foundation
 
 /// A type, writing data.
 public protocol Writer {
-    func write<T: Encodable>(value: T)
+    /// Encodes given encodable value and metadata, and writes to the destination.
+    /// - Parameter value: Encodable value to write.
+    /// - Parameter metadata: Encodable metadata to write.
+    func write<T: Encodable, M: Encodable>(value: T, metadata: M?)
 }
 
-public struct NOPWriter: Writer {
-    public init() { }
-
-    public func write<T>(value: T) where T: Encodable {}
+extension Writer {
+    /// Encodes given encodable value and writes to the destination.
+    /// Uses `write(value:metadata:)` with `nil` metadata.
+    /// - Parameter value: Encodable value to write.
+    public func write<T: Encodable>(value: T) {
+        let metadata: Data? = nil
+        write(value: value, metadata: metadata)
+    }
 }

DatadogInternal/Sources/Upload/Event.swift

@@ -22,55 +22,3 @@ public struct Event: Equatable {
         self.metadata = metadata
     }
 }
-
-/// Event generator that generates events from the given data blocks.
-internal struct EventGenerator: Sequence, IteratorProtocol {
-    private let dataBlocks: [DataBlock]
-    private var index: Int
-
-    init(dataBlocks: [DataBlock], index: Int = 0) {
-        self.dataBlocks = dataBlocks
-        self.index = index
-    }
-
-    /// Returns the next event.
-    ///
-    /// Data format
-    /// ```
-    /// [EVENT 1 METADATA] [EVENT 1] [EVENT 2 METADATA] [EVENT 2] [EVENT 3]
-    /// ```
-    ///
-    /// - Returns: The next event or `nil` if there are no more events.
-    /// - Note: a `DataBlock` with `.event` type marks the beginning of the event.
-    ///         It is either followed by another `DataBlock` with `.event` type or
-    ///         by a `DataBlock` with `.metadata` type.
-    mutating func next() -> Event? {
-        guard index < dataBlocks.count else {
-            return nil
-        }
-
-        var metadata: DataBlock? = nil
-        // If the next block is an event metadata, read it.
-        if dataBlocks[index].type == .eventMetadata {
-            metadata = dataBlocks[index]
-            index += 1
-        }
-
-        // If this is the last block, return nil.
-        // there cannot be a metadata block without an event block.
-        guard index < dataBlocks.count else {
-            return nil
-        }
-
-        // If the next block is an event, read it.
-        guard dataBlocks[index].type == .event else {
-            // this is safeguard against corrupted data.
-            // if there was a metadata block, it will be skipped.
-            return next()
-        }
-        let event = dataBlocks[index]
-        index += 1
-
-        return Event(data: event.data, metadata: metadata?.data)
-    }
-}

DatadogInternal/Tests/Upload/DataFormatTests.swift

@@ -0,0 +1,28 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-Present Datadog, Inc.
+ */
+
+import XCTest
+@testable import DatadogInternal
+
+final class DataFormatTests: XCTestCase {
+    func testFormat() throws {
+        let format = DataFormat(prefix: "prefix", suffix: "suffix", separator: "\n")
+        let events = [
+            "abc".data(using: .utf8)!,
+            "def".data(using: .utf8)!,
+            "ghi".data(using: .utf8)!
+        ]
+        let formatted = format.format(events)
+        let actual = String(data: formatted, encoding: .utf8)!
+        let expected =
+        """
+        prefixabc
+        def
+        ghisuffix
+        """
+        XCTAssertEqual(actual, expected)
+    }
+}

DatadogLogs/Sources/Feature/RequestBuilder.swift

@@ -20,7 +20,7 @@ internal struct RequestBuilder: FeatureRequestBuilder {
         self.customIntakeURL = customIntakeURL
     }
 
-    func request(for events: [Data], with context: DatadogContext) -> URLRequest {
+    func request(for events: [Event], with context: DatadogContext) -> URLRequest {
         let builder = URLRequestBuilder(
             url: url(with: context),
             queryItems: [
@@ -40,7 +40,7 @@ internal struct RequestBuilder: FeatureRequestBuilder {
             ]
         )
 
-        let data = format.format(events)
+        let data = format.format(events.map { $0.data })
         return builder.uploadRequest(with: data)
     }
 

DatadogRUM/Sources/Feature/RUMFeature.swift

@@ -75,7 +75,10 @@ internal final class RUMFeature: DatadogRemoteFeature {
             longTaskThreshold: configuration.longTaskThreshold,
             dateProvider: configuration.dateProvider
         )
-        self.requestBuilder = RequestBuilder(customIntakeURL: configuration.customEndpoint)
+        self.requestBuilder = RequestBuilder(
+            customIntakeURL: configuration.customEndpoint,
+            eventsFilter: RUMViewEventsFilter()
+        )
         self.messageReceiver = CombinedFeatureMessageReceiver(
             TelemetryReceiver(
                 dateProvider: configuration.dateProvider,

DatadogRUM/Sources/Feature/RUMViewEventsFilter.swift

@@ -0,0 +1,52 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-2020 Datadog, Inc.
+ */
+
+import Foundation
+import DatadogInternal
+
+internal struct RUMViewEventsFilter {
+    let decoder: JSONDecoder
+
+    init(decoder: JSONDecoder = JSONDecoder()) {
+        self.decoder = decoder
+    }
+
+    func filter(events: [Event]) -> [Event] {
+        var seen = Set<String>()
+        var skipped: [String: [Int64]] = [:]
+
+        // reversed is O(1) and no copy because it is view on the original array
+        let filtered = events.reversed().compactMap { event in
+            guard let metadata = event.metadata else {
+                // If there is no metadata, we can't filter it.
+                return event
+            }
+
+            guard let viewMetadata = try? decoder.decode(RUMViewEvent.Metadata.self, from: metadata) else {
+                // If we can't decode the metadata, we can't filter it.
+                return event
+            }
+
+            guard seen.contains(viewMetadata.id) == false else {
+                // If we've already seen this view, we can skip this
+                if skipped[viewMetadata.id] == nil {
+                    skipped[viewMetadata.id] = []
+                }
+                skipped[viewMetadata.id]?.append(viewMetadata.documentVersion)
+                return nil
+            }
+
+            seen.insert(viewMetadata.id)
+            return event
+        }
+
+        for (id, versions) in skipped {
+            DD.logger.debug("Skipping RUMViewEvent with id: \(id) and versions: \(versions.reversed().map(String.init).joined(separator: ", "))")
+        }
+
+        return filtered.reversed()
+    }
+}