Release 2.17.0

.gitlab-ci.yml

@@ -57,12 +57,18 @@ default:
   rules: 
     - if: '$CI_COMMIT_TAG || $RELEASE_GIT_TAG'
 
-.release-pipeline-delayed-job:
+.release-pipeline-20m-delayed-job:
   rules: 
     - if: '$CI_COMMIT_TAG || $RELEASE_GIT_TAG'
       when: delayed
       start_in: 20 minutes
 
+.release-pipeline-40m-delayed-job:
+  rules: 
+    - if: '$CI_COMMIT_TAG || $RELEASE_GIT_TAG'
+      when: delayed
+      start_in: 40 minutes
+
 ENV check:
   stage: pre
   rules: 
@@ -96,7 +102,7 @@ Unit Tests (iOS):
   script:
     - ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE"
     - make clean repo-setup ENV=ci
-    - make test-ios-all OS="$DEFAULT_IOS_OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
+    - make test-ios-all OS="$DEFAULT_IOS_OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE" USE_TEST_VISIBILITY=1
 
 Unit Tests (tvOS):
   stage: test
@@ -109,7 +115,7 @@ Unit Tests (tvOS):
   script:
     - ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE"
     - make clean repo-setup ENV=ci
-    - make test-tvos-all OS="$DEFAULT_TVOS_OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE"
+    - make test-tvos-all OS="$DEFAULT_TVOS_OS" PLATFORM="$PLATFORM" DEVICE="$DEVICE" USE_TEST_VISIBILITY=1
 
 UI Tests:
   stage: ui-test
@@ -164,6 +170,17 @@ Tools Tests:
     - make clean repo-setup ENV=ci
     - make tools-test
 
+Benchmark Build:
+  stage: smoke-test
+  rules:
+    - if: '$CI_COMMIT_BRANCH' # when on branch with following changes compared to develop
+      changes:
+        paths:
+          - "BenchmarkTests/**/*"
+        compare_to: 'develop'
+  script:
+    - make benchmark-build
+
 Smoke Tests (iOS):
   stage: smoke-test
   rules: 
@@ -265,7 +282,7 @@ E2E Test (upload to s8s):
     - ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE" --datadog-ci
     - make clean
     - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified
-    - make e2e-build-upload ARTIFACTS_PATH="artifacts/e2e"
+    - make e2e-upload ARTIFACTS_PATH="artifacts/e2e"
 
 # ┌────────────────────────────┐
 # │ Benchmark Test app upload: │
@@ -275,6 +292,7 @@ Benchmark Test (upload to s8s):
   stage: benchmark-test
   rules:
     - if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH'
+      allow_failure: true
   artifacts:
     paths:
       - artifacts
@@ -283,7 +301,7 @@ Benchmark Test (upload to s8s):
     - ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE" --datadog-ci
     - make clean
     - export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified
-    - make benchmark-build-upload ARTIFACTS_PATH="artifacts/benchmark"
+    - make benchmark-upload ARTIFACTS_PATH="artifacts/benchmark"
 
 # ┌─────────────────┐
 # │ SDK dogfooding: │
@@ -363,7 +381,7 @@ Publish CP podspecs (internal):
 Publish CP podspecs (dependent):
   stage: release-publish
   rules: 
-    - !reference [.release-pipeline-delayed-job, rules]
+    - !reference [.release-pipeline-20m-delayed-job, rules]
   before_script:
     - *export_MAKE_release_params
   script:
@@ -375,7 +393,7 @@ Publish CP podspecs (dependent):
 Publish CP podspecs (legacy):
   stage: release-publish
   rules: 
-    - !reference [.release-pipeline-delayed-job, rules]
+    - !reference [.release-pipeline-40m-delayed-job, rules]
   before_script:
     - *export_MAKE_release_params
   script:

BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

@@ -50,6 +50,18 @@
             ReferencedContainer = "container:BenchmarkTests.xcodeproj">
          </BuildableReference>
       </BuildableProductRunnable>
+      <EnvironmentVariables>
+         <EnvironmentVariable
+            key = "BENCHMARK_RUN"
+            value = "instrumented"
+            isEnabled = "YES">
+         </EnvironmentVariable>
+         <EnvironmentVariable
+            key = "BENCHMARK_SCENARIO"
+            value = "sessionReplay"
+            isEnabled = "YES">
+         </EnvironmentVariable>
+      </EnvironmentVariables>
    </LaunchAction>
    <ProfileAction
       buildConfiguration = "Release"

BenchmarkTests/Benchmarks/Package.swift

@@ -9,7 +9,7 @@ let package = Package(
     products: [
         .library(
             name: "DatadogBenchmarks",
-            targets: ["Benchmarks"]
+            targets: ["DatadogBenchmarks"]
         )
     ]
 )
@@ -28,12 +28,13 @@ func addOpenTelemetryDependency(_ version: Version) {
 
         package.targets = [
             .target(
-                name: "Benchmarks",
+                name: "DatadogBenchmarks",
                 dependencies: [
                     .product(name: "OpenTelemetryApi", package: "opentelemetry-swift"),
-                    .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift")
+                    .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift"),
+                    .product(name: "DatadogExporter", package: "opentelemetry-swift")
                 ],
-                swiftSettings: [.unsafeFlags(["-DOTEL_SWIFT"])]
+                swiftSettings: [.define("OTEL_SWIFT")]
             )
         ]
 
@@ -44,11 +45,11 @@ func addOpenTelemetryDependency(_ version: Version) {
 
         package.targets = [
             .target(
-                name: "Benchmarks",
+                name: "DatadogBenchmarks",
                 dependencies: [
                     .product(name: "OpenTelemetryApi", package: "opentelemetry-swift-packages")
                 ],
-                swiftSettings: [.unsafeFlags(["-DOTEL_API"])]
+                swiftSettings: [.define("OTEL_API")]
             )
         ]
     }

BenchmarkTests/Benchmarks/Sources/Benchmarks.swift

@@ -0,0 +1,171 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-Present Datadog, Inc.
+ */
+
+#if OTEL_API
+#error("Benchmarks depends on opentelemetry-swift. Please open the project with 'make benchmark-tests-open'.")
+#endif
+
+import Foundation
+import OpenTelemetryApi
+import OpenTelemetrySdk
+import DatadogExporter
+
+let instrumentationName = "benchmarks"
+let instrumentationVersion = "1.0.0"
+
+/// Benchmark entrypoint to configure opentelemetry with metrics meters
+/// and tracer.
+public enum Benchmarks {
+    /// Configuration of the Benchmarks library.
+    public struct Configuration {
+        /// Context of Benchmarks measures.
+        /// The context properties will be added to metrics as tags.
+        public struct Context {
+            var applicationIdentifier: String
+            var applicationName: String
+            var applicationVersion: String
+            var sdkVersion: String
+            var deviceModel: String
+            var osName: String
+            var osVersion: String
+            var run: String
+            var scenario: String
+            var branch: String
+
+            public init(
+                applicationIdentifier: String,
+                applicationName: String,
+                applicationVersion: String,
+                sdkVersion: String,
+                deviceModel: String,
+                osName: String,
+                osVersion: String,
+                run: String,
+                scenario: String,
+                branch: String
+            ) {
+                self.applicationIdentifier = applicationIdentifier
+                self.applicationName = applicationName
+                self.applicationVersion = applicationVersion
+                self.sdkVersion = sdkVersion
+                self.deviceModel = deviceModel
+                self.osName = osName
+                self.osVersion = osVersion
+                self.run = run
+                self.scenario = scenario
+                self.branch = branch
+            }
+        }
+
+        var clientToken: String
+        var apiKey: String
+        var context: Context
+
+        public init(
+            clientToken: String,
+            apiKey: String,
+            context: Context
+        ) {
+            self.clientToken = clientToken
+            self.apiKey = apiKey
+            self.context = context
+        }
+    }
+
+    /// Configure OpenTelemetry metrics meter and start measuring Memory.
+    ///
+    /// - Parameter configuration: The Benchmark configuration.
+    public static func enableMetrics(with configuration: Configuration) {
+        let metricExporter = MetricExporter(
+            configuration: MetricExporter.Configuration(
+                apiKey: configuration.apiKey,
+                version: instrumentationVersion
+            )
+        )
+
+        let meterProvider = MeterProviderBuilder()
+            .with(pushInterval: 10)
+            .with(processor: MetricProcessorSdk())
+            .with(exporter: metricExporter)
+            .with(resource: Resource())
+            .build()
+
+        let meter = meterProvider.get(
+            instrumentationName: instrumentationName,
+            instrumentationVersion: instrumentationVersion
+        )
+
+        let labels = [
+            "device_model": configuration.context.deviceModel,
+            "os": configuration.context.osName,
+            "os_version": configuration.context.osVersion,
+            "run": configuration.context.run,
+            "scenario": configuration.context.scenario,
+            "application_id": configuration.context.applicationIdentifier,
+            "sdk_version": configuration.context.sdkVersion,
+            "branch": configuration.context.branch,
+        ]
+
+        let queue = DispatchQueue(label: "com.datadoghq.benchmarks.metrics", qos: .utility)
+
+        let memory = Memory(queue: queue)
+        _ = meter.createDoubleObservableGauge(name: "ios.benchmark.memory") { metric in
+            // report the maximum memory footprint that was recorded during push interval
+            if let value = memory.aggregation?.max {
+                metric.observe(value: value, labels: labels)
+            }
+
+            memory.reset()
+        }
+
+        let cpu = CPU(queue: queue)
+        _ = meter.createDoubleObservableGauge(name: "ios.benchmark.cpu") { metric in
+            // report the average cpu usage that was recorded during push interval
+            if let value = cpu.aggregation?.avg {
+                metric.observe(value: value, labels: labels)
+            }
+
+            cpu.reset()
+        }
+
+        let fps = FPS()
+        _ = meter.createIntObservableGauge(name: "ios.benchmark.fps.min") { metric in
+            // report the minimum frame rate that was recorded during push interval
+            if let value = fps.aggregation?.min {
+                metric.observe(value: value, labels: labels)
+            }
+
+            fps.reset()
+        }
+
+        OpenTelemetry.registerMeterProvider(meterProvider: meterProvider)
+    }
+
+    /// Configure and register a OpenTelemetry Tracer.
+    ///
+    /// - Parameter configuration: The Benchmark configuration.
+    public static func enableTracer(with configuration: Configuration) {
+        let exporterConfiguration = ExporterConfiguration(
+            serviceName: configuration.context.applicationIdentifier,
+            resource: "Benchmark Tracer",
+            applicationName: configuration.context.applicationName,
+            applicationVersion: configuration.context.applicationVersion,
+            environment: "benchmarks",
+            apiKey: configuration.apiKey,
+            endpoint: .us1,
+            uploadCondition: { true }
+        )
+
+        let exporter = try! DatadogExporter(config: exporterConfiguration)
+        let processor = SimpleSpanProcessor(spanExporter: exporter)
+
+        let provider = TracerProviderBuilder()
+            .add(spanProcessor: processor)
+            .build()
+
+        OpenTelemetry.registerTracerProvider(tracerProvider: provider)
+    }
+}