Merge pull request #2282 from DataDog/release/2.27.0

Release `2.27.0`

Co-authored-by: maxep <maxime.epain@datadoghq.com>
Co-authored-by: dd-mergequeue[bot] <121105855+dd-mergequeue[bot]@users.noreply.github.com>
Co-authored-by: mariedm <marie.denis@datadoghq.com>
Co-authored-by: cdn34dd <carlos.nogueira@datadoghq.com>
Co-authored-by: simaoseica-dd <simao.seica@datadoghq.com>
Co-authored-by: xgouchet <xgouchet@users.noreply.github.com>
Co-authored-by: ncreated <maciek.grzybowski@datadoghq.com>

Author: 7 people

BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj

@@ -83,7 +83,7 @@ public enum Benchmarks {
             )
         )
 
-        let provider = MeterProviderBuilder()
+        return MeterProviderBuilder()
             .with(pushInterval: 10)
             .with(processor: MetricProcessorSdk())
             .with(exporter: metricExporter)
@@ -98,9 +98,6 @@ public enum Benchmarks {
                 "branch": .string(configuration.context.branch),
             ]))
             .build()
-
-        OpenTelemetry.registerMeterProvider(meterProvider: provider)
-        return provider
     }
 
     /// Configure an OpenTelemetry tracer provider.
@@ -121,11 +118,8 @@ public enum Benchmarks {
         let exporter = try! DatadogExporter(config: exporterConfiguration)
         let processor = SimpleSpanProcessor(spanExporter: exporter)
 
-        let provider = TracerProviderBuilder()
+        return TracerProviderBuilder()
             .add(spanProcessor: processor)
             .build()
-
-        OpenTelemetry.registerTracerProvider(tracerProvider: provider)
-        return provider
     }
 }

BenchmarkTests/Benchmarks/Sources/Benchmarks.swift

@@ -10,51 +10,6 @@ To open the project with the correct environment, make sure Xcode is closed and
 make benchmark-tests-open
 ```
 
-> [!TIP]
-> If Xcode is failing to install packages with a message such as **failed to clone**, **requires authentication** or **dependency failed**, it might be due to a Git configuration that forces HTTPS URLs to be rewritten as SSH. In that case, try commenting the following lines in your global Git config (~/.gitconfig).
-
-```bash
-[url "ssh://git@github.com/"]
-    insteadOf = https://github.com/
-```
-
-> Or remove it with the following command:
-
-```bash
-git config --global --unset url."ssh://git@github.com/".insteadOf
-```
-
-## CI
-
-CI continuously builds, signs, and uploads a runner application to Synthetics, which runs predefined tests.
-
-### Build
-
-Before building the application, make sure the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file is present and contains the `Mobile - Integration Org` client token, RUM application ID, and API Key. These values are sensitive and must be securely stored.
-
-```ini
-CLIENT_TOKEN=
-RUM_APPLICATION_ID=
-API_KEY=
-```
-
-### Sign
-
-To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. The certificate and profile are sensitive files and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails.
-
-> [!NOTE]
-> Certificate & Provisioning Profile are also available through the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi). But we don't have the tooling in place.
-
-### Upload
-
-The application version (build number) is set to the commit SHA of the current job, and the build is uploaded to Synthetics using the [datadog-ci](https://github.com/DataDog/datadog-ci) CLI. This step expects environment variables to authenticate with the `Mobile - Integration Org`:
-
-```bash
-export DATADOG_API_KEY=
-export DATADOG_APP_KEY=
-export S8S_APPLICATION_ID=
-```
-
 ## Development
 
 Each scenario is independent and can be considered as an app within the runner.
@@ -93,8 +48,66 @@ struct LogsScenario: Scenario {
 }
 ```
 
-Add the test to the [`SyntheticScenario`](Runner/Scenarios/SyntheticScenario.swift#L12) object so it can be selected by setting the `BENCHMARK_SCENARIO` environment variable.
+Once the scenario is created, add its name as an enum case of the [`SyntheticScenario`](Runner/Scenarios/SyntheticScenario.swift#L12) object so it can be selected by setting the `BENCHMARK_SCENARIO` environment variable.
+
+### Run a scenario
+
+There are two ways to execute a scenario:
+1. Use environment variables in the `Runner.xcscheme` (Edit Scheme from Xcode)
+```xml
+<EnvironmentVariables>
+    <EnvironmentVariable
+        key = "BENCHMARK_RUN"
+        value = "<run>"
+        isEnabled = "YES">
+    </EnvironmentVariable>
+    <EnvironmentVariable
+        key = "BENCHMARK_SCENARIO"
+        value = "<scenario>"
+        isEnabled = "YES">
+    </EnvironmentVariable>
+</EnvironmentVariables>
+```
+2. Open a deep link:
+```bash
+xcrun simctl openurl booted 'bench://start?scenario=<scenario>&run=<run>'
+```
+If you need to execute multiple scenarios/runs in the same application process, you must stop the previous execution with: 
+```bash
+xcrun simctl openurl booted 'bench://stop'
+```
 
 ### Synthetics Configuration
 
-Please refer to [Confluence page (internal)](https://datadoghq.atlassian.net/wiki/spaces/RUMP/pages/3981476482/Benchmarks+iOS)
+Please refer to the [Confluence page (internal)](https://datadoghq.atlassian.net/wiki/spaces/RUMP/pages/3981476482/Benchmarks+iOS)
+
+## CI
+
+CI continuously builds, signs, and uploads a runner application to Synthetics, which runs predefined tests.
+
+### Build
+
+Before building the application, make sure the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file is present and contains the `Mobile - Integration Org` client token, RUM application ID, and API Key. These values are sensitive and must be securely stored.
+
+```ini
+CLIENT_TOKEN=
+RUM_APPLICATION_ID=
+API_KEY=
+```
+
+### Sign
+
+To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. The certificate and profile are sensitive files and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails.
+
+> [!NOTE]
+> Certificate & Provisioning Profile are also available through the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi). But we don't have the tooling in place.
+
+### Upload
+
+The application version (build number) is set to the commit SHA of the current job, and the build is uploaded to Synthetics using the [datadog-ci](https://github.com/DataDog/datadog-ci) CLI. This step expects environment variables to authenticate with the `Mobile - Integration Org`:
+
+```bash
+export DATADOG_API_KEY=
+export DATADOG_APP_KEY=
+export S8S_APPLICATION_ID=
+```

BenchmarkTests/README.md

@@ -5,51 +5,67 @@
  */
 
 import UIKit
+
 import DatadogInternal
+import DatadogCore
 import DatadogBenchmarks
 
 @main
 class AppDelegate: UIResponder, UIApplicationDelegate {
     var window: UIWindow?
+    var applicationInfo: AppInfo! //swiftlint:disable:this implicitly_unwrapped_optional
+    var vitals: Vitals?
 
     func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
-        guard let scenario = SyntheticScenario() else {
+        applicationInfo = try! AppInfo() // crash if info are missing or malformed
+
+        window = UIWindow(frame: UIScreen.main.bounds)
+
+        if let scenario = SyntheticScenario() {
+            let run = SyntheticRun()
+            start(scenario: scenario, run: run)
+        } else {
+            window?.rootViewController = UIViewController()
+        }
+
+        window?.makeKeyAndVisible()
+        return true
+    }
+
+    func application(_ app: UIApplication, open url: URL, options: [UIApplication.OpenURLOptionsKey: Any] = [:]) -> Bool {
+        guard let components = URLComponents(url: url, resolvingAgainstBaseURL: false) else {
             return false
         }
 
-        let run = SyntheticRun()
-        let applicationInfo = try! AppInfo() // crash if info are missing or malformed
+        // bench://stop
+        if components.host == "stop" {
+            stop()
+            return true
+        }
 
-        // Collect metrics during all run
-        let meter = Meter(
-            provider: Benchmarks.meterProvider(
-                with: Benchmarks.Configuration(
-                    info: applicationInfo,
-                    scenario: scenario,
-                    run: run
-                )
-            )
-        )
+        // bench://start?scenario=<scenario>&run=<run>
+        if components.host == "start", let scenario = SyntheticScenario(urlComponents: components), let run = SyntheticRun(urlComponents: components) {
+            start(scenario: scenario, run: run)
+            return true
+        }
+
+        return false
+    }
 
+    /// Starts instruments for the given run and scenario.
+    ///
+    /// - Parameters:
+    ///   - scenario: The benchmark scenario.
+    ///   - run: The benchmark run.
+    private func start(
+        scenario: SyntheticScenario,
+        run: SyntheticRun
+    ) {
         switch run {
         case .baseline, .instrumented:
-            meter.observeCPU()
-            meter.observeMemory()
-            meter.observeFPS()
-
+            collectApplicationVitals(scenario: scenario, run: run)
         case .profiling:
-            // Collect traces during profiling run
-            let profiler = Profiler(
-                provider: Benchmarks.tracerProvider(
-                    with: Benchmarks.Configuration(
-                        info: applicationInfo,
-                        scenario: scenario,
-                        run: run
-                    )
-                )
-            )
-
-            DatadogInternal.bench = (profiler, meter)
+            profileSDK(scenario: scenario, run: run)
         case .none:
             break
         }
@@ -60,11 +76,77 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
             scenario.instrument(with: applicationInfo)
         }
 
-        window = UIWindow(frame: UIScreen.main.bounds)
         window?.rootViewController = scenario.initialViewController
-        window?.makeKeyAndVisible()
+    }
 
-        return true
+    /// Stops all current instruments.
+    ///
+    /// The same process can run multiple scenarios and instruments when receiving a deeplink.
+    /// It is important to stop current instruments before starting a new run.
+    private func stop() {
+        vitals = nil // stop collecting vitals
+        Datadog.stopInstance() // stop runner instrumentation
+        DatadogInternal.bench = (NOPBench(), NOPBench()) // stop profiling the sdk
+        window?.rootViewController = UIViewController()
+    }
+
+    /// Starts collection vitals of the runner application.
+    ///
+    /// - Parameters:
+    ///   - scenario: The benchmark scenario.
+    ///   - run: The benchmark run.
+    private func collectApplicationVitals(
+        scenario: SyntheticScenario,
+        run: SyntheticRun
+    ) {
+        let vitals = Vitals(
+            provider: Benchmarks.meterProvider(
+                with: Benchmarks.Configuration(
+                    info: applicationInfo,
+                    scenario: scenario,
+                    run: run
+                )
+            )
+        )
+
+        vitals.observeCPU()
+        vitals.observeMemory()
+        vitals.observeFPS()
+
+        self.vitals = vitals // Keep vitals in memory
+    }
+
+    /// Starts profiling the SDK while running the application.
+    ///
+    /// - Parameters:
+    ///   - scenario: The benchmark scenario.
+    ///   - run: The benchmark run.
+    private func profileSDK(
+        scenario: SyntheticScenario,
+        run: SyntheticRun
+    ) {
+        // Collect traces during profiling run
+        let profiler = Profiler(
+            provider: Benchmarks.tracerProvider(
+                with: Benchmarks.Configuration(
+                    info: applicationInfo,
+                    scenario: scenario,
+                    run: run
+                )
+            )
+        )
+
+        let meter = Meter(
+            provider: Benchmarks.meterProvider(
+                with: Benchmarks.Configuration(
+                    info: applicationInfo,
+                    scenario: scenario,
+                    run: run
+                )
+            )
+        )
+
+        DatadogInternal.bench = (profiler, meter) // Inject profiler and meter to collect telemetry
     }
 }
 

BenchmarkTests/Runner/AppDelegate.swift

@@ -8,59 +8,17 @@ import Foundation
 import DatadogInternal
 import DatadogBenchmarks
 import OpenTelemetryApi
-import OpenTelemetrySdk
 
 internal final class Meter: DatadogInternal.BenchmarkMeter {
     let meter: OpenTelemetryApi.Meter
 
-    let queue = DispatchQueue(label: "com.datadoghq.benchmarks.metrics", target: .global(qos: .utility))
-
     init(provider: MeterProvider) {
         self.meter = provider.get(
             instrumentationName: "benchmarks",
             instrumentationVersion: nil
         )
     }
 
-    @discardableResult
-    func observeMemory() -> OpenTelemetryApi.DoubleObserverMetric {
-        let memory = Memory(queue: queue)
-        return meter.createDoubleObservableGauge(name: "ios.benchmark.memory") { metric in
-            // report the maximum memory footprint that was recorded during push interval
-            if let value = memory.aggregation?.max {
-                metric.observe(value: value, labelset: .empty)
-            }
-
-            memory.reset()
-        }
-    }
-
-    @discardableResult
-    func observeCPU() -> OpenTelemetryApi.DoubleObserverMetric {
-        let cpu = CPU(queue: queue)
-        return meter.createDoubleObservableGauge(name: "ios.benchmark.cpu") { metric in
-            // report the average cpu usage that was recorded during push interval
-            if let value = cpu.aggregation?.avg {
-                metric.observe(value: value, labelset: .empty)
-            }
-
-            cpu.reset()
-        }
-    }
-
-    @discardableResult
-    func observeFPS() -> OpenTelemetryApi.IntObserverMetric {
-        let fps = FPS()
-        return meter.createIntObservableGauge(name: "ios.benchmark.fps.min") { metric in
-            // report the minimum frame rate that was recorded during push interval
-            if let value = fps.aggregation?.min {
-                metric.observe(value: value, labelset: .empty)
-            }
-
-            fps.reset()
-        }
-    }
-
     func counter(metric: @autoclosure () -> String) -> DatadogInternal.BenchmarkCounter {
         meter.createDoubleCounter(name: metric())
     }

BenchmarkTests/Runner/BenchmarkMeter.swift

@@ -8,7 +8,6 @@ import Foundation
 import DatadogInternal
 import DatadogBenchmarks
 import OpenTelemetryApi
-import OpenTelemetrySdk
 
 internal final class Profiler: DatadogInternal.BenchmarkProfiler {
     let provider: TracerProvider