internal: (studio) add telemetry around time it takes to download the bundle and initialize studio fully

packages/server/lib/cloud/studio/StudioLifecycleManager.ts

@@ -18,6 +18,10 @@ import { ensureStudioBundle } from './ensure_studio_bundle'
 import chokidar from 'chokidar'
 import { readFile } from 'fs/promises'
 import { getCloudMetadata } from '../get_cloud_metadata'
+import { initializeTelemetryReporter, reportTelemetry } from './telemetry/TelemetryReporter'
+import { telemetryManager } from './telemetry/TelemetryManager'
+import { BUNDLE_LIFECYCLE_MARK_NAMES, BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES } from './telemetry/constants/bundle-lifecycle'
+import { INITIALIZATION_TELEMETRY_GROUP_NAMES } from './telemetry/constants/initialization'
 
 const debug = Debug('cypress:server:studio-lifecycle-manager')
 const routes = require('../routes')
@@ -98,6 +102,11 @@ export class StudioLifecycleManager {
       // Clean up any registered listeners
       this.listeners = []
 
+      telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END)
+      reportTelemetry(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, {
+        success: false,
+      })
+
       return null
     })
 
@@ -112,6 +121,12 @@ export class StudioLifecycleManager {
   }
 
   isStudioReady (): boolean {
+    if (!this.studioManager) {
+      telemetryManager.addGroupMetadata(INITIALIZATION_TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, {
+        studioRequestedBeforeReady: true,
+      })
+    }
+
     return !!this.studioManager
   }
 
@@ -143,10 +158,21 @@ export class StudioLifecycleManager {
     let studioPath: string
     let studioHash: string
 
+    initializeTelemetryReporter({
+      projectSlug: projectId,
+      cloudDataSource,
+    })
+
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_START)
+
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_START)
     const studioSession = await postStudioSession({
       projectId,
     })
 
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_END)
+
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_START)
     if (!process.env.CYPRESS_LOCAL_STUDIO_PATH) {
       // The studio hash is the last part of the studio URL, after the last slash and before the extension
       studioHash = studioSession.studioUrl.split('/').pop()?.split('.')[0]
@@ -170,11 +196,15 @@ export class StudioLifecycleManager {
       studioHash = 'local'
     }
 
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_END)
+
     const serverFilePath = path.join(studioPath, 'server', 'index.js')
 
     const script = await readFile(serverFilePath, 'utf8')
     const studioManager = new StudioManager()
 
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_START)
+
     const { cloudUrl, cloudHeaders } = await getCloudMetadata(cloudDataSource)
 
     await studioManager.setup({
@@ -192,11 +222,18 @@ export class StudioLifecycleManager {
       shouldEnableStudio: this.cloudStudioRequested,
     })
 
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_END)
+
     if (studioManager.status === 'ENABLED') {
       debug('Cloud studio is enabled - setting up protocol')
       const protocolManager = new ProtocolManager()
+
+      telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_START)
       const script = await api.getCaptureProtocolScript(studioSession.protocolUrl)
 
+      telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_END)
+
+      telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_START)
       await protocolManager.prepareProtocol(script, {
         runId: 'studio',
         projectId: cfg.projectId,
@@ -212,6 +249,8 @@ export class StudioLifecycleManager {
         mode: 'studio',
       })
 
+      telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_END)
+
       studioManager.protocolManager = protocolManager
     } else {
       debug('Cloud studio is not enabled - skipping protocol setup')
@@ -222,6 +261,11 @@ export class StudioLifecycleManager {
     this.callRegisteredListeners()
     this.updateStatus(studioManager.status)
 
+    telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END)
+    reportTelemetry(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, {
+      success: true,
+    })
+
     return studioManager
   }
 

packages/server/lib/cloud/studio/telemetry/TelemetryManager.ts

@@ -0,0 +1,111 @@
+import { performance } from 'perf_hooks'
+import { BUNDLE_LIFECYCLE_MARK_NAMES, BUNDLE_LIFECYCLE_MEASURE_NAMES, BUNDLE_LIFECYCLE_MEASURES, BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES, BUNDLE_LIFECYCLE_TELEMETRY_GROUPS } from './constants/bundle-lifecycle'
+import { INITIALIZATION_MARK_NAMES, INITIALIZATION_MEASURE_NAMES, INITIALIZATION_MEASURES, INITIALIZATION_TELEMETRY_GROUP_NAMES, INITIALIZATION_TELEMETRY_GROUPS } from './constants/initialization'
+
+export const MARK_NAMES = Object.freeze({
+  ...BUNDLE_LIFECYCLE_MARK_NAMES,
+  ...INITIALIZATION_MARK_NAMES,
+} as const)
+
+type MarkName = (typeof MARK_NAMES)[keyof typeof MARK_NAMES]
+
+export const MEASURE_NAMES = Object.freeze({
+  ...BUNDLE_LIFECYCLE_MEASURE_NAMES,
+  ...INITIALIZATION_MEASURE_NAMES,
+} as const)
+
+type MeasureName = (typeof MEASURE_NAMES)[keyof typeof MEASURE_NAMES]
+
+const MEASURES: Record<MeasureName, [MarkName, MarkName]> = Object.freeze({
+  ...BUNDLE_LIFECYCLE_MEASURES,
+  ...INITIALIZATION_MEASURES,
+} as const)
+
+export const TELEMETRY_GROUP_NAMES = Object.freeze({
+  ...BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES,
+  ...INITIALIZATION_TELEMETRY_GROUP_NAMES,
+} as const)
+
+export const TELEMETRY_GROUPS = Object.freeze({
+  ...BUNDLE_LIFECYCLE_TELEMETRY_GROUPS,
+  ...INITIALIZATION_TELEMETRY_GROUPS,
+} as const)
+
+export type TelemetryGroupName = keyof typeof TELEMETRY_GROUPS
+
+class TelemetryManager {
+  private static instance: TelemetryManager
+  private groupMetadata: Partial<Record<TelemetryGroupName, Record<string, unknown>>> = {}
+
+  private constructor () {}
+
+  public static getInstance (): TelemetryManager {
+    if (!TelemetryManager.instance) {
+      TelemetryManager.instance = new TelemetryManager()
+    }
+
+    return TelemetryManager.instance
+  }
+
+  public mark (name: MarkName) {
+    performance.mark(name)
+  }
+
+  public getMeasure (measureName: MeasureName): number {
+    const [startMark, endMark] = MEASURES[measureName]
+
+    try {
+      const measure = performance.measure(measureName, startMark, endMark)
+
+      return measure?.duration ?? -1
+    } catch (error) {
+      return -1
+    }
+  }
+
+  public getMeasures (
+    names: MeasureName[],
+    clear: boolean = false,
+  ): Partial<Record<MeasureName, number>> {
+    const result: Partial<Record<MeasureName, number>> = {}
+
+    for (const name of names) {
+      result[name] = this.getMeasure(name)
+    }
+    if (clear) {
+      this.clearMeasures(names)
+    }
+
+    return result
+  }
+
+  public clearMeasureGroup (groupName: TelemetryGroupName) {
+    const measures = TELEMETRY_GROUPS[groupName]
+
+    this.clearMeasures(measures)
+  }
+
+  public clearMeasures (names: MeasureName[]) {
+    for (const name of names) {
+      performance.clearMeasures(name)
+      performance.clearMarks(MEASURES[name][0])
+      performance.clearMarks(MEASURES[name][1])
+    }
+  }
+
+  public addGroupMetadata (groupName: TelemetryGroupName, metadata: Record<string, unknown>) {
+    this.groupMetadata[groupName] = this.groupMetadata[groupName] || {}
+    this.groupMetadata[groupName] = {
+      ...this.groupMetadata[groupName],
+      ...metadata,
+    }
+  }
+
+  public reset () {
+    performance.clearMarks()
+    performance.clearMeasures()
+    this.groupMetadata = {}
+  }
+}
+
+export const telemetryManager = TelemetryManager.getInstance()

packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts

@@ -0,0 +1,113 @@
+import Debug from 'debug'
+import {
+  TELEMETRY_GROUPS,
+  TelemetryGroupName,
+  telemetryManager,
+} from './TelemetryManager'
+import type { CloudDataSource } from '@packages/data-context/src/sources/CloudDataSource'
+import { CloudRequest } from '../../api/cloud_request'
+import { getCloudMetadata } from '../../get_cloud_metadata'
+
+const debug = Debug('cypress:server:cloud:studio:telemetry:reporter')
+
+interface TelemetryReporterOptions {
+  projectSlug?: string
+  cloudDataSource: CloudDataSource
+}
+
+export class TelemetryReporter {
+  private static instance: TelemetryReporter
+  private projectSlug?: string
+  private cloudDataSource: CloudDataSource
+
+  private constructor ({
+    projectSlug,
+    cloudDataSource,
+  }: TelemetryReporterOptions) {
+    this.projectSlug = projectSlug
+    this.cloudDataSource = cloudDataSource
+  }
+
+  public static initialize (options: TelemetryReporterOptions): void {
+    if (TelemetryReporter.instance) {
+      // initialize gets called multiple times (e.g. if you switch between projects)
+      // we need to reset the telemetry manager to avoid accumulating measures
+      telemetryManager.reset()
+    }
+
+    TelemetryReporter.instance = new TelemetryReporter(options)
+  }
+
+  public static getInstance (): TelemetryReporter {
+    if (!TelemetryReporter.instance) {
+      throw new Error('TelemetryReporter not initialized')
+    }
+
+    return TelemetryReporter.instance
+  }
+
+  public reportTelemetry (
+    telemetryGroupName: TelemetryGroupName,
+    metadata?: Record<string, unknown>,
+  ): void {
+    this._reportTelemetry(telemetryGroupName, metadata).catch((e: unknown) => {
+      debug(
+        'Error reporting telemetry to cloud: %o, original telemetry: %s',
+        e,
+        telemetryGroupName,
+      )
+    })
+  }
+
+  private async _reportTelemetry (
+    telemetryGroupName: TelemetryGroupName,
+    metadata?: Record<string, unknown>,
+  ): Promise<void> {
+    debug('Reporting telemetry for group: %s', telemetryGroupName)
+
+    try {
+      const groupMeasures = [...TELEMETRY_GROUPS[telemetryGroupName]]
+      const measures = telemetryManager.getMeasures(groupMeasures)
+
+      const payload = {
+        projectSlug: this.projectSlug,
+        telemetryGroupName,
+        measures,
+        metadata,
+      }
+
+      const { cloudUrl, cloudHeaders } = await getCloudMetadata(this.cloudDataSource)
+
+      await CloudRequest.post(
+        `${cloudUrl}/studio/telemetry`,
+        payload,
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            ...cloudHeaders,
+          },
+        },
+      )
+    } catch (e: unknown) {
+      debug(
+        'Error reporting telemetry to cloud: %o, original telemetry: %s',
+        e,
+        telemetryGroupName,
+      )
+    }
+  }
+}
+
+export const initializeTelemetryReporter = (
+  options: TelemetryReporterOptions,
+) => {
+  TelemetryReporter.initialize(options)
+}
+
+export const reportTelemetry = (
+  telemetryGroupName: TelemetryGroupName,
+  metadata?: Record<string, unknown>,
+) => {
+  TelemetryReporter.getInstance().reportTelemetry(telemetryGroupName, metadata)
+  telemetryManager.clearMeasureGroup(telemetryGroupName)
+}