From 7d87bc45f823e488f87e8c71fc28da98786619d0 Mon Sep 17 00:00:00 2001 From: Ryan Manuel Date: Thu, 5 Jun 2025 16:38:57 -0500 Subject: [PATCH 1/2] internal: (studio) add telemetry around time it takes to download the bundle and initialize studio fully --- .../cloud/studio/StudioLifecycleManager.ts | 44 ++ .../studio/telemetry/TelemetryManager.ts | 111 ++++ .../studio/telemetry/TelemetryReporter.ts | 113 ++++ .../telemetry/constants/bundle-lifecycle.ts | 71 +++ .../telemetry/constants/initialization.ts | 55 ++ packages/server/lib/project-base.ts | 160 +++-- .../studio/StudioLifecycleManager_spec.ts | 91 +++ .../studio/telemetry/TelemetryManager_spec.ts | 314 ++++++++++ .../telemetry/TelemetryReporter_spec.ts | 202 +++++++ packages/server/test/unit/project_spec.js | 560 ++++++++++-------- 10 files changed, 1408 insertions(+), 313 deletions(-) create mode 100644 packages/server/lib/cloud/studio/telemetry/TelemetryManager.ts create mode 100644 packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts create mode 100644 packages/server/lib/cloud/studio/telemetry/constants/bundle-lifecycle.ts create mode 100644 packages/server/lib/cloud/studio/telemetry/constants/initialization.ts create mode 100644 packages/server/test/unit/cloud/studio/telemetry/TelemetryManager_spec.ts create mode 100644 packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts diff --git a/packages/server/lib/cloud/studio/StudioLifecycleManager.ts b/packages/server/lib/cloud/studio/StudioLifecycleManager.ts index ed96964c6577..61d91f8f7f9a 100644 --- a/packages/server/lib/cloud/studio/StudioLifecycleManager.ts +++ b/packages/server/lib/cloud/studio/StudioLifecycleManager.ts @@ -18,6 +18,10 @@ import { ensureStudioBundle } from './ensure_studio_bundle' import chokidar from 'chokidar' import { readFile } from 'fs/promises' import { getCloudMetadata } from '../get_cloud_metadata' +import { initializeTelemetryReporter, reportTelemetry } from './telemetry/TelemetryReporter' +import { telemetryManager } from './telemetry/TelemetryManager' +import { BUNDLE_LIFECYCLE_MARK_NAMES, BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES } from './telemetry/constants/bundle-lifecycle' +import { INITIALIZATION_TELEMETRY_GROUP_NAMES } from './telemetry/constants/initialization' const debug = Debug('cypress:server:studio-lifecycle-manager') const routes = require('../routes') @@ -98,6 +102,11 @@ export class StudioLifecycleManager { // Clean up any registered listeners this.listeners = [] + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END) + reportTelemetry(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, { + success: false, + }) + return null }) @@ -112,6 +121,12 @@ export class StudioLifecycleManager { } isStudioReady (): boolean { + if (!this.studioManager) { + telemetryManager.addGroupMetadata(INITIALIZATION_TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + studioRequestedBeforeReady: true, + }) + } + return !!this.studioManager } @@ -143,10 +158,21 @@ export class StudioLifecycleManager { let studioPath: string let studioHash: string + initializeTelemetryReporter({ + projectSlug: projectId, + cloudDataSource, + }) + + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_START) + + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_START) const studioSession = await postStudioSession({ projectId, }) + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_END) + + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_START) if (!process.env.CYPRESS_LOCAL_STUDIO_PATH) { // The studio hash is the last part of the studio URL, after the last slash and before the extension studioHash = studioSession.studioUrl.split('/').pop()?.split('.')[0] @@ -170,11 +196,15 @@ export class StudioLifecycleManager { studioHash = 'local' } + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_END) + const serverFilePath = path.join(studioPath, 'server', 'index.js') const script = await readFile(serverFilePath, 'utf8') const studioManager = new StudioManager() + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_START) + const { cloudUrl, cloudHeaders } = await getCloudMetadata(cloudDataSource) await studioManager.setup({ @@ -192,11 +222,18 @@ export class StudioLifecycleManager { shouldEnableStudio: this.cloudStudioRequested, }) + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_END) + if (studioManager.status === 'ENABLED') { debug('Cloud studio is enabled - setting up protocol') const protocolManager = new ProtocolManager() + + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_START) const script = await api.getCaptureProtocolScript(studioSession.protocolUrl) + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_END) + + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_START) await protocolManager.prepareProtocol(script, { runId: 'studio', projectId: cfg.projectId, @@ -212,6 +249,8 @@ export class StudioLifecycleManager { mode: 'studio', }) + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_END) + studioManager.protocolManager = protocolManager } else { debug('Cloud studio is not enabled - skipping protocol setup') @@ -222,6 +261,11 @@ export class StudioLifecycleManager { this.callRegisteredListeners() this.updateStatus(studioManager.status) + telemetryManager.mark(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END) + reportTelemetry(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, { + success: true, + }) + return studioManager } diff --git a/packages/server/lib/cloud/studio/telemetry/TelemetryManager.ts b/packages/server/lib/cloud/studio/telemetry/TelemetryManager.ts new file mode 100644 index 000000000000..147a78b6b6cd --- /dev/null +++ b/packages/server/lib/cloud/studio/telemetry/TelemetryManager.ts @@ -0,0 +1,111 @@ +import { performance } from 'perf_hooks' +import { BUNDLE_LIFECYCLE_MARK_NAMES, BUNDLE_LIFECYCLE_MEASURE_NAMES, BUNDLE_LIFECYCLE_MEASURES, BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES, BUNDLE_LIFECYCLE_TELEMETRY_GROUPS } from './constants/bundle-lifecycle' +import { INITIALIZATION_MARK_NAMES, INITIALIZATION_MEASURE_NAMES, INITIALIZATION_MEASURES, INITIALIZATION_TELEMETRY_GROUP_NAMES, INITIALIZATION_TELEMETRY_GROUPS } from './constants/initialization' + +export const MARK_NAMES = Object.freeze({ + ...BUNDLE_LIFECYCLE_MARK_NAMES, + ...INITIALIZATION_MARK_NAMES, +} as const) + +type MarkName = (typeof MARK_NAMES)[keyof typeof MARK_NAMES] + +export const MEASURE_NAMES = Object.freeze({ + ...BUNDLE_LIFECYCLE_MEASURE_NAMES, + ...INITIALIZATION_MEASURE_NAMES, +} as const) + +type MeasureName = (typeof MEASURE_NAMES)[keyof typeof MEASURE_NAMES] + +const MEASURES: Record = Object.freeze({ + ...BUNDLE_LIFECYCLE_MEASURES, + ...INITIALIZATION_MEASURES, +} as const) + +export const TELEMETRY_GROUP_NAMES = Object.freeze({ + ...BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES, + ...INITIALIZATION_TELEMETRY_GROUP_NAMES, +} as const) + +export const TELEMETRY_GROUPS = Object.freeze({ + ...BUNDLE_LIFECYCLE_TELEMETRY_GROUPS, + ...INITIALIZATION_TELEMETRY_GROUPS, +} as const) + +export type TelemetryGroupName = keyof typeof TELEMETRY_GROUPS + +class TelemetryManager { + private static instance: TelemetryManager + private groupMetadata: Partial>> = {} + + private constructor () {} + + public static getInstance (): TelemetryManager { + if (!TelemetryManager.instance) { + TelemetryManager.instance = new TelemetryManager() + } + + return TelemetryManager.instance + } + + public mark (name: MarkName) { + performance.mark(name) + } + + public getMeasure (measureName: MeasureName): number { + const [startMark, endMark] = MEASURES[measureName] + + try { + const measure = performance.measure(measureName, startMark, endMark) + + return measure?.duration ?? -1 + } catch (error) { + return -1 + } + } + + public getMeasures ( + names: MeasureName[], + clear: boolean = false, + ): Partial> { + const result: Partial> = {} + + for (const name of names) { + result[name] = this.getMeasure(name) + } + if (clear) { + this.clearMeasures(names) + } + + return result + } + + public clearMeasureGroup (groupName: TelemetryGroupName) { + const measures = TELEMETRY_GROUPS[groupName] + + this.clearMeasures(measures) + } + + public clearMeasures (names: MeasureName[]) { + for (const name of names) { + performance.clearMeasures(name) + performance.clearMarks(MEASURES[name][0]) + performance.clearMarks(MEASURES[name][1]) + } + } + + public addGroupMetadata (groupName: TelemetryGroupName, metadata: Record) { + this.groupMetadata[groupName] = this.groupMetadata[groupName] || {} + this.groupMetadata[groupName] = { + ...this.groupMetadata[groupName], + ...metadata, + } + } + + public reset () { + performance.clearMarks() + performance.clearMeasures() + this.groupMetadata = {} + } +} + +export const telemetryManager = TelemetryManager.getInstance() diff --git a/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts b/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts new file mode 100644 index 000000000000..3cfbf97aef40 --- /dev/null +++ b/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts @@ -0,0 +1,113 @@ +import Debug from 'debug' +import { + TELEMETRY_GROUPS, + TelemetryGroupName, + telemetryManager, +} from './TelemetryManager' +import { CloudDataSource } from '@packages/data-context/src/sources/CloudDataSource' +import { CloudRequest } from '../../api/cloud_request' +import { getCloudMetadata } from '../../get_cloud_metadata' + +const debug = Debug('cypress:server:cloud:studio:telemetry:reporter') + +interface TelemetryReporterOptions { + projectSlug?: string + cloudDataSource: CloudDataSource +} + +export class TelemetryReporter { + private static instance: TelemetryReporter + private projectSlug?: string + private cloudDataSource: CloudDataSource + + private constructor ({ + projectSlug, + cloudDataSource, + }: TelemetryReporterOptions) { + this.projectSlug = projectSlug + this.cloudDataSource = cloudDataSource + } + + public static initialize (options: TelemetryReporterOptions): void { + if (TelemetryReporter.instance) { + // initialize gets called multiple times (e.g. if you switch between projects) + // we need to reset the telemetry manager to avoid accumulating measures + telemetryManager.reset() + } + + TelemetryReporter.instance = new TelemetryReporter(options) + } + + public static getInstance (): TelemetryReporter { + if (!TelemetryReporter.instance) { + throw new Error('TelemetryReporter not initialized') + } + + return TelemetryReporter.instance + } + + public reportTelemetry ( + telemetryGroupName: TelemetryGroupName, + metadata?: Record, + ): void { + this._reportTelemetry(telemetryGroupName, metadata).catch((e: unknown) => { + debug( + 'Error reporting telemetry to cloud: %o, original telemetry: %s', + e, + telemetryGroupName, + ) + }) + } + + private async _reportTelemetry ( + telemetryGroupName: TelemetryGroupName, + metadata?: Record, + ): Promise { + debug('Reporting telemetry for group: %s', telemetryGroupName) + + try { + const groupMeasures = [...TELEMETRY_GROUPS[telemetryGroupName]] + const measures = telemetryManager.getMeasures(groupMeasures) + + const payload = { + projectSlug: this.projectSlug, + telemetryGroupName, + measures, + metadata, + } + + const { cloudUrl, cloudHeaders } = await getCloudMetadata(this.cloudDataSource) + + await CloudRequest.post( + `${cloudUrl}/studio/telemetry`, + payload, + { + headers: { + 'Content-Type': 'application/json', + ...cloudHeaders, + }, + }, + ) + } catch (e: unknown) { + debug( + 'Error reporting telemetry to cloud: %o, original telemetry: %s', + e, + telemetryGroupName, + ) + } + } +} + +export const initializeTelemetryReporter = ( + options: TelemetryReporterOptions, +) => { + TelemetryReporter.initialize(options) +} + +export const reportTelemetry = ( + telemetryGroupName: TelemetryGroupName, + metadata?: Record, +) => { + TelemetryReporter.getInstance().reportTelemetry(telemetryGroupName, metadata) + telemetryManager.clearMeasureGroup(telemetryGroupName) +} diff --git a/packages/server/lib/cloud/studio/telemetry/constants/bundle-lifecycle.ts b/packages/server/lib/cloud/studio/telemetry/constants/bundle-lifecycle.ts new file mode 100644 index 000000000000..294410c539c3 --- /dev/null +++ b/packages/server/lib/cloud/studio/telemetry/constants/bundle-lifecycle.ts @@ -0,0 +1,71 @@ +export const BUNDLE_LIFECYCLE_MARK_NAMES = { + BUNDLE_LIFECYCLE_START: 'bundle_lifecycle_start', + POST_STUDIO_SESSION_START: 'post_studio_session_start', + POST_STUDIO_SESSION_END: 'post_studio_session_end', + ENSURE_STUDIO_BUNDLE_START: 'ensure_studio_bundle_start', + ENSURE_STUDIO_BUNDLE_END: 'ensure_studio_bundle_end', + STUDIO_MANAGER_SETUP_START: 'studio_manager_setup_start', + STUDIO_MANAGER_SETUP_END: 'studio_manager_setup_end', + STUDIO_PROTOCOL_GET_START: 'studio_protocol_get_start', + STUDIO_PROTOCOL_GET_END: 'studio_protocol_get_end', + STUDIO_PROTOCOL_PREPARE_START: 'studio_protocol_prepare_start', + STUDIO_PROTOCOL_PREPARE_END: 'studio_protocol_prepare_end', + BUNDLE_LIFECYCLE_END: 'bundle_lifecycle_end', +} as const + +type BundleLifecycleMarkName = (typeof BUNDLE_LIFECYCLE_MARK_NAMES)[keyof typeof BUNDLE_LIFECYCLE_MARK_NAMES] + +export const BUNDLE_LIFECYCLE_MEASURE_NAMES = { + BUNDLE_LIFECYCLE_DURATION: 'bundle_lifecycle_duration', + POST_STUDIO_SESSION_DURATION: 'post_studio_session_duration', + ENSURE_STUDIO_BUNDLE_DURATION: 'ensure_studio_bundle_duration', + STUDIO_MANAGER_SETUP_DURATION: 'studio_manager_setup_duration', + STUDIO_PROTOCOL_GET_DURATION: 'studio_protocol_get_duration', + STUDIO_PROTOCOL_PREPARE_DURATION: 'studio_protocol_prepare_duration', +} as const + +type BundleLifecycleMeasureName = (typeof BUNDLE_LIFECYCLE_MEASURE_NAMES)[keyof typeof BUNDLE_LIFECYCLE_MEASURE_NAMES] + +export const BUNDLE_LIFECYCLE_MEASURES: Record = { + [BUNDLE_LIFECYCLE_MEASURE_NAMES.BUNDLE_LIFECYCLE_DURATION]: [ + BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_START, + BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END, + ], + [BUNDLE_LIFECYCLE_MEASURE_NAMES.POST_STUDIO_SESSION_DURATION]: [ + BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_START, + BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_END, + ], + [BUNDLE_LIFECYCLE_MEASURE_NAMES.ENSURE_STUDIO_BUNDLE_DURATION]: [ + BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_START, + BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_END, + ], + [BUNDLE_LIFECYCLE_MEASURE_NAMES.STUDIO_MANAGER_SETUP_DURATION]: [ + BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_START, + BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_END, + ], + [BUNDLE_LIFECYCLE_MEASURE_NAMES.STUDIO_PROTOCOL_GET_DURATION]: [ + BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_START, + BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_END, + ], + [BUNDLE_LIFECYCLE_MEASURE_NAMES.STUDIO_PROTOCOL_PREPARE_DURATION]: [ + BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_START, + BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_END, + ], +} + +export const BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES = { + COMPLETE_BUNDLE_LIFECYCLE: 'complete-bundle-lifecycle', +} as const + +type BundleLifecycleTelemetryGroupName = (typeof BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES)[keyof typeof BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES] + +export const BUNDLE_LIFECYCLE_TELEMETRY_GROUPS: Record = { + [BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE]: [ + BUNDLE_LIFECYCLE_MEASURE_NAMES.BUNDLE_LIFECYCLE_DURATION, + BUNDLE_LIFECYCLE_MEASURE_NAMES.POST_STUDIO_SESSION_DURATION, + BUNDLE_LIFECYCLE_MEASURE_NAMES.ENSURE_STUDIO_BUNDLE_DURATION, + BUNDLE_LIFECYCLE_MEASURE_NAMES.STUDIO_MANAGER_SETUP_DURATION, + BUNDLE_LIFECYCLE_MEASURE_NAMES.STUDIO_PROTOCOL_GET_DURATION, + BUNDLE_LIFECYCLE_MEASURE_NAMES.STUDIO_PROTOCOL_PREPARE_DURATION, + ], +} diff --git a/packages/server/lib/cloud/studio/telemetry/constants/initialization.ts b/packages/server/lib/cloud/studio/telemetry/constants/initialization.ts new file mode 100644 index 000000000000..a268745c74ab --- /dev/null +++ b/packages/server/lib/cloud/studio/telemetry/constants/initialization.ts @@ -0,0 +1,55 @@ +export const INITIALIZATION_MARK_NAMES = Object.freeze({ + INITIALIZATION_START: 'initialization-start', + INITIALIZATION_END: 'initialization-end', + CAN_ACCESS_STUDIO_AI_START: 'can-access-studio-ai-start', + CAN_ACCESS_STUDIO_AI_END: 'can-access-studio-ai-end', + CONNECT_PROTOCOL_TO_BROWSER_START: 'connect-protocol-to-browser-start', + CONNECT_PROTOCOL_TO_BROWSER_END: 'connect-protocol-to-browser-end', + INITIALIZE_STUDIO_AI_START: 'initialize-studio-ai-start', + INITIALIZE_STUDIO_AI_END: 'initialize-studio-ai-end', +} as const) + +type InitializationMarkName = (typeof INITIALIZATION_MARK_NAMES)[keyof typeof INITIALIZATION_MARK_NAMES] + +export const INITIALIZATION_MEASURE_NAMES = Object.freeze({ + INITIALIZATION_DURATION: 'initialization-duration', + CAN_ACCESS_STUDIO_AI_DURATION: 'can-access-studio-ai-duration', + CONNECT_PROTOCOL_TO_BROWSER_DURATION: 'connect-protocol-to-browser-duration', + INITIALIZE_STUDIO_AI_DURATION: 'initialize-studio-ai-duration', +} as const) + +type InitializationMeasureName = (typeof INITIALIZATION_MEASURE_NAMES)[keyof typeof INITIALIZATION_MEASURE_NAMES] + +export const INITIALIZATION_MEASURES: Record = Object.freeze({ + [INITIALIZATION_MEASURE_NAMES.INITIALIZATION_DURATION]: [ + INITIALIZATION_MARK_NAMES.INITIALIZATION_START, + INITIALIZATION_MARK_NAMES.INITIALIZATION_END, + ], + [INITIALIZATION_MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION]: [ + INITIALIZATION_MARK_NAMES.CAN_ACCESS_STUDIO_AI_START, + INITIALIZATION_MARK_NAMES.CAN_ACCESS_STUDIO_AI_END, + ], + [INITIALIZATION_MEASURE_NAMES.CONNECT_PROTOCOL_TO_BROWSER_DURATION]: [ + INITIALIZATION_MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_START, + INITIALIZATION_MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_END, + ], + [INITIALIZATION_MEASURE_NAMES.INITIALIZE_STUDIO_AI_DURATION]: [ + INITIALIZATION_MARK_NAMES.INITIALIZE_STUDIO_AI_START, + INITIALIZATION_MARK_NAMES.INITIALIZE_STUDIO_AI_END, + ], +} as const) + +export const INITIALIZATION_TELEMETRY_GROUP_NAMES = Object.freeze({ + INITIALIZE_STUDIO: 'initialize-studio', +} as const) + +type InitializationTelemetryGroupName = (typeof INITIALIZATION_TELEMETRY_GROUP_NAMES)[keyof typeof INITIALIZATION_TELEMETRY_GROUP_NAMES] + +export const INITIALIZATION_TELEMETRY_GROUPS: Record = Object.freeze({ + [INITIALIZATION_TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO]: [ + INITIALIZATION_MEASURE_NAMES.INITIALIZATION_DURATION, + INITIALIZATION_MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION, + INITIALIZATION_MEASURE_NAMES.CONNECT_PROTOCOL_TO_BROWSER_DURATION, + INITIALIZATION_MEASURE_NAMES.INITIALIZE_STUDIO_AI_DURATION, + ], +} as const) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 67c6ec14175d..80b91df158ff 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -30,6 +30,9 @@ import { CloudRequest } from './cloud/api/cloud_request' import { isRetryableError } from './cloud/network/is_retryable_error' import { asyncRetry } from './util/async_retry' import { getCloudMetadata } from './cloud/get_cloud_metadata' +import { telemetryManager } from './cloud/studio/telemetry/TelemetryManager' +import { INITIALIZATION_MARK_NAMES, INITIALIZATION_TELEMETRY_GROUP_NAMES } from './cloud/studio/telemetry/constants/initialization' +import { TelemetryReporter } from './cloud/studio/telemetry/TelemetryReporter' export interface Cfg extends ReceivedCypressOptions { projectId?: string @@ -404,84 +407,119 @@ export class ProjectBase extends EE { closeExtraTargets: this.closeExtraTargets, onStudioInit: async () => { - const isStudioReady = this.ctx.coreData.studioLifecycleManager?.isStudioReady() + telemetryManager.mark(INITIALIZATION_MARK_NAMES.INITIALIZATION_START) - if (!isStudioReady) { - debug('User entered studio mode before cloud studio was initialized') - const { cloudUrl, cloudHeaders } = await getCloudMetadata(this.ctx.cloud) - - reportStudioError({ - cloudApi: { - cloudUrl, - cloudHeaders, - CloudRequest, - isRetryableError, - asyncRetry, - }, - studioHash: this.id, - projectSlug: this.cfg.projectId, - error: new Error('User entered studio before cloud studio was initialized'), - studioMethod: 'onStudioInit', - studioMethodArgs: [], - }) + const endTelemetry = ({ status, canAccessStudioAI }: { status: string, canAccessStudioAI: boolean }) => { + telemetryManager.mark(INITIALIZATION_MARK_NAMES.INITIALIZATION_END) - return { canAccessStudioAI: false } + TelemetryReporter.getInstance().reportTelemetry(INITIALIZATION_TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + status, + canAccessStudioAI, + }) } - const studio = await this.ctx.coreData.studioLifecycleManager?.getStudio() - - // only capture studio started event if the user is accessing legacy studio - if (!this.ctx.coreData.studioLifecycleManager?.cloudStudioRequested) { - try { - studio?.captureStudioEvent({ - type: StudioMetricsTypes.STUDIO_STARTED, - machineId: await this.ctx.coreData.machineId, - projectId: this.cfg.projectId, - browser: this.browser ? { - name: this.browser.name, - family: this.browser.family, - channel: this.browser.channel, - version: this.browser.version, - } : undefined, - cypressVersion: pkg.version, + try { + const isStudioReady = this.ctx.coreData.studioLifecycleManager?.isStudioReady() + + if (!isStudioReady) { + debug('User entered studio mode before cloud studio was initialized') + const { cloudUrl, cloudHeaders } = await getCloudMetadata(this.ctx.cloud) + + reportStudioError({ + cloudApi: { + cloudUrl, + cloudHeaders, + CloudRequest, + isRetryableError, + asyncRetry, + }, + studioHash: this.id, + projectSlug: this.cfg.projectId, + error: new Error('User entered studio before cloud studio was initialized'), + studioMethod: 'onStudioInit', + studioMethodArgs: [], }) - } catch (error) { - debug('Error capturing studio event:', error) - } - } - if (this.spec && studio?.protocolManager) { - const canAccessStudioAI = await studio?.canAccessStudioAI(this.browser) ?? false + endTelemetry({ status: 'studio-not-ready', canAccessStudioAI: false }) - if (!canAccessStudioAI) { - return { canAccessStudioAI } + return { canAccessStudioAI: false } } - this.protocolManager = studio.protocolManager - this.protocolManager.setupProtocol() - this.protocolManager.beforeSpec({ - ...this.spec, - instanceId: v4(), - }) + const studio = await this.ctx.coreData.studioLifecycleManager?.getStudio() + + // only capture studio started event if the user is accessing legacy studio + if (!this.ctx.coreData.studioLifecycleManager?.cloudStudioRequested) { + try { + studio?.captureStudioEvent({ + type: StudioMetricsTypes.STUDIO_STARTED, + machineId: await this.ctx.coreData.machineId, + projectId: this.cfg.projectId, + browser: this.browser ? { + name: this.browser.name, + family: this.browser.family, + channel: this.browser.channel, + version: this.browser.version, + } : undefined, + cypressVersion: pkg.version, + }) + } catch (error) { + debug('Error capturing studio event:', error) + } + } - await browsers.connectProtocolToBrowser({ browser: this.browser, foundBrowsers: this.options.browsers, protocolManager: studio.protocolManager }) + if (this.spec && studio?.protocolManager) { + telemetryManager.mark(INITIALIZATION_MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + const canAccessStudioAI = await studio?.canAccessStudioAI(this.browser) ?? false - if (!studio.protocolManager.dbPath) { - debug('Protocol database path is not set after initializing protocol manager') + telemetryManager.mark(INITIALIZATION_MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) - return { canAccessStudioAI: false } + if (!canAccessStudioAI) { + endTelemetry({ status: 'success', canAccessStudioAI }) + + return { canAccessStudioAI } + } + + this.protocolManager = studio.protocolManager + this.protocolManager.setupProtocol() + this.protocolManager.beforeSpec({ + ...this.spec, + instanceId: v4(), + }) + + telemetryManager.mark(INITIALIZATION_MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_START) + await browsers.connectProtocolToBrowser({ browser: this.browser, foundBrowsers: this.options.browsers, protocolManager: studio.protocolManager }) + telemetryManager.mark(INITIALIZATION_MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_END) + + if (!studio.protocolManager.dbPath) { + debug('Protocol database path is not set after initializing protocol manager') + + endTelemetry({ status: 'protocol-db-path-not-set', canAccessStudioAI: false }) + + return { canAccessStudioAI: false } + } + + telemetryManager.mark(INITIALIZATION_MARK_NAMES.INITIALIZE_STUDIO_AI_START) + await studio.initializeStudioAI({ + protocolDbPath: studio.protocolManager.dbPath, + }) + + telemetryManager.mark(INITIALIZATION_MARK_NAMES.INITIALIZE_STUDIO_AI_END) + + endTelemetry({ status: 'success', canAccessStudioAI: true }) + + return { canAccessStudioAI: true } } - await studio.initializeStudioAI({ - protocolDbPath: studio.protocolManager.dbPath, - }) + this.protocolManager = undefined - return { canAccessStudioAI: true } - } + endTelemetry({ status: 'success', canAccessStudioAI: false }) - this.protocolManager = undefined + return { canAccessStudioAI: false } + } catch (error) { + endTelemetry({ status: 'exception', canAccessStudioAI: false }) - return { canAccessStudioAI: false } + return { canAccessStudioAI: false } + } }, onStudioDestroy: async () => { diff --git a/packages/server/test/unit/cloud/studio/StudioLifecycleManager_spec.ts b/packages/server/test/unit/cloud/studio/StudioLifecycleManager_spec.ts index b0609e3428f6..dbd6c31e4251 100644 --- a/packages/server/test/unit/cloud/studio/StudioLifecycleManager_spec.ts +++ b/packages/server/test/unit/cloud/studio/StudioLifecycleManager_spec.ts @@ -13,6 +13,8 @@ import { Cfg } from '../../../../lib/project-base' import ProtocolManager from '../../../../lib/cloud/protocol' import * as reportStudioErrorPath from '../../../../lib/cloud/api/studio/report_studio_error' +import { INITIALIZATION_TELEMETRY_GROUP_NAMES } from '../../../../lib/cloud/studio/telemetry/constants/initialization' +import { BUNDLE_LIFECYCLE_MARK_NAMES, BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES } from '../../../../lib/cloud/studio/telemetry/constants/bundle-lifecycle' const api = require('../../../../lib/cloud/api').default // Helper to wait for next tick in event loop @@ -37,6 +39,10 @@ describe('StudioLifecycleManager', () => { let watcherOnStub: sinon.SinonStub let watcherCloseStub: sinon.SinonStub let studioManagerDestroyStub: sinon.SinonStub + let addGroupMetadataStub: sinon.SinonStub + let markStub: sinon.SinonStub + let initializeTelemetryReporterStub: sinon.SinonStub + let reportTelemetryStub: sinon.SinonStub beforeEach(() => { postStudioSessionStub = sinon.stub() @@ -50,6 +56,9 @@ describe('StudioLifecycleManager', () => { watcherOnStub = sinon.stub() watcherCloseStub = sinon.stub() studioManagerDestroyStub = sinon.stub() + addGroupMetadataStub = sinon.stub() + markStub = sinon.stub() + initializeTelemetryReporterStub = sinon.stub() mockStudioManager = { status: 'INITIALIZED', setup: studioManagerSetupStub.resolves(), @@ -57,6 +66,8 @@ describe('StudioLifecycleManager', () => { } as unknown as StudioManager readFileStub = sinon.stub() + reportTelemetryStub = sinon.stub() + StudioLifecycleManager = proxyquire('../lib/cloud/studio/StudioLifecycleManager', { './ensure_studio_bundle': { ensureStudioBundle: ensureStudioBundleStub, @@ -92,6 +103,16 @@ describe('StudioLifecycleManager', () => { '../routes': { apiUrl: 'http://localhost:1234/', }, + './telemetry/TelemetryManager': { + telemetryManager: { + mark: markStub, + addGroupMetadata: addGroupMetadataStub, + }, + }, + './telemetry/TelemetryReporter': { + initializeTelemetryReporter: initializeTelemetryReporterStub, + reportTelemetry: reportTelemetryStub, + }, }).StudioLifecycleManager studioLifecycleManager = new StudioLifecycleManager() @@ -226,6 +247,28 @@ describe('StudioLifecycleManager', () => { expect(getCaptureProtocolScriptStub).not.to.be.called expect(prepareProtocolStub).not.to.be.called + + expect(initializeTelemetryReporterStub).to.be.calledWith({ + projectSlug: 'test-project-id', + cloudDataSource: mockCloudDataSource, + }) + + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_END) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_START) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_END) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_START) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_END) + + expect(reportTelemetryStub).to.be.calledWith(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, { + success: true, + }) }) it('initializes the studio manager and registers it in the data context and sets up protocol when studio is enabled', async () => { @@ -299,6 +342,28 @@ describe('StudioLifecycleManager', () => { debugData: {}, mode: 'studio', }) + + expect(initializeTelemetryReporterStub).to.be.calledWith({ + projectSlug: 'test-project-id', + cloudDataSource: mockCloudDataSource, + }) + + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_END) + + expect(reportTelemetryStub).to.be.calledWith(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, { + success: true, + }) }) it('initializes the studio manager in watch mode when CYPRESS_LOCAL_STUDIO_PATH is set', async () => { @@ -462,12 +527,38 @@ describe('StudioLifecycleManager', () => { expect(result).to.be.null } + + expect(initializeTelemetryReporterStub).to.be.calledWith({ + projectSlug: 'test-project-id', + cloudDataSource: mockCloudDataSource, + }) + + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.BUNDLE_LIFECYCLE_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_START) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.POST_STUDIO_SESSION_END) + expect(markStub).to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_START) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.ENSURE_STUDIO_BUNDLE_END) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_START) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_MANAGER_SETUP_END) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_START) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_GET_END) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_START) + expect(markStub).not.to.be.calledWith(BUNDLE_LIFECYCLE_MARK_NAMES.STUDIO_PROTOCOL_PREPARE_END) + + expect(reportTelemetryStub).to.be.calledWith(BUNDLE_LIFECYCLE_TELEMETRY_GROUP_NAMES.COMPLETE_BUNDLE_LIFECYCLE, { + success: false, + }) }) }) describe('isStudioReady', () => { it('returns false when studio manager has not been initialized', () => { expect(studioLifecycleManager.isStudioReady()).to.be.false + + expect(addGroupMetadataStub).to.be.calledWith(INITIALIZATION_TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + studioRequestedBeforeReady: true, + }) }) it('returns true when studio has been initialized', async () => { diff --git a/packages/server/test/unit/cloud/studio/telemetry/TelemetryManager_spec.ts b/packages/server/test/unit/cloud/studio/telemetry/TelemetryManager_spec.ts new file mode 100644 index 000000000000..3e94de01d868 --- /dev/null +++ b/packages/server/test/unit/cloud/studio/telemetry/TelemetryManager_spec.ts @@ -0,0 +1,314 @@ +import { performance } from 'perf_hooks' +import { + telemetryManager, + MARK_NAMES, + MEASURE_NAMES, + TELEMETRY_GROUP_NAMES, +} from '../../../../../lib/cloud/studio/telemetry/TelemetryManager' +import { expect } from 'chai' + +// Helper function to create a controlled delay +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +describe('TelemetryManager', () => { + beforeEach(() => { + // Reset performance marks and measures before each test + performance.clearMarks() + performance.clearMeasures() + }) + + describe('getInstance', () => { + it('should return the same instance on multiple calls', () => { + const instance1 = telemetryManager + const instance2 = telemetryManager + + expect(instance1).to.equal(instance2) + }) + }) + + describe('mark', () => { + it('should create a performance mark', () => { + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + const marks = performance.getEntriesByType('mark') + + expect(marks).to.have.lengthOf(1) + expect(marks[0].name).to.equal(MARK_NAMES.INITIALIZATION_START) + }) + + it('should create multiple marks', () => { + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + const marks = performance.getEntriesByType('mark') + + expect(marks).to.have.lengthOf(2) + expect(marks.map((m) => m.name)).to.include( + MARK_NAMES.INITIALIZATION_START, + ) + + expect(marks.map((m) => m.name)).to.include( + MARK_NAMES.INITIALIZATION_END, + ) + }) + }) + + describe('getMeasure', () => { + it('should return -1 when marks are not present', () => { + const duration = telemetryManager.getMeasure( + MEASURE_NAMES.INITIALIZATION_DURATION, + ) + + expect(duration).to.equal(-1) + }) + + it('should return accurate duration when marks are present', async () => { + const expectedDelay = 50 // 50ms delay + + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + await delay(expectedDelay) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + const duration = telemetryManager.getMeasure( + MEASURE_NAMES.INITIALIZATION_DURATION, + ) + const measure = performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.INITIALIZATION_DURATION) + + expect(measure?.duration).to.equal(duration) + }) + }) + + describe('getMeasures', () => { + it('should return object with -1 measures when no measures are present', () => { + const measures = telemetryManager.getMeasures([ + MEASURE_NAMES.INITIALIZATION_DURATION, + ]) + + expect(measures).to.deep.equal({ + [MEASURE_NAMES.INITIALIZATION_DURATION]: -1, + }) + }) + + it('should return accurate measures for multiple timers', async () => { + const expectedDelay1 = 50 // 50ms delay + const expectedDelay2 = 100 // 100ms delay + + // Set up marks for first timer + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + await delay(expectedDelay1) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + // Set up marks for second timer + telemetryManager.mark(MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + await delay(expectedDelay2) + telemetryManager.mark(MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) + + const measures = telemetryManager.getMeasures([ + MEASURE_NAMES.INITIALIZATION_DURATION, + MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION, + ]) + + expect(Object.keys(measures)).to.have.lengthOf(2) + + // Check first timer + const measure1 = performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.INITIALIZATION_DURATION) + + expect(measure1?.duration).to.equal( + measures[MEASURE_NAMES.INITIALIZATION_DURATION], + ) + + // Check second timer + const measure2 = performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION) + + expect(measure2?.duration).to.equal( + measures[MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION], + ) + }) + + it('should clear measures when clear option is true', async () => { + const expectedDelay = 50 // 50ms delay + + // Set up marks + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + await delay(expectedDelay) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + // Get measures with clear=true + const measures = telemetryManager.getMeasures( + [MEASURE_NAMES.INITIALIZATION_DURATION], + true, + ) + + expect(measures).to.have.property(MEASURE_NAMES.INITIALIZATION_DURATION).that.is.a('number') + + // Verify measures were cleared + const remainingMeasures = performance.getEntriesByType('measure') + + expect(remainingMeasures).to.have.lengthOf(0) + }) + }) + + describe('addGroupMetadata', () => { + it('should add metadata to a group', () => { + telemetryManager.addGroupMetadata(TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + test: 'test', + }) + + telemetryManager.addGroupMetadata(TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + test: 'test', + }) + + expect(telemetryManager['groupMetadata'][TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO]).to.deep.equal({ + test: 'test', + }) + }) + }) + + describe('clearTimerGroup', () => { + it('should clear specified timer group and its measures', async () => { + const expectedDelay = 50 // 50ms delay + + // Set up marks and measures for first timer + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + await delay(expectedDelay) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + const measure = telemetryManager.getMeasure( + MEASURE_NAMES.INITIALIZATION_DURATION, + ) + + expect(measure).to.equal( + performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.INITIALIZATION_DURATION)?.duration, + ) + + // Clear the timer group + telemetryManager.clearMeasureGroup( + TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + ) + + // Verify measures and marks are cleared + const measures = performance.getEntriesByType('measure') + const marks = performance.getEntriesByType('mark') + + expect(measures).to.have.lengthOf(0) + expect(marks).to.have.lengthOf(0) + }) + }) + + describe('clearMeasures', () => { + it('should clear specified measures and their marks', async () => { + const expectedDelay = 50 // 50ms delay + + // Set up marks and measure + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + await delay(expectedDelay) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + const measure = telemetryManager.getMeasure( + MEASURE_NAMES.INITIALIZATION_DURATION, + ) + + expect(measure).to.equal( + performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.INITIALIZATION_DURATION)?.duration, + ) + + // Clear the measure + telemetryManager.clearMeasures([MEASURE_NAMES.INITIALIZATION_DURATION]) + + // Verify measure and marks are cleared + const measures = performance.getEntriesByType('measure') + const marks = performance.getEntriesByType('mark') + + expect(measures).to.have.lengthOf(0) + expect(marks).to.have.lengthOf(0) + }) + + it('should clear multiple measures', async () => { + const expectedDelay1 = 50 // 50ms delay + const expectedDelay2 = 100 // 100ms delay + + // Set up marks and measures for two timers + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + await delay(expectedDelay1) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + const measure1 = telemetryManager.getMeasure( + MEASURE_NAMES.INITIALIZATION_DURATION, + ) + + expect(measure1).to.equal( + performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.INITIALIZATION_DURATION)?.duration, + ) + + telemetryManager.mark(MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + await delay(expectedDelay2) + telemetryManager.mark(MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) + + const measure2 = telemetryManager.getMeasure( + MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION, + ) + + expect(measure2).to.equal( + performance + .getEntriesByType('measure') + .find((m) => m.name === MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION)?.duration, + ) + + // Clear both measures + telemetryManager.clearMeasures([ + MEASURE_NAMES.INITIALIZATION_DURATION, + MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION, + ]) + + // Verify all measures and marks are cleared + const measures = performance.getEntriesByType('measure') + const marks = performance.getEntriesByType('mark') + + expect(measures).to.have.lengthOf(0) + expect(marks).to.have.lengthOf(0) + }) + }) + + describe('reset', () => { + it('should clear all marks and measures', async () => { + // Set up multiple marks and measures + telemetryManager.mark(MARK_NAMES.INITIALIZATION_START) + telemetryManager.mark(MARK_NAMES.INITIALIZATION_END) + + const measure1 = telemetryManager.getMeasure( + MEASURE_NAMES.INITIALIZATION_DURATION, + ) + + expect(measure1).to.be.greaterThan(0) + + telemetryManager.mark(MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + telemetryManager.mark(MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) + + const measure2 = telemetryManager.getMeasure( + MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION, + ) + + expect(measure2).to.be.greaterThan(0) + + // Reset everything + telemetryManager.reset() + + // Verify all marks and measures are cleared + const measures = performance.getEntriesByType('measure') + const marks = performance.getEntriesByType('mark') + + expect(measures).to.have.lengthOf(0) + expect(marks).to.have.lengthOf(0) + }) + }) +}) diff --git a/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts b/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts new file mode 100644 index 000000000000..087f182d9e4e --- /dev/null +++ b/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts @@ -0,0 +1,202 @@ +import { + MEASURE_NAMES, + TELEMETRY_GROUP_NAMES, +} from '../../../../../lib/cloud/studio/telemetry/TelemetryManager' +import { expect } from 'chai' +import { proxyquire, sinon } from '../../../../spec_helper' + +proxyquire.noPreserveCache() + +describe('TelemetryReporter', () => { + let TelemetryReporter: typeof import('../../../../../lib/cloud/studio/telemetry/TelemetryReporter').TelemetryReporter + let initializeTelemetryReporter: typeof import('../../../../../lib/cloud/studio/telemetry/TelemetryReporter').initializeTelemetryReporter + let reportTelemetry: typeof import('../../../../../lib/cloud/studio/telemetry/TelemetryReporter').reportTelemetry + let consoleErrorStub: sinon.SinonStub + let originalNodeEnv: string | undefined + let mockPost: sinon.SinonStub + let telemetryManager: any + + const mockOptions = { + studioHash: 'test-hash', + projectSlug: 'test-project', + cloudApi: { + CloudRequest: { + post: mockPost, + }, + cloudUrl: 'https://cloud.cypress.io', + cloudHeaders: { + 'x-cypress-version': 'test-version', + }, + }, + } + + beforeEach(() => { + sinon.reset() + originalNodeEnv = process.env.NODE_ENV + consoleErrorStub = sinon.stub(console, 'error').callsFake(() => {}) + + telemetryManager = { + getMeasures: sinon.stub().returns({ + [MEASURE_NAMES.INITIALIZATION_DURATION]: 100, + [MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION]: 200, + }), + clearMeasureGroup: sinon.stub().resolves(), + } + + mockPost = sinon.stub().resolves() + const TelemetryReporterDefinition = proxyquire('../lib/cloud/studio/telemetry/TelemetryReporter', { + '../../get_cloud_metadata': { + getCloudMetadata: sinon.stub().resolves({ + cloudUrl: 'https://cloud.cypress.io', + cloudHeaders: { + 'x-cypress-version': 'test-version', + }, + }), + }, + '../../api/cloud_request': { + CloudRequest: { + post: mockPost, + }, + }, + './TelemetryManager': { + telemetryManager, + }, + }) as typeof import('../../../../../lib/cloud/studio/telemetry/TelemetryReporter') + + TelemetryReporter = TelemetryReporterDefinition.TelemetryReporter + initializeTelemetryReporter = TelemetryReporterDefinition.initializeTelemetryReporter + reportTelemetry = TelemetryReporterDefinition.reportTelemetry + }) + + afterEach(() => { + if (originalNodeEnv) { + process.env.NODE_ENV = originalNodeEnv as + | 'development' + | 'test' + | 'production' + } else { + delete process.env.NODE_ENV + } + + consoleErrorStub.restore() + }) + + describe('getInstance', () => { + it('throws error if not initialized', async () => { + expect(() => { + TelemetryReporter.getInstance() + }).to.throw('TelemetryReporter not initialized') + }) + + it('returns the same instance on multiple calls', async () => { + initializeTelemetryReporter(mockOptions as any) + const instance1 = TelemetryReporter.getInstance() + const instance2 = TelemetryReporter.getInstance() + + expect(instance1).to.equal(instance2) + }) + }) + + describe('reportTelemetry', () => { + beforeEach(() => { + initializeTelemetryReporter(mockOptions as any) + }) + + it('sends telemetry to cloud with correct payload', async () => { + TelemetryReporter.getInstance().reportTelemetry( + TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + ) + + // Await the post promise to resolve + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(mockPost).to.have.been.calledWith( + 'https://cloud.cypress.io/studio/telemetry', + { + projectSlug: 'test-project', + telemetryGroupName: TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + measures: { + [MEASURE_NAMES.INITIALIZATION_DURATION]: 100, + [MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION]: 200, + }, + metadata: undefined, + }, + { + headers: { + 'Content-Type': 'application/json', + 'x-cypress-version': 'test-version', + }, + }, + ) + }) + + it('handles cloud request errors gracefully', async () => { + const cloudError = new Error('Cloud request failed') + + mockPost.rejects(cloudError) + + TelemetryReporter.getInstance().reportTelemetry( + TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + ) + + // Wait for the promise to resolve + await new Promise((resolve) => setTimeout(resolve, 0)) + + // Verify the error was handled gracefully (no uncaught exceptions) + expect(mockPost).to.have.been.called + }) + + it('handles telemetry manager errors gracefully', async () => { + telemetryManager.getMeasures.throws(new Error('Failed to get measures')) + + TelemetryReporter.getInstance().reportTelemetry( + TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + ) + + // Wait for the promise to resolve + await new Promise(setImmediate) + + // Verify the error was handled gracefully (no uncaught exceptions) + expect(mockPost).to.not.have.been.called + }) + }) + + describe('reportTelemetry function', () => { + beforeEach(() => { + initializeTelemetryReporter(mockOptions as any) + }) + + it('uses the singleton instance', async () => { + reportTelemetry(TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + test: 'test', + }) + + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(mockPost).to.have.been.calledWith( + 'https://cloud.cypress.io/studio/telemetry', + { + projectSlug: 'test-project', + telemetryGroupName: TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + measures: { + [MEASURE_NAMES.INITIALIZATION_DURATION]: 100, + [MEASURE_NAMES.CAN_ACCESS_STUDIO_AI_DURATION]: 200, + }, + metadata: { + test: 'test', + }, + }, + { + headers: { + 'Content-Type': 'application/json', + 'x-cypress-version': 'test-version', + }, + }, + ) + + expect(telemetryManager.clearMeasureGroup).to.have.been.calledWith( + TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, + ) + }) + }) +}) diff --git a/packages/server/test/unit/project_spec.js b/packages/server/test/unit/project_spec.js index a855032e4786..d134421de793 100644 --- a/packages/server/test/unit/project_spec.js +++ b/packages/server/test/unit/project_spec.js @@ -17,6 +17,8 @@ const { getCtx } = require(`../../lib/makeDataContext`) const browsers = require('../../lib/browsers') const { StudioLifecycleManager } = require('../../lib/cloud/studio/StudioLifecycleManager') const { StudioManager } = require('../../lib/cloud/studio/studio') +const { telemetryManager, MARK_NAMES, TELEMETRY_GROUP_NAMES } = require('../../lib/cloud/studio/telemetry/TelemetryManager') +const { TelemetryReporter } = require('../../lib/cloud/studio/telemetry/TelemetryReporter') let ctx @@ -654,347 +656,401 @@ This option will not have an effect in Some-other-name. Tests that rely on web s expect(fn).to.be.calledOnce }) - it('passes onStudioInit callback with AI enabled and a protocol manager', async function () { - const mockSetupProtocol = sinon.stub() - const mockBeforeSpec = sinon.stub() - const mockAccessStudioAI = sinon.stub().resolves(true) - const mockCaptureStudioEvent = sinon.stub().resolves() + describe('studio', () => { + let markStub + let reportTelemetryStub - this.project.spec = {} + beforeEach(function () { + markStub = sinon.stub() + reportTelemetryStub = sinon.stub() - this.project._cfg = this.project._cfg || {} - this.project._cfg.projectId = 'test-project-id' - this.project.ctx.coreData.user = { email: 'test@example.com' } - this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') + telemetryManager.mark = markStub + TelemetryReporter.getInstance = sinon.stub().returns({ + reportTelemetry: reportTelemetryStub, + }) + }) - const studioManager = new StudioManager() + it('passes onStudioInit callback with AI enabled and a protocol manager', async function () { + const mockSetupProtocol = sinon.stub() + const mockBeforeSpec = sinon.stub() + const mockAccessStudioAI = sinon.stub().resolves(true) + const mockCaptureStudioEvent = sinon.stub().resolves() - studioManager.canAccessStudioAI = mockAccessStudioAI - studioManager.captureStudioEvent = mockCaptureStudioEvent - studioManager.protocolManager = { - setupProtocol: mockSetupProtocol, - beforeSpec: mockBeforeSpec, - db: { test: 'db' }, - dbPath: 'test-db-path', - } + this.project.spec = {} - const studioLifecycleManager = new StudioLifecycleManager() + this.project._cfg = this.project._cfg || {} + this.project._cfg.projectId = 'test-project-id' + this.project.ctx.coreData.user = { email: 'test@example.com' } + this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') - this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager + const studioManager = new StudioManager() - // Set up the studio manager promise directly - studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) - studioLifecycleManager.isStudioReady = sinon.stub().returns(true) + studioManager.canAccessStudioAI = mockAccessStudioAI + studioManager.captureStudioEvent = mockCaptureStudioEvent + studioManager.protocolManager = { + setupProtocol: mockSetupProtocol, + beforeSpec: mockBeforeSpec, + db: { test: 'db' }, + dbPath: 'test-db-path', + } - // Create a browser object - this.project.browser = { - name: 'chrome', - family: 'chromium', - } + const studioLifecycleManager = new StudioLifecycleManager() - this.project.options = { browsers: [this.project.browser] } + this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager - sinon.stub(browsers, 'closeProtocolConnection').resolves() + // Set up the studio manager promise directly + studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) + studioLifecycleManager.isStudioReady = sinon.stub().returns(true) - sinon.stub(browsers, 'connectProtocolToBrowser').resolves() - sinon.stub(this.project, 'protocolManager').get(() => { - return this.project['_protocolManager'] - }).set((protocolManager) => { - this.project['_protocolManager'] = protocolManager - }) + // Create a browser object + this.project.browser = { + name: 'chrome', + family: 'chromium', + } - let studioInitPromise + this.project.options = { browsers: [this.project.browser] } - this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { - studioInitPromise = callbacks.onStudioInit() - }) + sinon.stub(browsers, 'closeProtocolConnection').resolves() - this.project.startWebsockets({}, {}) + sinon.stub(browsers, 'connectProtocolToBrowser').resolves() + sinon.stub(this.project, 'protocolManager').get(() => { + return this.project['_protocolManager'] + }).set((protocolManager) => { + this.project['_protocolManager'] = protocolManager + }) - const { canAccessStudioAI } = await studioInitPromise + let studioInitPromise - expect(canAccessStudioAI).to.be.true - expect(mockCaptureStudioEvent).to.be.calledWith({ - type: 'studio:started', - machineId: 'test-machine-id', - projectId: 'test-project-id', - browser: { - name: 'chrome', + this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { + studioInitPromise = callbacks.onStudioInit() + }) + + this.project.startWebsockets({}, {}) + + const { canAccessStudioAI } = await studioInitPromise + + expect(canAccessStudioAI).to.be.true + expect(mockCaptureStudioEvent).to.be.calledWith({ + type: 'studio:started', + machineId: 'test-machine-id', + projectId: 'test-project-id', + browser: { + name: 'chrome', + family: 'chromium', + channel: undefined, + version: undefined, + }, + cypressVersion: pkg.version, + }) + + expect(mockSetupProtocol).to.be.calledOnce + expect(mockBeforeSpec).to.be.calledOnce + expect(mockAccessStudioAI).to.be.calledWith({ family: 'chromium', - channel: undefined, - version: undefined, - }, - cypressVersion: pkg.version, - }) + name: 'chrome', + }) - expect(mockSetupProtocol).to.be.calledOnce - expect(mockBeforeSpec).to.be.calledOnce - expect(mockAccessStudioAI).to.be.calledWith({ - family: 'chromium', - name: 'chrome', - }) + expect(browsers.connectProtocolToBrowser).to.be.calledWith({ + browser: this.project.browser, + foundBrowsers: this.project.options.browsers, + protocolManager: studioManager.protocolManager, + }) - expect(browsers.connectProtocolToBrowser).to.be.calledWith({ - browser: this.project.browser, - foundBrowsers: this.project.options.browsers, - protocolManager: studioManager.protocolManager, + expect(this.project['_protocolManager']).to.eq(studioManager.protocolManager) + + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZATION_START) + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZATION_END) + expect(markStub).to.be.calledWith(MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + expect(markStub).to.be.calledWith(MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) + expect(markStub).to.be.calledWith(MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_START) + expect(markStub).to.be.calledWith(MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_END) + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZE_STUDIO_AI_START) + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZE_STUDIO_AI_END) + expect(reportTelemetryStub).to.be.calledWith(TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + status: 'success', + canAccessStudioAI: true, + }) }) - expect(this.project['_protocolManager']).to.eq(studioManager.protocolManager) - }) + it('passes onStudioInit callback with AI enabled but no protocol manager', async function () { + const mockSetupProtocol = sinon.stub() + const mockBeforeSpec = sinon.stub() + const mockAccessStudioAI = sinon.stub().resolves(true) + const mockCaptureStudioEvent = sinon.stub().resolves() - it('passes onStudioInit callback with AI enabled but no protocol manager', async function () { - const mockSetupProtocol = sinon.stub() - const mockBeforeSpec = sinon.stub() - const mockAccessStudioAI = sinon.stub().resolves(true) - const mockCaptureStudioEvent = sinon.stub().resolves() + this.project.spec = {} - this.project.spec = {} + this.project._cfg = this.project._cfg || {} + this.project._cfg.projectId = 'test-project-id' + this.project.ctx.coreData.user = { email: 'test@example.com' } + this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') - this.project._cfg = this.project._cfg || {} - this.project._cfg.projectId = 'test-project-id' - this.project.ctx.coreData.user = { email: 'test@example.com' } - this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') + const studioManager = new StudioManager() - const studioManager = new StudioManager() + studioManager.canAccessStudioAI = mockAccessStudioAI + studioManager.captureStudioEvent = mockCaptureStudioEvent + const studioLifecycleManager = new StudioLifecycleManager() - studioManager.canAccessStudioAI = mockAccessStudioAI - studioManager.captureStudioEvent = mockCaptureStudioEvent - const studioLifecycleManager = new StudioLifecycleManager() + this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager - this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager + studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) - studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) + studioLifecycleManager.isStudioReady = sinon.stub().returns(true) - studioLifecycleManager.isStudioReady = sinon.stub().returns(true) + // Create a browser object + this.project.browser = { + name: 'chrome', + family: 'chromium', + } - // Create a browser object - this.project.browser = { - name: 'chrome', - family: 'chromium', - } + this.project.options = { browsers: [this.project.browser] } - this.project.options = { browsers: [this.project.browser] } + sinon.stub(browsers, 'closeProtocolConnection').resolves() - sinon.stub(browsers, 'closeProtocolConnection').resolves() + sinon.stub(browsers, 'connectProtocolToBrowser').resolves() + sinon.stub(this.project, 'protocolManager').get(() => { + return this.project['_protocolManager'] + }).set((protocolManager) => { + this.project['_protocolManager'] = protocolManager + }) - sinon.stub(browsers, 'connectProtocolToBrowser').resolves() - sinon.stub(this.project, 'protocolManager').get(() => { - return this.project['_protocolManager'] - }).set((protocolManager) => { - this.project['_protocolManager'] = protocolManager - }) + let studioInitPromise - let studioInitPromise + this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { + studioInitPromise = callbacks.onStudioInit() + }) - this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { - studioInitPromise = callbacks.onStudioInit() - }) + this.project.startWebsockets({}, {}) - this.project.startWebsockets({}, {}) + const { canAccessStudioAI } = await studioInitPromise - const { canAccessStudioAI } = await studioInitPromise + expect(canAccessStudioAI).to.be.false + expect(mockCaptureStudioEvent).to.be.calledWith({ + type: 'studio:started', + machineId: 'test-machine-id', + projectId: 'test-project-id', + browser: { + name: 'chrome', + family: 'chromium', + channel: undefined, + version: undefined, + }, + cypressVersion: pkg.version, + }) - expect(canAccessStudioAI).to.be.false - expect(mockCaptureStudioEvent).to.be.calledWith({ - type: 'studio:started', - machineId: 'test-machine-id', - projectId: 'test-project-id', - browser: { - name: 'chrome', - family: 'chromium', - channel: undefined, - version: undefined, - }, - cypressVersion: pkg.version, + expect(mockSetupProtocol).not.to.be.called + expect(mockBeforeSpec).not.to.be.called + expect(mockAccessStudioAI).not.to.be.called + + expect(browsers.connectProtocolToBrowser).not.to.be.called + expect(this.project['_protocolManager']).to.be.undefined + + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZATION_START) + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZATION_END) + expect(markStub).not.to.be.calledWith(MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + expect(markStub).not.to.be.calledWith(MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) + expect(markStub).not.to.be.calledWith(MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_START) + expect(markStub).not.to.be.calledWith(MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_END) + expect(markStub).not.to.be.calledWith(MARK_NAMES.INITIALIZE_STUDIO_AI_START) + expect(markStub).not.to.be.calledWith(MARK_NAMES.INITIALIZE_STUDIO_AI_END) + expect(reportTelemetryStub).to.be.calledWith(TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + status: 'success', + canAccessStudioAI: false, + }) }) - expect(mockSetupProtocol).not.to.be.called - expect(mockBeforeSpec).not.to.be.called - expect(mockAccessStudioAI).not.to.be.called + it('passes onStudioInit callback with AI disabled', async function () { + const mockSetupProtocol = sinon.stub() + const mockBeforeSpec = sinon.stub() + const mockAccessStudioAI = sinon.stub().resolves(false) + const mockCaptureStudioEvent = sinon.stub().resolves() - expect(browsers.connectProtocolToBrowser).not.to.be.called - expect(this.project['_protocolManager']).to.be.undefined - }) + this.project.spec = {} - it('passes onStudioInit callback with AI disabled', async function () { - const mockSetupProtocol = sinon.stub() - const mockBeforeSpec = sinon.stub() - const mockAccessStudioAI = sinon.stub().resolves(false) - const mockCaptureStudioEvent = sinon.stub().resolves() + this.project._cfg = this.project._cfg || {} + this.project._cfg.projectId = 'test-project-id' + this.project.ctx.coreData.user = { email: 'test@example.com' } + this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') - this.project.spec = {} + const studioManager = new StudioManager() - this.project._cfg = this.project._cfg || {} - this.project._cfg.projectId = 'test-project-id' - this.project.ctx.coreData.user = { email: 'test@example.com' } - this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') + studioManager.canAccessStudioAI = mockAccessStudioAI + studioManager.captureStudioEvent = mockCaptureStudioEvent + studioManager.protocolManager = { + setupProtocol: mockSetupProtocol, + beforeSpec: mockBeforeSpec, + } - const studioManager = new StudioManager() + const studioLifecycleManager = new StudioLifecycleManager() - studioManager.canAccessStudioAI = mockAccessStudioAI - studioManager.captureStudioEvent = mockCaptureStudioEvent - studioManager.protocolManager = { - setupProtocol: mockSetupProtocol, - beforeSpec: mockBeforeSpec, - } + this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager - const studioLifecycleManager = new StudioLifecycleManager() + studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) - this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager + studioLifecycleManager.isStudioReady = sinon.stub().returns(true) - studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) - - studioLifecycleManager.isStudioReady = sinon.stub().returns(true) + this.project.browser = { + name: 'chrome', + family: 'chromium', + } - this.project.browser = { - name: 'chrome', - family: 'chromium', - } + this.project.options = { browsers: [this.project.browser] } - this.project.options = { browsers: [this.project.browser] } + sinon.stub(browsers, 'closeProtocolConnection').resolves() - sinon.stub(browsers, 'closeProtocolConnection').resolves() + sinon.stub(browsers, 'connectProtocolToBrowser').resolves() + sinon.stub(this.project, 'protocolManager').get(() => { + return this.project['_protocolManager'] + }).set((protocolManager) => { + this.project['_protocolManager'] = protocolManager + }) - sinon.stub(browsers, 'connectProtocolToBrowser').resolves() - sinon.stub(this.project, 'protocolManager').get(() => { - return this.project['_protocolManager'] - }).set((protocolManager) => { - this.project['_protocolManager'] = protocolManager - }) + let studioInitPromise - let studioInitPromise + this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { + studioInitPromise = callbacks.onStudioInit() + }) - this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { - studioInitPromise = callbacks.onStudioInit() - }) + this.project.startWebsockets({}, {}) - this.project.startWebsockets({}, {}) + const { canAccessStudioAI } = await studioInitPromise - const { canAccessStudioAI } = await studioInitPromise + expect(canAccessStudioAI).to.be.false + expect(mockCaptureStudioEvent).to.be.calledWith({ + type: 'studio:started', + machineId: 'test-machine-id', + projectId: 'test-project-id', + browser: { + name: 'chrome', + family: 'chromium', + channel: undefined, + version: undefined, + }, + cypressVersion: pkg.version, + }) - expect(canAccessStudioAI).to.be.false - expect(mockCaptureStudioEvent).to.be.calledWith({ - type: 'studio:started', - machineId: 'test-machine-id', - projectId: 'test-project-id', - browser: { - name: 'chrome', - family: 'chromium', - channel: undefined, - version: undefined, - }, - cypressVersion: pkg.version, + expect(mockSetupProtocol).not.to.be.called + expect(mockBeforeSpec).not.to.be.called + expect(browsers.connectProtocolToBrowser).not.to.be.called + expect(this.project['_protocolManager']).to.be.undefined + + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZATION_START) + expect(markStub).to.be.calledWith(MARK_NAMES.INITIALIZATION_END) + expect(markStub).to.be.calledWith(MARK_NAMES.CAN_ACCESS_STUDIO_AI_START) + expect(markStub).to.be.calledWith(MARK_NAMES.CAN_ACCESS_STUDIO_AI_END) + expect(markStub).not.to.be.calledWith(MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_START) + expect(markStub).not.to.be.calledWith(MARK_NAMES.CONNECT_PROTOCOL_TO_BROWSER_END) + expect(markStub).not.to.be.calledWith(MARK_NAMES.INITIALIZE_STUDIO_AI_START) + expect(markStub).not.to.be.calledWith(MARK_NAMES.INITIALIZE_STUDIO_AI_END) + expect(reportTelemetryStub).to.be.calledWith(TELEMETRY_GROUP_NAMES.INITIALIZE_STUDIO, { + status: 'success', + canAccessStudioAI: false, + }) }) - expect(mockSetupProtocol).not.to.be.called - expect(mockBeforeSpec).not.to.be.called - expect(browsers.connectProtocolToBrowser).not.to.be.called - expect(this.project['_protocolManager']).to.be.undefined - }) - - it('does not capture studio started event if the user is accessing cloud studio', async function () { - process.env.CYPRESS_ENABLE_CLOUD_STUDIO = 'true' - process.env.CYPRESS_LOCAL_STUDIO_PATH = 'false' + it('does not capture studio started event if the user is accessing cloud studio', async function () { + process.env.CYPRESS_ENABLE_CLOUD_STUDIO = 'true' + process.env.CYPRESS_LOCAL_STUDIO_PATH = 'false' - const mockAccessStudioAI = sinon.stub().resolves(true) - const mockCaptureStudioEvent = sinon.stub().resolves() + const mockAccessStudioAI = sinon.stub().resolves(true) + const mockCaptureStudioEvent = sinon.stub().resolves() - this.project.spec = {} + this.project.spec = {} - this.project._cfg = this.project._cfg || {} - this.project._cfg.projectId = 'test-project-id' - this.project.ctx.coreData.user = { email: 'test@example.com' } - this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') + this.project._cfg = this.project._cfg || {} + this.project._cfg.projectId = 'test-project-id' + this.project.ctx.coreData.user = { email: 'test@example.com' } + this.project.ctx.coreData.machineId = Promise.resolve('test-machine-id') - const studioManager = new StudioManager() + const studioManager = new StudioManager() - studioManager.canAccessStudioAI = mockAccessStudioAI - studioManager.captureStudioEvent = mockCaptureStudioEvent - const studioLifecycleManager = new StudioLifecycleManager() + studioManager.canAccessStudioAI = mockAccessStudioAI + studioManager.captureStudioEvent = mockCaptureStudioEvent + const studioLifecycleManager = new StudioLifecycleManager() - this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager + this.project.ctx.coreData.studioLifecycleManager = studioLifecycleManager - studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) + studioLifecycleManager.studioManagerPromise = Promise.resolve(studioManager) - studioLifecycleManager.isStudioReady = sinon.stub().returns(true) + studioLifecycleManager.isStudioReady = sinon.stub().returns(true) - // Create a browser object - this.project.browser = { - name: 'chrome', - family: 'chromium', - } + // Create a browser object + this.project.browser = { + name: 'chrome', + family: 'chromium', + } - this.project.options = { browsers: [this.project.browser] } + this.project.options = { browsers: [this.project.browser] } - sinon.stub(browsers, 'closeProtocolConnection').resolves() + sinon.stub(browsers, 'closeProtocolConnection').resolves() - sinon.stub(browsers, 'connectProtocolToBrowser').resolves() - sinon.stub(this.project, 'protocolManager').get(() => { - return this.project['_protocolManager'] - }).set((protocolManager) => { - this.project['_protocolManager'] = protocolManager - }) + sinon.stub(browsers, 'connectProtocolToBrowser').resolves() + sinon.stub(this.project, 'protocolManager').get(() => { + return this.project['_protocolManager'] + }).set((protocolManager) => { + this.project['_protocolManager'] = protocolManager + }) - let studioInitPromise + let studioInitPromise - this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { - studioInitPromise = callbacks.onStudioInit() - }) + this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { + studioInitPromise = callbacks.onStudioInit() + }) - this.project.startWebsockets({}, {}) + this.project.startWebsockets({}, {}) - const { canAccessStudioAI } = await studioInitPromise + const { canAccessStudioAI } = await studioInitPromise - expect(canAccessStudioAI).to.be.false - expect(mockCaptureStudioEvent).not.to.be.called - }) + expect(canAccessStudioAI).to.be.false + expect(mockCaptureStudioEvent).not.to.be.called + }) - it('passes onStudioDestroy callback', async function () { - // Set up minimal required properties - this.project.ctx = this.project.ctx || {} - this.project.ctx.coreData = this.project.ctx.coreData || {} + it('passes onStudioDestroy callback', async function () { + // Set up minimal required properties + this.project.ctx = this.project.ctx || {} + this.project.ctx.coreData = this.project.ctx.coreData || {} - // Create a studio manager with minimal properties - const protocolManager = { close: sinon.stub().resolves() } - const studioManager = { - destroy: sinon.stub().resolves(), - protocolManager, - } + // Create a studio manager with minimal properties + const protocolManager = { close: sinon.stub().resolves() } + const studioManager = { + destroy: sinon.stub().resolves(), + protocolManager, + } - this.project.ctx.coreData.studioLifecycleManager = { - getStudio: sinon.stub().resolves(studioManager), - isStudioReady: sinon.stub().resolves(true), - } + this.project.ctx.coreData.studioLifecycleManager = { + getStudio: sinon.stub().resolves(studioManager), + isStudioReady: sinon.stub().resolves(true), + } - this.project['_protocolManager'] = protocolManager + this.project['_protocolManager'] = protocolManager - // Create a browser object - this.project.browser = { - name: 'chrome', - family: 'chromium', - } + // Create a browser object + this.project.browser = { + name: 'chrome', + family: 'chromium', + } - this.project.options = { browsers: [this.project.browser] } + this.project.options = { browsers: [this.project.browser] } - sinon.stub(browsers, 'closeProtocolConnection').resolves() + sinon.stub(browsers, 'closeProtocolConnection').resolves() - // Modify the startWebsockets stub to track the callbacks - const callbackPromise = new Promise((resolve) => { - this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { - await callbacks.onStudioDestroy() - resolve() + // Modify the startWebsockets stub to track the callbacks + const callbackPromise = new Promise((resolve) => { + this.project.server.startWebsockets.callsFake(async (automation, config, callbacks) => { + await callbacks.onStudioDestroy() + resolve() + }) }) - }) - this.project.startWebsockets({}, {}) + this.project.startWebsockets({}, {}) - await callbackPromise + await callbackPromise - expect(studioManager.destroy).to.have.been.calledOnce - expect(browsers.closeProtocolConnection).to.have.been.calledOnce - expect(protocolManager.close).to.have.been.calledOnce - expect(this.project['_protocolManager']).to.be.undefined + expect(studioManager.destroy).to.have.been.calledOnce + expect(browsers.closeProtocolConnection).to.have.been.calledOnce + expect(protocolManager.close).to.have.been.calledOnce + expect(this.project['_protocolManager']).to.be.undefined + }) }) }) From 55495adf71c3c34808f5581728ee03e1c364a7c4 Mon Sep 17 00:00:00 2001 From: Ryan Manuel Date: Thu, 5 Jun 2025 21:43:43 -0500 Subject: [PATCH 2/2] fix tests --- .../server/lib/cloud/studio/telemetry/TelemetryReporter.ts | 2 +- .../unit/cloud/studio/telemetry/TelemetryReporter_spec.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts b/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts index 3cfbf97aef40..b1d06480b98c 100644 --- a/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts +++ b/packages/server/lib/cloud/studio/telemetry/TelemetryReporter.ts @@ -4,7 +4,7 @@ import { TelemetryGroupName, telemetryManager, } from './TelemetryManager' -import { CloudDataSource } from '@packages/data-context/src/sources/CloudDataSource' +import type { CloudDataSource } from '@packages/data-context/src/sources/CloudDataSource' import { CloudRequest } from '../../api/cloud_request' import { getCloudMetadata } from '../../get_cloud_metadata' diff --git a/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts b/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts index 087f182d9e4e..961f7d3ef9fe 100644 --- a/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts +++ b/packages/server/test/unit/cloud/studio/telemetry/TelemetryReporter_spec.ts @@ -108,7 +108,7 @@ describe('TelemetryReporter', () => { ) // Await the post promise to resolve - await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise((resolve) => setTimeout(resolve, 5)) expect(mockPost).to.have.been.calledWith( 'https://cloud.cypress.io/studio/telemetry', @@ -140,7 +140,7 @@ describe('TelemetryReporter', () => { ) // Wait for the promise to resolve - await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise((resolve) => setTimeout(resolve, 5)) // Verify the error was handled gracefully (no uncaught exceptions) expect(mockPost).to.have.been.called @@ -171,7 +171,7 @@ describe('TelemetryReporter', () => { test: 'test', }) - await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise((resolve) => setTimeout(resolve, 5)) expect(mockPost).to.have.been.calledWith( 'https://cloud.cypress.io/studio/telemetry',