✨[RUM-260] Support bfcache restore for web vitals (#3527)

* ✨ Add support for Back-Forward Cache (BFCache) views and related metrics

* rum event format

* Refactor BFCache metrics tracking

* Refactor

* ✨ Enhance BFCache metrics tracking with additional tests and comments

* ✨ Improve BFCache handling and testing: streamline event creation, enhance metrics tracking, and add unit tests for BFCache restoration.

* nit

* ✨ TrackViewManually should not track bfCacheView, update clock, update test clarity

* ✨ Refactor BFCache metrics tracking

* delete stopBfCacheMetricsTracking

Author: rgaignault

packages/core/src/domain/telemetry/telemetryEvent.types.ts

@@ -161,6 +161,10 @@ export type TelemetryConfigurationEvent = CommonTelemetryProperties & {
        * Whether long tasks are tracked
        */
       track_long_task?: boolean
+      /**
+       * Whether views loaded from the bfcache are tracked
+       */
+      track_bfcache_views?: boolean
       /**
        * Whether a secure cross-site session cookie is used (deprecated)
        */
@@ -257,6 +261,10 @@ export type TelemetryConfigurationEvent = CommonTelemetryProperties & {
        * Whether console.error logs, uncaught exceptions and network errors are tracked
        */
       forward_errors_to_logs?: boolean
+      /**
+       * The number of displays available to the device
+       */
+      number_of_displays?: number
       /**
        * The console.* tracked
        */
@@ -407,6 +415,10 @@ export type TelemetryConfigurationEvent = CommonTelemetryProperties & {
        * Whether the anonymous users are tracked
        */
       track_anonymous_user?: boolean
+      /**
+       * Whether a list of allowed origins is used to control SDK execution in browser extension contexts. When enabled, the SDK will check if the current origin matches the allowed origins list before running.
+       */
+      use_allowed_tracking_origins?: boolean
       [k: string]: unknown
     }
     [k: string]: unknown

packages/rum-core/src/domain/configuration/configuration.spec.ts

@@ -535,6 +535,7 @@ describe('serializeRumConfiguration', () => {
       trackViewsManually: true,
       trackResources: true,
       trackLongTasks: true,
+      trackBfcacheViews: true,
       remoteConfigurationId: '123',
       plugins: [{ name: 'foo', getConfigurationTelemetry: () => ({ bar: true }) }],
       trackFeatureFlagsForEvents: ['vital'],
@@ -578,6 +579,7 @@ describe('serializeRumConfiguration', () => {
       enable_privacy_for_action_name: false,
       track_resources: true,
       track_long_task: true,
+      track_bfcache_views: true,
       use_worker_url: true,
       compress_intake_requests: true,
       plugins: [{ name: 'foo', bar: true }],

packages/rum-core/src/domain/configuration/configuration.ts

@@ -122,6 +122,11 @@ export interface RumInitConfiguration extends InitConfiguration {
    * Allows you to control RUM views creation. See [Override default RUM view names](https://docs.datadoghq.com/real_user_monitoring/browser/advanced_configuration/?tab=npm#override-default-rum-view-names) for further information.
    */
   trackViewsManually?: boolean | undefined
+  /**
+   * Enable the creation of dedicated views for pages restored from the Back-Forward cache.
+   * @default false
+   */
+  trackBfcacheViews?: boolean | undefined
   /**
    * Enables collection of resource events.
    * @default true
@@ -175,6 +180,7 @@ export interface RumConfiguration extends Configuration {
   trackViewsManually: boolean
   trackResources: boolean
   trackLongTasks: boolean
+  trackBfcacheViews: boolean
   version?: string
   subdomain?: string
   customerDataTelemetrySampleRate: number
@@ -245,6 +251,7 @@ export function validateAndBuildRumConfiguration(
     trackViewsManually: !!initConfiguration.trackViewsManually,
     trackResources: !!(initConfiguration.trackResources ?? true),
     trackLongTasks: !!(initConfiguration.trackLongTasks ?? true),
+    trackBfcacheViews: !!initConfiguration.trackBfcacheViews,
     subdomain: initConfiguration.subdomain,
     defaultPrivacyLevel: objectHasValue(DefaultPrivacyLevel, initConfiguration.defaultPrivacyLevel)
       ? initConfiguration.defaultPrivacyLevel
@@ -337,6 +344,7 @@ export function serializeRumConfiguration(configuration: RumInitConfiguration) {
     track_user_interactions: configuration.trackUserInteractions,
     track_resources: configuration.trackResources,
     track_long_task: configuration.trackLongTasks,
+    track_bfcache_views: configuration.trackBfcacheViews,
     plugins: configuration.plugins?.map((plugin) => ({
       name: plugin.name,
       ...plugin.getConfigurationTelemetry?.(),

packages/rum-core/src/domain/view/bfCacheSupport.spec.ts

@@ -0,0 +1,19 @@
+import { createNewEvent, registerCleanupTask } from '@datadog/browser-core/test'
+import { mockRumConfiguration } from '../../../test'
+import { onBFCacheRestore } from './bfCacheSupport'
+
+describe('onBFCacheRestore', () => {
+  it('should invoke the callback only for BFCache restoration and stop listening when stopped', () => {
+    const configuration = mockRumConfiguration()
+    const callback = jasmine.createSpy('callback')
+
+    const stop = onBFCacheRestore(configuration, callback)
+    registerCleanupTask(stop)
+
+    window.dispatchEvent(createNewEvent('pageshow', { persisted: false }))
+    expect(callback).not.toHaveBeenCalled()
+
+    window.dispatchEvent(createNewEvent('pageshow', { persisted: true }))
+    expect(callback).toHaveBeenCalledTimes(1)
+  })
+})

packages/rum-core/src/domain/view/bfCacheSupport.ts

@@ -0,0 +1,20 @@
+import type { Configuration } from '@datadog/browser-core'
+import { addEventListener, DOM_EVENT } from '@datadog/browser-core'
+
+export function onBFCacheRestore(
+  configuration: Configuration,
+  callback: (event: PageTransitionEvent) => void
+): () => void {
+  const { stop } = addEventListener(
+    configuration,
+    window,
+    DOM_EVENT.PAGE_SHOW,
+    (event: PageTransitionEvent) => {
+      if (event.persisted) {
+        callback(event)
+      }
+    },
+    { capture: true }
+  )
+  return stop
+}

packages/rum-core/src/domain/view/trackViews.spec.ts

@@ -9,7 +9,7 @@ import {
 } from '@datadog/browser-core'
 
 import type { Clock } from '@datadog/browser-core/test'
-import { mockClock, registerCleanupTask } from '@datadog/browser-core/test'
+import { mockClock, registerCleanupTask, createNewEvent } from '@datadog/browser-core/test'
 import { createPerformanceEntry, mockPerformanceObserver } from '../../../test'
 import { RumEventType, ViewLoadingType } from '../../rawRumEvent.types'
 import type { RumEvent } from '../../rumEvent.types'
@@ -1029,3 +1029,31 @@ describe('service and version', () => {
     expect(getViewUpdate(0).version).toEqual('view version')
   })
 })
+
+describe('BFCache views', () => {
+  const lifeCycle = new LifeCycle()
+  let viewTest: ViewTest
+
+  beforeEach(() => {
+    viewTest = setupViewTest({ lifeCycle, partialConfig: { trackBfcacheViews: true } })
+
+    registerCleanupTask(() => {
+      viewTest.stop()
+    })
+  })
+
+  it('should create a new "bf_cache" view when restoring from the BFCache', () => {
+    const { getViewCreateCount, getViewEndCount, getViewUpdate, getViewUpdateCount } = viewTest
+
+    expect(getViewCreateCount()).toBe(1)
+    expect(getViewEndCount()).toBe(0)
+
+    const event = createNewEvent('pageshow', { persisted: true })
+
+    window.dispatchEvent(event)
+
+    expect(getViewEndCount()).toBe(1)
+    expect(getViewCreateCount()).toBe(2)
+    expect(getViewUpdate(getViewUpdateCount() - 1).loadingType).toBe(ViewLoadingType.BF_CACHE)
+  })
+})

packages/rum-core/src/domain/view/trackViews.ts

@@ -17,6 +17,7 @@ import {
   throttle,
   clocksNow,
   clocksOrigin,
+  relativeToClocks,
   timeStampNow,
   display,
   looksLikeRelativeTime,
@@ -39,6 +40,8 @@ import { trackInitialViewMetrics } from './viewMetrics/trackInitialViewMetrics'
 import type { InitialViewMetrics } from './viewMetrics/trackInitialViewMetrics'
 import type { CommonViewMetrics } from './viewMetrics/trackCommonViewMetrics'
 import { trackCommonViewMetrics } from './viewMetrics/trackCommonViewMetrics'
+import { onBFCacheRestore } from './bfCacheSupport'
+import { trackBfcacheMetrics } from './viewMetrics/trackBfcacheMetrics'
 
 export interface ViewEvent {
   id: string
@@ -110,12 +113,20 @@ export function trackViews(
 ) {
   const activeViews: Set<ReturnType<typeof newView>> = new Set()
   let currentView = startNewView(ViewLoadingType.INITIAL_LOAD, clocksOrigin(), initialViewOptions)
+  let stopOnBFCacheRestore: (() => void) | undefined
 
   startViewLifeCycle()
 
   let locationChangeSubscription: Subscription
   if (areViewsTrackedAutomatically) {
     locationChangeSubscription = renewViewOnLocationChange(locationChangeObservable)
+    if (configuration.trackBfcacheViews) {
+      stopOnBFCacheRestore = onBFCacheRestore(configuration, (pageshowEvent) => {
+        currentView.end()
+        const startClocks = relativeToClocks(pageshowEvent.timeStamp as RelativeTime)
+        currentView = startNewView(ViewLoadingType.BF_CACHE, startClocks, undefined)
+      })
+    }
   }
 
   function startNewView(loadingType: ViewLoadingType, startClocks?: ClocksState, viewOptions?: ViewOptions) {
@@ -184,6 +195,9 @@ export function trackViews(
       if (locationChangeSubscription) {
         locationChangeSubscription.unsubscribe()
       }
+      if (stopOnBFCacheRestore) {
+        stopOnBFCacheRestore()
+      }
       currentView.end()
       activeViews.forEach((view) => view.stop())
     },
@@ -256,6 +270,11 @@ function newView(
       ? trackInitialViewMetrics(configuration, setLoadEvent, scheduleViewUpdate)
       : { stop: noop, initialViewMetrics: {} as InitialViewMetrics }
 
+  // Start BFCache-specific metrics when restoring from BFCache
+  if (loadingType === ViewLoadingType.BF_CACHE) {
+    trackBfcacheMetrics(startClocks, initialViewMetrics, scheduleViewUpdate)
+  }
+
   const { stop: stopEventCountsTracking, eventCounts } = trackViewEventCounts(lifeCycle, id, scheduleViewUpdate)
 
   // Session keep alive

packages/rum-core/src/domain/view/viewMetrics/trackBfcacheMetrics.spec.ts

@@ -0,0 +1,42 @@
+import type { Duration, RelativeTime, TimeStamp } from '@datadog/browser-core'
+import type { Clock } from '@datadog/browser-core/test'
+import { createNewEvent, mockClock, registerCleanupTask } from '@datadog/browser-core/test'
+import { trackBfcacheMetrics } from './trackBfcacheMetrics'
+import type { InitialViewMetrics } from './trackInitialViewMetrics'
+
+describe('trackBfcacheMetrics', () => {
+  let clock: Clock
+
+  beforeEach(() => {
+    clock = mockClock()
+    registerCleanupTask(clock.cleanup)
+
+    spyOn(window, 'requestAnimationFrame').and.callFake((cb: FrameRequestCallback): number => {
+      cb(performance.now())
+      return 0
+    })
+  })
+
+  function createPageshowEvent() {
+    return createNewEvent('pageshow', { timeStamp: performance.now() })
+  }
+
+  it('should compute FCP and LCP from the next frame after BFCache restore', () => {
+    const pageshow = createPageshowEvent() as PageTransitionEvent
+
+    const metrics: InitialViewMetrics = {}
+    const scheduleSpy = jasmine.createSpy('schedule')
+
+    clock.tick(50)
+
+    const startClocks = {
+      relative: pageshow.timeStamp as RelativeTime,
+      timeStamp: 0 as TimeStamp,
+    }
+    trackBfcacheMetrics(startClocks, metrics, scheduleSpy)
+
+    expect(metrics.firstContentfulPaint).toEqual(50 as Duration)
+    expect(metrics.largestContentfulPaint?.value).toEqual(50 as RelativeTime)
+    expect(scheduleSpy).toHaveBeenCalled()
+  })
+})

packages/rum-core/src/domain/view/viewMetrics/trackBfcacheMetrics.ts

@@ -0,0 +1,20 @@
+import type { RelativeTime, ClocksState } from '@datadog/browser-core'
+import type { InitialViewMetrics } from './trackInitialViewMetrics'
+import { trackRestoredFirstContentfulPaint } from './trackFirstContentfulPaint'
+
+/**
+ * BFCache keeps a full in-memory snapshot of the DOM. When the page is restored, nothing needs to be fetched, so the whole
+ * viewport repaints in a single frame. Consequently, LCP almost always equals FCP.
+ * (See: https://github.com/GoogleChrome/web-vitals/pull/87)
+ */
+export function trackBfcacheMetrics(
+  viewStart: ClocksState,
+  metrics: InitialViewMetrics,
+  scheduleViewUpdate: () => void
+) {
+  trackRestoredFirstContentfulPaint(viewStart.relative, (paintTime) => {
+    metrics.firstContentfulPaint = paintTime
+    metrics.largestContentfulPaint = { value: paintTime as RelativeTime }
+    scheduleViewUpdate()
+  })
+}

packages/rum-core/src/domain/view/viewMetrics/trackFirstContentfulPaint.ts

@@ -1,5 +1,5 @@
-import type { RelativeTime } from '@datadog/browser-core'
-import { ONE_MINUTE } from '@datadog/browser-core'
+import type { Duration, RelativeTime } from '@datadog/browser-core'
+import { ONE_MINUTE, elapsed, relativeNow } from '@datadog/browser-core'
 import type { RumPerformancePaintTiming } from '../../../browser/performanceObservable'
 import { createPerformanceObservable, RumPerformanceEntryType } from '../../../browser/performanceObservable'
 import type { RumConfiguration } from '../../configuration'
@@ -32,3 +32,16 @@ export function trackFirstContentfulPaint(
     stop: performanceSubscription.unsubscribe,
   }
 }
+
+/**
+ * Measure the First Contentful Paint after a BFCache restoration.
+ * The DOM is restored synchronously, so we approximate the FCP with the first frame
+ * rendered just after the pageshow event, using two nested requestAnimationFrame calls.
+ */
+export function trackRestoredFirstContentfulPaint(viewStartRelative: RelativeTime, callback: (fcp: Duration) => void) {
+  requestAnimationFrame(() => {
+    requestAnimationFrame(() => {
+      callback(elapsed(viewStartRelative, relativeNow()))
+    })
+  })
+}

packages/rum-core/src/domain/view/viewMetrics/trackLoadingTime.ts

@@ -7,6 +7,12 @@ import { ViewLoadingType } from '../../../rawRumEvent.types'
 import type { RumMutationRecord } from '../../../browser/domMutationObservable'
 import { trackFirstHidden } from './trackFirstHidden'
 
+/**
+ * For non-initial views (such as route changes or BFCache restores), the regular load event does not fire
+ * In these cases, trackLoadingTime can only emit a loadingTime  if waitPageActivityEnd detects some post-restore activity.
+ * If nothing happens after the view starts,no candidate is recorded and loadingTime stays undefined.
+ */
+
 export function trackLoadingTime(
   lifeCycle: LifeCycle,
   domMutationObservable: Observable<RumMutationRecord[]>,

packages/rum-core/src/rawRumEvent.types.ts

@@ -188,6 +188,7 @@ export type PageStateServerEntry = { state: PageState; start: ServerDuration }
 export const enum ViewLoadingType {
   INITIAL_LOAD = 'initial_load',
   ROUTE_CHANGE = 'route_change',
+  BF_CACHE = 'bf_cache',
 }
 
 export interface ViewCustomTimings {