diff --git a/docs/recipes/network.md b/docs/recipes/network.md new file mode 100644 index 0000000000..b62b77a939 --- /dev/null +++ b/docs/recipes/network.md @@ -0,0 +1,97 @@ +# network recorder and replayer + +Starting from v2.0.0, we add the plugin to record network output. +This feature aims to provide developers with more information about the bug scene. There are some options for recording and replaying network output. + +### Enable recording network + +You can enable using default option like this: + +```js +rrweb.record({ + emit: function emit(event) { + events.push(event); + }, + // to use default record option + plugins: [rrweb.getRecordNetworkPlugin()], +}); +``` + +You can also customize the behavior of logger like this: + +```js +rrweb.record({ + emit: function emit(event) { + fetch('https://api.my-server.com/events', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + events: [event], + }), + }); + }, + // customized record options + plugins: [ + rrweb.getRecordNetworkPlugin({ + initiatorTypes: ['fetch', 'xmlhttprequest'], + // block recording event for request to upload events to server + ignoreRequestFn: (request) => { + if (request.url === 'https://api.my-server.com/events') { + return true; + } + return false; + }, + recordHeaders: true, + recordBody: true, + recordInitialRequests: false, + }), + ], +}); +``` + +**alert**: If you are uploading events to a server, you should always use `ignoreRequestFn` to block recording events for these requests or else you will cause a nasty loop. + +All options are described below: +| key | default | description | +| ---------------- | --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| initiatorTypes | `['fetch','xmlhttprequest','img',...]` | Default value contains names of all [initiator types](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming/initiatorType). You can override it by setting the types you need. | +| ignoreRequestFn | `() => false` | Block recording events for specific requests | +| recordHeaders | `false` | Record the request & response headers for `fetch` and `xmlhttprequest` requests | +| recordBody | `false` | Record the request & response bodies for `fetch` and `xmlhttprequest` requests | +| recordInitialRequests | `false` | Record an event for all requests prior to rrweb.record() being called | + +## replay network + +It is up to you to decide how to best replay your network events using the `onNetworkData` callback. + +```js +const replayer = new rrweb.Replayer(events, { + plugins: [ + rrweb.getReplayNetworkPlugin({ + onNetworkData: ({ requests }) => { + for (const request of requests) { + const url = request.url; + const method = request.method; + const status = request.status; + console.log(`${method} ${url} ${status}`); + } + }, + }), + ], +}); +replayer.play(); +``` + +Description of replay option is as follows: + +| key | default | description | +| ------------- | ----------- | ------------------------------------------------------------------------------------------ | +| onNetworkData | `undefined` | You could use this interface to replay the network requests in a simulated browser console | + +## technical implementation + +This implementation records [`fetch`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) and [`XMLHttpRequest`](https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest) by patching their object & methods. We record document navigation using [`PerformanceNavigationTiming`](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceNavigationTiming) and we use [`PerformanceResourceTiming`](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming) for recording everything else (script, img, link etc.) via [`PerformanceObserver`](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceObserver) API. + +For more information please see [[network-plugin] Feat: Capture network events #1105](https://github.com/rrweb-io/rrweb/pull/1105) PR. diff --git a/packages/rrweb/src/entries/all.ts b/packages/rrweb/src/entries/all.ts index d67ff92447..ccfa14f69f 100644 --- a/packages/rrweb/src/entries/all.ts +++ b/packages/rrweb/src/entries/all.ts @@ -2,3 +2,5 @@ export * from '../index'; export * from '../packer'; export * from '../plugins/console/record'; export * from '../plugins/console/replay'; +export * from '../plugins/network/record'; +export * from '../plugins/network/replay'; diff --git a/packages/rrweb/src/plugins/console/record/index.ts b/packages/rrweb/src/plugins/console/record/index.ts index 47cd513695..9648b24d3d 100644 --- a/packages/rrweb/src/plugins/console/record/index.ts +++ b/packages/rrweb/src/plugins/console/record/index.ts @@ -1,22 +1,7 @@ import type { listenerHandler, RecordPlugin, IWindow } from '@rrweb/types'; import { patch } from '../../../utils'; -import { ErrorStackParser, StackFrame } from './error-stack-parser'; -import { stringify } from './stringify'; - -export type StringifyOptions = { - // limit of string length - stringLengthLimit?: number; - /** - * limit of number of keys in an object - * if an object contains more keys than this limit, we would call its toString function directly - */ - numOfKeysLimit: number; - /** - * limit number of depth in an object - * if an object is too deep, toString process may cause browser OOM - */ - depthOfLimit: number; -}; +import { ErrorStackParser, StackFrame } from '../../utils/error-stack-parser'; +import { stringify, StringifyOptions } from '../../utils/stringify'; type LogRecordOptions = { level?: LogLevel[]; diff --git a/packages/rrweb/src/plugins/network/record/index.ts b/packages/rrweb/src/plugins/network/record/index.ts new file mode 100644 index 0000000000..2c59e78844 --- /dev/null +++ b/packages/rrweb/src/plugins/network/record/index.ts @@ -0,0 +1,491 @@ +import type { IWindow, listenerHandler, RecordPlugin } from '@rrweb/types'; +import { patch } from '../../../utils'; +import { findLast } from '../../utils/find-last'; + +export type InitiatorType = + | 'audio' + | 'beacon' + | 'body' + | 'css' + | 'early-hint' + | 'embed' + | 'fetch' + | 'frame' + | 'iframe' + | 'icon' + | 'image' + | 'img' + | 'input' + | 'link' + | 'navigation' + | 'object' + | 'ping' + | 'script' + | 'track' + | 'video' + | 'xmlhttprequest'; + +type NetworkRecordOptions = { + initiatorTypes?: InitiatorType[]; + ignoreRequestFn?: (data: NetworkRequest) => boolean; + recordHeaders?: boolean | { request: boolean; response: boolean }; + recordBody?: + | boolean + | string[] + | { request: boolean | string[]; response: boolean | string[] }; + recordInitialRequests?: boolean; +}; + +const defaultNetworkOptions: NetworkRecordOptions = { + initiatorTypes: [ + 'audio', + 'beacon', + 'body', + 'css', + 'early-hint', + 'embed', + 'fetch', + 'frame', + 'iframe', + 'icon', + 'image', + 'img', + 'input', + 'link', + 'navigation', + 'object', + 'ping', + 'script', + 'track', + 'video', + 'xmlhttprequest', + ], + ignoreRequestFn: () => false, + recordHeaders: false, + recordBody: false, + recordInitialRequests: false, +}; + +type Headers = Record; +type Body = + | string + | Document + | Blob + | ArrayBufferView + | ArrayBuffer + | FormData + | URLSearchParams + | ReadableStream + | null; + +type NetworkRequest = { + url: string; + method?: string; + initiatorType: InitiatorType; + status?: number; + startTime: number; + endTime: number; + requestHeaders?: Headers; + requestBody?: Body; + responseHeaders?: Headers; + responseBody?: Body; +}; + +export type NetworkData = { + requests: NetworkRequest[]; + isInitial?: boolean; +}; + +type networkCallback = (data: NetworkData) => void; + +const isNavigationTiming = ( + entry: PerformanceEntry, +): entry is PerformanceNavigationTiming => entry.entryType === 'navigation'; +const isResourceTiming = ( + entry: PerformanceEntry, +): entry is PerformanceResourceTiming => entry.entryType === 'resource'; + +type ObservedPerformanceEntry = ( + | PerformanceNavigationTiming + | PerformanceResourceTiming +) & { + responseStatus?: number; +}; + +function initPerformanceObserver( + cb: networkCallback, + win: IWindow, + options: Required, +) { + if (options.recordInitialRequests) { + const initialPerformanceEntries = win.performance + .getEntries() + .filter( + (entry): entry is ObservedPerformanceEntry => + isNavigationTiming(entry) || + (isResourceTiming(entry) && + options.initiatorTypes.includes( + entry.initiatorType as InitiatorType, + )), + ); + cb({ + requests: initialPerformanceEntries.map((entry) => ({ + url: entry.name, + initiatorType: entry.initiatorType as InitiatorType, + status: 'responseStatus' in entry ? entry.responseStatus : undefined, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), + })), + isInitial: true, + }); + } + const observer = new win.PerformanceObserver((entries) => { + const performanceEntries = entries + .getEntries() + .filter( + (entry): entry is ObservedPerformanceEntry => + isNavigationTiming(entry) || + (isResourceTiming(entry) && + options.initiatorTypes.includes( + entry.initiatorType as InitiatorType, + ) && + entry.initiatorType !== 'xmlhttprequest' && + entry.initiatorType !== 'fetch'), + ); + cb({ + requests: performanceEntries.map((entry) => ({ + url: entry.name, + initiatorType: entry.initiatorType as InitiatorType, + status: 'responseStatus' in entry ? entry.responseStatus : undefined, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), + })), + }); + }); + observer.observe({ entryTypes: ['navigation', 'resource'] }); + return () => { + observer.disconnect(); + }; +} + +function shouldRecordHeaders( + type: 'request' | 'response', + recordHeaders: NetworkRecordOptions['recordHeaders'], +) { + return ( + !!recordHeaders && + (typeof recordHeaders === 'boolean' || recordHeaders[type]) + ); +} + +function shouldRecordBody( + type: 'request' | 'response', + recordBody: NetworkRecordOptions['recordBody'], + headers: Headers, +) { + function matchesContentType(contentTypes: string[]) { + const contentTypeHeader = Object.keys(headers).find( + (key) => key.toLowerCase() === 'content-type', + ); + const contentType = contentTypeHeader && headers[contentTypeHeader]; + return contentTypes.some((ct) => contentType?.includes(ct)); + } + if (!recordBody) return false; + if (typeof recordBody === 'boolean') return true; + if (Array.isArray(recordBody)) return matchesContentType(recordBody); + const recordBodyType = recordBody[type]; + if (typeof recordBodyType === 'boolean') return recordBodyType; + return matchesContentType(recordBodyType); +} + +async function getRequestPerformanceEntry( + win: IWindow, + initiatorType: string, + url: string, + after?: number, + before?: number, + attempt = 0, +): Promise { + if (attempt > 10) { + throw new Error('Cannot find performance entry'); + } + const urlPerformanceEntries = win.performance.getEntriesByName( + url, + ) as PerformanceResourceTiming[]; + const performanceEntry = findLast( + urlPerformanceEntries, + (entry) => + isResourceTiming(entry) && + entry.initiatorType === initiatorType && + (!after || entry.startTime >= after) && + (!before || entry.startTime <= before), + ); + if (!performanceEntry) { + await new Promise((resolve) => setTimeout(resolve, 50 * attempt)); + return getRequestPerformanceEntry( + win, + initiatorType, + url, + after, + before, + attempt + 1, + ); + } + return performanceEntry; +} + +function initXhrObserver( + cb: networkCallback, + win: IWindow, + options: Required, +): listenerHandler { + if (!options.initiatorTypes.includes('xmlhttprequest')) { + return () => { + // + }; + } + const recordRequestHeaders = shouldRecordHeaders( + 'request', + options.recordHeaders, + ); + const recordResponseHeaders = shouldRecordHeaders( + 'response', + options.recordHeaders, + ); + const restorePatch = patch( + win.XMLHttpRequest.prototype, + 'open', + (originalOpen: typeof XMLHttpRequest.prototype.open) => { + return function ( + method: string, + url: string | URL, + async = true, + username?: string | null, + password?: string | null, + ) { + const xhr = this as XMLHttpRequest; + const req = new Request(url); + const networkRequest: Partial = {}; + let after: number | undefined; + let before: number | undefined; + const requestHeaders: Headers = {}; + const originalSetRequestHeader = xhr.setRequestHeader.bind(xhr); + xhr.setRequestHeader = (header: string, value: string) => { + requestHeaders[header] = value; + return originalSetRequestHeader(header, value); + }; + if (recordRequestHeaders) { + networkRequest.requestHeaders = requestHeaders; + } + const originalSend = xhr.send.bind(xhr); + xhr.send = (body) => { + if (shouldRecordBody('request', options.recordBody, requestHeaders)) { + if (body === undefined || body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = body; + } + } + after = win.performance.now(); + return originalSend(body); + }; + xhr.addEventListener('readystatechange', () => { + if (xhr.readyState !== xhr.DONE) { + return; + } + before = win.performance.now(); + const responseHeaders: Headers = {}; + const rawHeaders = xhr.getAllResponseHeaders(); + const headers = rawHeaders.trim().split(/[\r\n]+/); + headers.forEach((line) => { + const parts = line.split(': '); + const header = parts.shift(); + const value = parts.join(': '); + if (header) { + responseHeaders[header] = value; + } + }); + if (recordResponseHeaders) { + networkRequest.responseHeaders = responseHeaders; + } + if ( + shouldRecordBody('response', options.recordBody, responseHeaders) + ) { + if (xhr.response === undefined || xhr.response === null) { + networkRequest.responseBody = null; + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + networkRequest.responseBody = xhr.response; + } + } + getRequestPerformanceEntry( + win, + 'xmlhttprequest', + req.url, + after, + before, + ) + .then((entry) => { + const request: NetworkRequest = { + url: entry.name, + method: req.method, + initiatorType: entry.initiatorType as InitiatorType, + status: xhr.status, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), + requestHeaders: networkRequest.requestHeaders, + requestBody: networkRequest.requestBody, + responseHeaders: networkRequest.responseHeaders, + responseBody: networkRequest.responseBody, + }; + cb({ requests: [request] }); + }) + .catch(() => { + // + }); + }); + originalOpen.call(xhr, method, url, async, username, password); + }; + }, + ); + return () => { + restorePatch(); + }; +} + +function initFetchObserver( + cb: networkCallback, + win: IWindow, + options: Required, +): listenerHandler { + if (!options.initiatorTypes.includes('fetch')) { + return () => { + // + }; + } + const recordRequestHeaders = shouldRecordHeaders( + 'request', + options.recordHeaders, + ); + const recordResponseHeaders = shouldRecordHeaders( + 'response', + options.recordHeaders, + ); + const restorePatch = patch(win, 'fetch', (originalFetch: typeof fetch) => { + return async function ( + url: URL | RequestInfo, + init?: RequestInit | undefined, + ) { + const req = new Request(url, init); + let res: Response | undefined; + const networkRequest: Partial = {}; + let after: number | undefined; + let before: number | undefined; + try { + const requestHeaders: Headers = {}; + req.headers.forEach((value, header) => { + requestHeaders[header] = value; + }); + if (recordRequestHeaders) { + networkRequest.requestHeaders = requestHeaders; + } + if (shouldRecordBody('request', options.recordBody, requestHeaders)) { + if (req.body === undefined || req.body === null) { + networkRequest.requestBody = null; + } else { + networkRequest.requestBody = req.body; + } + } + after = win.performance.now(); + res = await originalFetch(req); + before = win.performance.now(); + const responseHeaders: Headers = {}; + res.headers.forEach((value, header) => { + responseHeaders[header] = value; + }); + if (recordResponseHeaders) { + networkRequest.responseHeaders = responseHeaders; + } + if (shouldRecordBody('response', options.recordBody, responseHeaders)) { + let body: string | undefined; + try { + body = await res.clone().text(); + } catch { + // + } + if (res.body === undefined || res.body === null) { + networkRequest.responseBody = null; + } else { + networkRequest.responseBody = body; + } + } + return res; + } finally { + getRequestPerformanceEntry(win, 'fetch', req.url, after, before) + .then((entry) => { + const request: NetworkRequest = { + url: entry.name, + method: req.method, + initiatorType: entry.initiatorType as InitiatorType, + status: res?.status, + startTime: Math.round(entry.startTime), + endTime: Math.round(entry.responseEnd), + requestHeaders: networkRequest.requestHeaders, + requestBody: networkRequest.requestBody, + responseHeaders: networkRequest.responseHeaders, + responseBody: networkRequest.responseBody, + }; + cb({ requests: [request] }); + }) + .catch(() => { + // + }); + } + }; + }); + return () => { + restorePatch(); + }; +} + +function initNetworkObserver( + callback: networkCallback, + win: IWindow, // top window or in an iframe + options: NetworkRecordOptions, +): listenerHandler { + if (!('performance' in win)) { + return () => { + // + }; + } + const networkOptions = (options + ? Object.assign({}, defaultNetworkOptions, options) + : defaultNetworkOptions) as Required; + + const cb: networkCallback = (data) => { + const requests = data.requests.filter( + (request) => !networkOptions.ignoreRequestFn(request), + ); + if (requests.length > 0 || data.isInitial) { + callback({ ...data, requests }); + } + }; + const performanceObserver = initPerformanceObserver(cb, win, networkOptions); + const xhrObserver = initXhrObserver(cb, win, networkOptions); + const fetchObserver = initFetchObserver(cb, win, networkOptions); + return () => { + performanceObserver(); + xhrObserver(); + fetchObserver(); + }; +} + +export const NETWORK_PLUGIN_NAME = 'rrweb/network@1'; + +export const getRecordNetworkPlugin: ( + options?: NetworkRecordOptions, +) => RecordPlugin = (options) => ({ + name: NETWORK_PLUGIN_NAME, + observer: initNetworkObserver, + options: options, +}); diff --git a/packages/rrweb/src/plugins/network/replay/index.ts b/packages/rrweb/src/plugins/network/replay/index.ts new file mode 100644 index 0000000000..0d639ec37f --- /dev/null +++ b/packages/rrweb/src/plugins/network/replay/index.ts @@ -0,0 +1,26 @@ +import { NetworkData, NETWORK_PLUGIN_NAME } from '../record'; +import type { eventWithTime } from '@rrweb/types'; +import { EventType } from '@rrweb/types'; +import type { ReplayPlugin } from '../../../types'; + +export type OnNetworkData = (data: NetworkData) => void; + +export type NetworkReplayOptions = { + onNetworkData: OnNetworkData; +}; + +export const getReplayNetworkPlugin: ( + options: NetworkReplayOptions, +) => ReplayPlugin = (options) => { + return { + handler(event: eventWithTime) { + if ( + event.type === EventType.Plugin && + event.data.plugin === NETWORK_PLUGIN_NAME + ) { + const networkData = event.data.payload as NetworkData; + options.onNetworkData(networkData); + } + }, + }; +}; diff --git a/packages/rrweb/src/plugins/console/record/error-stack-parser.ts b/packages/rrweb/src/plugins/utils/error-stack-parser.ts similarity index 100% rename from packages/rrweb/src/plugins/console/record/error-stack-parser.ts rename to packages/rrweb/src/plugins/utils/error-stack-parser.ts diff --git a/packages/rrweb/src/plugins/utils/find-last.ts b/packages/rrweb/src/plugins/utils/find-last.ts new file mode 100644 index 0000000000..a614538614 --- /dev/null +++ b/packages/rrweb/src/plugins/utils/find-last.ts @@ -0,0 +1,11 @@ +export function findLast( + array: Array, + predicate: (value: T) => boolean, +): T | undefined { + const length = array.length; + for (let i = length - 1; i >= 0; i -= 1) { + if (predicate(array[i])) { + return array[i]; + } + } +} diff --git a/packages/rrweb/src/plugins/console/record/stringify.ts b/packages/rrweb/src/plugins/utils/stringify.ts similarity index 91% rename from packages/rrweb/src/plugins/console/record/stringify.ts rename to packages/rrweb/src/plugins/utils/stringify.ts index eef8c38a40..5598de5567 100644 --- a/packages/rrweb/src/plugins/console/record/stringify.ts +++ b/packages/rrweb/src/plugins/utils/stringify.ts @@ -2,8 +2,20 @@ * this file is used to serialize log message to string * */ - -import type { StringifyOptions } from './index'; +export type StringifyOptions = { + // limit of string length + stringLengthLimit?: number; + /** + * limit of number of keys in an object + * if an object contains more keys than this limit, we would call its toString function directly + */ + numOfKeysLimit: number; + /** + * limit number of depth in an object + * if an object is too deep, toString process may cause browser OOM + */ + depthOfLimit: number; +}; /** * transfer the node path in Event to string