From ad1703ca75dcc08751aef225c762d5d62a5358b9 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Fri, 8 Nov 2024 16:28:35 +0100 Subject: [PATCH 01/11] react-native: add Stream --- package-lock.json | 11 +++- packages/react-native/package.json | 4 +- .../react-native/src/storage/FileSystem.ts | 3 +- .../src/storage/ReactNativeFileSystem.ts | 6 ++- .../react-native/src/storage/StreamWriter.ts | 51 ++++++++++++++++++- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index ef1b9c3a..f83ed57a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24211,7 +24211,8 @@ "version": "0.1.1", "license": "MIT", "dependencies": { - "@backtrace/sdk-core": "^0.6.0" + "@backtrace/sdk-core": "^0.6.0", + "web-streams-polyfill": "^4.0.0" }, "devDependencies": { "@react-native-community/eslint-config": "^3.0.2", @@ -24250,6 +24251,14 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "packages/react-native/node_modules/web-streams-polyfill": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz", + "integrity": "sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw==", + "engines": { + "node": ">= 8" + } + }, "packages/react/node_modules/@testing-library/react": { "version": "14.3.1", "dev": true, diff --git a/packages/react-native/package.json b/packages/react-native/package.json index 1b194451..21454c13 100644 --- a/packages/react-native/package.json +++ b/packages/react-native/package.json @@ -28,6 +28,7 @@ ], "scripts": { "build": "bob build", + "watch": "tsc --noEmit --watch", "clean": "rimraf \"lib\"", "format:check": "eslint \"**/*.{js,ts,tsx}\"", "prepublishOnly": "npm run clean && cross-env NODE_ENV=production bob build", @@ -90,6 +91,7 @@ "typescript": "^5.0.2" }, "dependencies": { - "@backtrace/sdk-core": "^0.6.0" + "@backtrace/sdk-core": "^0.6.0", + "web-streams-polyfill": "^4.0.0" } } diff --git a/packages/react-native/src/storage/FileSystem.ts b/packages/react-native/src/storage/FileSystem.ts index 02b09973..7a5d50c3 100644 --- a/packages/react-native/src/storage/FileSystem.ts +++ b/packages/react-native/src/storage/FileSystem.ts @@ -1,8 +1,9 @@ import { type FileSystem as CoreFileSystem } from '@backtrace/sdk-core'; -import { type StreamWriter } from './StreamWriter'; +import type { FileWritableStream, StreamWriter } from './StreamWriter'; export interface FileSystem extends CoreFileSystem { copy(sourceFile: string, destinationFile: string): Promise; copySync(sourceFile: string, destinationFile: string): boolean; applicationDirectory(): string; streamWriter: StreamWriter; + createWriteStream(path: string): FileWritableStream; } diff --git a/packages/react-native/src/storage/ReactNativeFileSystem.ts b/packages/react-native/src/storage/ReactNativeFileSystem.ts index f8c8d6e2..f9c0b604 100644 --- a/packages/react-native/src/storage/ReactNativeFileSystem.ts +++ b/packages/react-native/src/storage/ReactNativeFileSystem.ts @@ -4,7 +4,7 @@ import { BacktraceFileAttachment } from '../attachment/BacktraceFileAttachment'; import { type FileSystem } from './FileSystem'; import { type ReactNativeDirectoryProvider } from './ReactNativeDirectoryProvider'; import { type ReactNativeFileProvider } from './ReactNativeFileProvider'; -import { type StreamWriter } from './StreamWriter'; +import { FileWritableStream, type StreamWriter } from './StreamWriter'; export class ReactNativeFileSystem implements FileSystem { private readonly _fileSystemProvider: ReactNativeFileProvider = NativeModules.BacktraceFileSystemProvider; private readonly _directoryProvider: ReactNativeDirectoryProvider = NativeModules.BacktraceDirectoryProvider; @@ -89,4 +89,8 @@ export class ReactNativeFileSystem implements FileSystem { public createAttachment(path: string, name?: string | undefined): BacktraceAttachment { return new BacktraceFileAttachment(this, path, name); } + + public createWriteStream(path: string): FileWritableStream { + return new FileWritableStream(path, this.streamWriter); + } } diff --git a/packages/react-native/src/storage/StreamWriter.ts b/packages/react-native/src/storage/StreamWriter.ts index c618f2e0..371de052 100644 --- a/packages/react-native/src/storage/StreamWriter.ts +++ b/packages/react-native/src/storage/StreamWriter.ts @@ -1,9 +1,11 @@ +import { WritableStream } from 'web-streams-polyfill'; + export interface StreamWriter { /** * Creates a new stream writer. Returns a key to stream writer. * @param source path to the file */ - create(source: string): string; + create(source: string): string | undefined; /** * Appends a string to a file using a stream writer pointed by the key * @param key stream writer key @@ -17,3 +19,50 @@ export interface StreamWriter { */ close(key: string): boolean; } + +export class FileWritableStream extends WritableStream { + constructor(public readonly path: string, streamWriter: StreamWriter) { + super(new NativeUnderlyingSink(path, streamWriter)); + } +} + +export class NativeUnderlyingSink implements UnderlyingSink { + private _streamId?: string; + + constructor(public readonly path: string, private readonly _streamWriter: StreamWriter) {} + + public async start() { + this._streamId = this._streamWriter.create(this.path); + if (!this._streamId) { + throw new Error(`Failed to open file ${this.path}.`); + } + } + + public close() { + if (!this._streamId) { + return; + } + + if (!this._streamWriter.close(this._streamId)) { + throw new Error(`Failed to close file ${this.path}.`); + } + } + + public async write(chunk: string) { + if (!this._streamId) { + throw new Error('File is not open.'); + } + + if (!(await this._streamWriter.append(this._streamId, chunk))) { + throw new Error(`Failed to write data to file ${this.path}.`); + } + } + + public abort() { + if (!this._streamId) { + return; + } + + this._streamWriter.close(this._streamId); + } +} From 0b785cc406e9579c028fbe8baa06d3a06f04bd6c Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Fri, 8 Nov 2024 16:28:41 +0100 Subject: [PATCH 02/11] react-native: replace AlternatingFileWriter with Chunkifier --- .../src/breadcrumbs/AlternatingFileWriter.ts | 105 ------------------ .../src/breadcrumbs/FileBreadcrumbsStorage.ts | 54 +++++---- .../react-native/src/storage/Chunkifier.ts | 92 +++++++++++++++ .../react-native/src/storage/FileChunkSink.ts | 83 ++++++++++++++ .../react-native/src/storage/FileSystem.ts | 3 +- .../src/storage/lineChunkSplitter.ts | 37 ++++++ 6 files changed, 248 insertions(+), 126 deletions(-) delete mode 100644 packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts create mode 100644 packages/react-native/src/storage/Chunkifier.ts create mode 100644 packages/react-native/src/storage/FileChunkSink.ts create mode 100644 packages/react-native/src/storage/lineChunkSplitter.ts diff --git a/packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts b/packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts deleted file mode 100644 index 4dadccd9..00000000 --- a/packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { type FileSystem } from '../storage'; -import { type StreamWriter } from '../storage/StreamWriter'; - -export class AlternatingFileWriter { - private _streamId?: string; - private _count = 0; - private _disposed = false; - - private readonly _streamWriter: StreamWriter; - - private readonly _logQueue: string[] = []; - - private _currentAppendedLog?: string; - - constructor( - private readonly _mainFile: string, - private readonly _fallbackFile: string, - private readonly _fileCapacity: number, - private readonly _fileSystem: FileSystem, - ) { - if (this._fileCapacity <= 0) { - throw new Error('File capacity may not be less or equal to 0.'); - } - this._streamWriter = this._fileSystem.streamWriter; - } - - public writeLine(value: string) { - if (this._disposed) { - throw new Error('This instance has been disposed.'); - } - - this._logQueue.push(value); - if (!this._currentAppendedLog) { - this.process(); - } - } - - private process() { - this._currentAppendedLog = this._logQueue.shift(); - - if (!this._currentAppendedLog) { - return; - } - - this.prepareBreadcrumbStream(); - - if (!this._streamId) { - this._logQueue.unshift(this._currentAppendedLog); - this._currentAppendedLog = undefined; - return; - } - - // if the queue is full and we can save more item in a batch - // try to save as much as possible to speed up potential native operations - this._count += 1; - const logsToAppend = [this._currentAppendedLog]; - - const restAppendingLogs = this._logQueue.splice(0, this._fileCapacity - this._count); - this._count = this._count + restAppendingLogs.length; - logsToAppend.push(...restAppendingLogs); - - this._streamWriter - .append(this._streamId, logsToAppend.join('\n') + '\n') - .catch(() => { - // handle potential issues with appending logs. - // we can't do really too much here other than retry - // logging the error might also cause a breadcrumb loop, that we should try to avoid - this._logQueue.unshift(...logsToAppend); - }) - .finally(() => { - if (this._logQueue.length !== 0) { - return this.process(); - } else { - this._currentAppendedLog = undefined; - } - }); - } - - private prepareBreadcrumbStream() { - if (!this._streamId) { - this._streamId = this._streamWriter.create(this._mainFile); - } else if (this._count >= this._fileCapacity) { - const closeResult = this._streamWriter.close(this._streamId); - if (!closeResult) { - return; - } - this._streamId = undefined; - - const renameResult = this._fileSystem.copySync(this._mainFile, this._fallbackFile); - if (!renameResult) { - return; - } - this._streamId = this._streamWriter.create(this._mainFile); - - this._count = 0; - } - } - - public dispose() { - if (this._streamId) { - this._streamWriter.close(this._streamId); - } - this._disposed = true; - } -} diff --git a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts index 6ca27320..02da0890 100644 --- a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts +++ b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts @@ -5,13 +5,17 @@ import { SessionFiles, TimeHelper, type BacktraceAttachment, + type BacktraceAttachmentProvider, type Breadcrumb, type BreadcrumbsStorage, type RawBreadcrumb, } from '@backtrace/sdk-core'; +import { WritableStream } from 'web-streams-polyfill'; import { BacktraceFileAttachment } from '..'; import { type FileSystem } from '../storage'; -import { AlternatingFileWriter } from './AlternatingFileWriter'; +import { ChunkifierSink } from '../storage/Chunkifier'; +import { FileChunkSink } from '../storage/FileChunkSink'; +import { lineChunkSplitter } from '../storage/lineChunkSplitter'; const FILE_PREFIX = 'bt-breadcrumbs'; @@ -21,32 +25,44 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { } private _lastBreadcrumbId: number = TimeHelper.toTimestampInSec(TimeHelper.now()); - private readonly _writer: AlternatingFileWriter; + private readonly _dest: WritableStream; + private readonly _writer: WritableStreamDefaultWriter; + private readonly _sink: FileChunkSink; - constructor( - private readonly _fileSystem: FileSystem, - private readonly _mainFile: string, - private readonly _fallbackFile: string, - maximumBreadcrumbs: number, - ) { - this._writer = new AlternatingFileWriter( - _mainFile, - _fallbackFile, - Math.floor(maximumBreadcrumbs / 2), - _fileSystem, + constructor(session: SessionFiles, private readonly _fileSystem: FileSystem, maximumBreadcrumbs: number) { + this._sink = new FileChunkSink({ + maxFiles: 2, + fs: this._fileSystem, + file: (n) => session.getFileName(FileBreadcrumbsStorage.getFileName(n)), + }); + + this._dest = new WritableStream( + new ChunkifierSink({ + sink: this._sink.getSink(), + splitter: () => lineChunkSplitter(Math.ceil(maximumBreadcrumbs / 2)), + }), ); + + this._writer = this._dest.getWriter(); } public static create(fileSystem: FileSystem, session: SessionFiles, maximumBreadcrumbs: number) { - const file1 = session.getFileName(this.getFileName(0)); - const file2 = session.getFileName(this.getFileName(1)); - return new FileBreadcrumbsStorage(fileSystem, file1, file2, maximumBreadcrumbs); + return new FileBreadcrumbsStorage(session, fileSystem, maximumBreadcrumbs); } public getAttachments(): BacktraceAttachment[] { + const files = [...this._sink.files].map((f) => f.path); + return files.map( + (f, i) => new BacktraceFileAttachment(this._fileSystem, f, `bt-breadcrumbs-${i}`, 'application/json'), + ); + } + + public getAttachmentProviders(): BacktraceAttachmentProvider[] { return [ - new BacktraceFileAttachment(this._fileSystem, this._mainFile, 'bt-breadcrumbs-0'), - new BacktraceFileAttachment(this._fileSystem, this._fallbackFile, 'bt-breadcrumbs-1'), + { + get: () => this.getAttachments(), + type: 'dynamic', + }, ]; } @@ -73,7 +89,7 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { }; const breadcrumbJson = JSON.stringify(breadcrumb, jsonEscaper()); - this._writer.writeLine(breadcrumbJson); + this._writer.write(breadcrumbJson + '\n'); return id; } diff --git a/packages/react-native/src/storage/Chunkifier.ts b/packages/react-native/src/storage/Chunkifier.ts new file mode 100644 index 00000000..85af79e6 --- /dev/null +++ b/packages/react-native/src/storage/Chunkifier.ts @@ -0,0 +1,92 @@ +export type ChunkSplitterFactory = () => ChunkSplitter; + +/** + * Implementation of splitter should return either one or two `Buffer`s. + * + * The first `Buffer` will be written to the current chunk. + * If the second `Buffer` is returned, `chunkifier` will create a new chunk and write the + * second buffer to the new chunk. + */ +export type ChunkSplitter = (chunk: string) => [string, string?]; + +/** + * Implementation of chunk sink should return each time a new writable stream. + * + * `n` determines which stream it is in sequence. + */ +export type ChunkSink = (n: number) => S; + +export interface ChunkifierOptions { + /** + * Chunk splitter factory. The factory will be called when creating a new chunk. + */ + readonly splitter: ChunkSplitterFactory; + + /** + * Chunk sink. The sink will be called when creating a new chunk. + */ + readonly sink: ChunkSink; + + readonly allowEmptyChunks?: boolean; +} + +interface StreamContext { + readonly stream: WritableStream; + readonly streamWriter: WritableStreamDefaultWriter; + isEmptyChunk: boolean; +} + +export class ChunkifierSink implements UnderlyingSink { + private _context?: StreamContext; + private _splitter?: ChunkSplitter; + private _chunkCount = 0; + + constructor(private readonly _options: ChunkifierOptions) {} + + public async write(data: string): Promise { + // If data is empty from the start, forward the write directly to current stream + if (!data.length) { + return await (this._context ??= this.createStreamContext()).streamWriter.write(data); + } + + while (data) { + if (!data.length) { + break; + } + + this._splitter ??= this._options.splitter(); + const [currentChunk, nextChunk] = this._splitter(data); + if (nextChunk === undefined) { + const current = (this._context ??= this.createStreamContext()); + if (currentChunk.length) { + current.isEmptyChunk = false; + } + + return await current.streamWriter.write(currentChunk); + } + + data = nextChunk; + if (this._context ? this._context.isEmptyChunk : !currentChunk.length && !this._options.allowEmptyChunks) { + continue; + } + + const current = (this._context ??= this.createStreamContext()); + await current.streamWriter.write(currentChunk); + current.streamWriter.releaseLock(); + + // On next loop iteration, or write, create new stream again + this._context = undefined; + this._splitter = undefined; + } + } + + public async close() { + return await this._context?.streamWriter.close(); + } + + private createStreamContext(): StreamContext { + const stream = this._options.sink(this._chunkCount++); + const writer = stream.getWriter(); + return { stream, streamWriter: writer, isEmptyChunk: true }; + } +} diff --git a/packages/react-native/src/storage/FileChunkSink.ts b/packages/react-native/src/storage/FileChunkSink.ts new file mode 100644 index 00000000..70f68dd6 --- /dev/null +++ b/packages/react-native/src/storage/FileChunkSink.ts @@ -0,0 +1,83 @@ +import type { ChunkSink } from './Chunkifier'; +import type { FileSystem } from './FileSystem'; +import type { FileWritableStream } from './StreamWriter'; + +interface FileChunkSinkOptions { + /** + * Maximum number of files. + */ + readonly maxFiles: number; + + /** + * Full path to the chunk file. + */ + readonly file: (n: number) => string; + + /** + * File system to use. + */ + readonly fs: FileSystem; +} + +/** + * Chunk sink which writes data to disk. + * + * Each time a new chunk is created, a new stream is created with path provided from options. + */ +export class FileChunkSink { + private readonly _streamTracker: LimitedFifo; + + /** + * Returns all files that have been written to and are not deleted. + */ + public get files() { + return this._streamTracker.elements; + } + + constructor(private readonly _options: FileChunkSinkOptions) { + // Track files using a FIFO queue + this._streamTracker = limitedFifo(_options.maxFiles, async (stream) => { + await stream.close().finally(() => _options.fs.unlink(stream.path)); + }); + } + + /** + * Returns `ChunkSink`. Pass this to `chunkifier`. + */ + public getSink(): ChunkSink { + return (n) => { + const stream = this.createStream(n); + this._streamTracker.push(stream); + return stream; + }; + } + + private createStream(n: number) { + const path = this._options.file(n); + // TODO: What to do if this returns undefined? + return this._options.fs.createWriteStream(path); + } +} + +/** + * Limited FIFO queue. Each time the capacity is exceeded, the first element is removed + * and `onShift` is called with the removed element. + * @param capacity Maximum capacity. + */ +function limitedFifo(capacity: number, onShift: (t: T) => void) { + const elements: T[] = []; + + function push(element: T) { + elements.push(element); + if (elements.length > capacity) { + const first = elements.shift(); + if (first) { + onShift(first); + } + } + } + + return { elements: elements as readonly T[], push }; +} + +type LimitedFifo = ReturnType>; diff --git a/packages/react-native/src/storage/FileSystem.ts b/packages/react-native/src/storage/FileSystem.ts index 7a5d50c3..61329edc 100644 --- a/packages/react-native/src/storage/FileSystem.ts +++ b/packages/react-native/src/storage/FileSystem.ts @@ -1,9 +1,8 @@ import { type FileSystem as CoreFileSystem } from '@backtrace/sdk-core'; -import type { FileWritableStream, StreamWriter } from './StreamWriter'; +import type { FileWritableStream } from './StreamWriter'; export interface FileSystem extends CoreFileSystem { copy(sourceFile: string, destinationFile: string): Promise; copySync(sourceFile: string, destinationFile: string): boolean; applicationDirectory(): string; - streamWriter: StreamWriter; createWriteStream(path: string): FileWritableStream; } diff --git a/packages/react-native/src/storage/lineChunkSplitter.ts b/packages/react-native/src/storage/lineChunkSplitter.ts new file mode 100644 index 00000000..6e8f7b6b --- /dev/null +++ b/packages/react-native/src/storage/lineChunkSplitter.ts @@ -0,0 +1,37 @@ +import type { ChunkSplitter } from './Chunkifier'; + +/** + * Splits data into chunks with maximum lines. + * @param maxLines Maximum lines in one chunk. + */ +export function lineChunkSplitter(maxLines: number): ChunkSplitter { + let seen = 0; + + function findNthLine(data: string, remaining: number): [number, number] { + let lastIndex = -1; + let count = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + lastIndex = data.indexOf('\n', lastIndex + 1); + if (lastIndex === -1) { + return [-1, count]; + } + + if (remaining === ++count) { + return [lastIndex + 1, count]; + } + } + } + + return function lineChunkSplitter(data) { + const remainingLines = maxLines - seen; + const [index, count] = findNthLine(data, remainingLines); + if (index === -1) { + seen += count; + return [data]; + } + + seen = 0; + return [data.substring(0, index), data.substring(index)]; + }; +} From bb604a538ca979f33c681f53f66ef5291c38e767 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Tue, 12 Nov 2024 12:00:31 +0100 Subject: [PATCH 03/11] react-native: add Chunkifier tests --- packages/react-native/jest.config.js | 1 + packages/react-native/jest.rng.mjs | 13 ++ packages/react-native/package.json | 1 + .../tests/_helpers/blackholeChunkSink.ts | 11 ++ .../react-native/tests/_helpers/chunks.ts | 48 +++++ .../react-native/tests/_helpers/generators.ts | 82 +++++++++ .../tests/_helpers/memoryChunkSink.ts | 22 +++ .../react-native/tests/_helpers/random.ts | 6 + .../react-native/tests/_mocks/fileSystem.ts | 43 +++++ .../tests/storage/chunkifier.spec.ts | 174 ++++++++++++++++++ .../tests/storage/fileChunkSink.spec.ts | 56 ++++++ .../tests/storage/lineChunkSplitter.spec.ts | 79 ++++++++ 12 files changed, 536 insertions(+) create mode 100644 packages/react-native/jest.rng.mjs create mode 100644 packages/react-native/tests/_helpers/blackholeChunkSink.ts create mode 100644 packages/react-native/tests/_helpers/chunks.ts create mode 100644 packages/react-native/tests/_helpers/generators.ts create mode 100644 packages/react-native/tests/_helpers/memoryChunkSink.ts create mode 100644 packages/react-native/tests/_helpers/random.ts create mode 100644 packages/react-native/tests/_mocks/fileSystem.ts create mode 100644 packages/react-native/tests/storage/chunkifier.spec.ts create mode 100644 packages/react-native/tests/storage/fileChunkSink.spec.ts create mode 100644 packages/react-native/tests/storage/lineChunkSplitter.spec.ts diff --git a/packages/react-native/jest.config.js b/packages/react-native/jest.config.js index edb63f81..a6b2304b 100644 --- a/packages/react-native/jest.config.js +++ b/packages/react-native/jest.config.js @@ -3,4 +3,5 @@ module.exports = { preset: 'react-native', testEnvironment: 'node', setupFiles: ['./jest.setup.js'], + globalSetup: './jest.rng.mjs', }; diff --git a/packages/react-native/jest.rng.mjs b/packages/react-native/jest.rng.mjs new file mode 100644 index 00000000..8f685e8a --- /dev/null +++ b/packages/react-native/jest.rng.mjs @@ -0,0 +1,13 @@ +import crypto from 'crypto'; + +function getRandomSeed() { + return crypto.randomBytes(16).toString('hex'); +} + +export default function () { + if (!process.env.TEST_SEED) { + process.env.TEST_SEED = getRandomSeed(); + } + + console.log(`\n=== Using random seed ${process.env.TEST_SEED} ===`); +} diff --git a/packages/react-native/package.json b/packages/react-native/package.json index 21454c13..dad1cc63 100644 --- a/packages/react-native/package.json +++ b/packages/react-native/package.json @@ -83,6 +83,7 @@ "jest": "^29.7.0", "pod-install": "^0.1.0", "prettier": "^2.0.5", + "random-seed": "^0.3.0", "react": "18.2.0", "react-native": "^0.72.4", "react-native-builder-bob": "^0.21.3", diff --git a/packages/react-native/tests/_helpers/blackholeChunkSink.ts b/packages/react-native/tests/_helpers/blackholeChunkSink.ts new file mode 100644 index 00000000..cbc8bcc9 --- /dev/null +++ b/packages/react-native/tests/_helpers/blackholeChunkSink.ts @@ -0,0 +1,11 @@ +import { ChunkSink } from '../../src/storage/Chunkifier'; + +export function blackholeChunkSink(): ChunkSink { + return () => { + return new WritableStream({ + write() { + // Do nothing + }, + }); + }; +} diff --git a/packages/react-native/tests/_helpers/chunks.ts b/packages/react-native/tests/_helpers/chunks.ts new file mode 100644 index 00000000..0043bd47 --- /dev/null +++ b/packages/react-native/tests/_helpers/chunks.ts @@ -0,0 +1,48 @@ +import { ChunkSplitter } from '../../src/storage/Chunkifier'; + +/** + * Trims array from right until `predicate` returns `false`. + * @returns Trimmed array. + */ +function trimRightIf(t: T[], predicate: (t: T) => boolean) { + for (let i = t.length - 1; i >= 0; i--) { + if (predicate(t[i])) { + continue; + } + + return t.slice(0, i + 1); + } + + return []; +} + +export async function splitToEnd(readable: ReadableStream, splitter: ChunkSplitter) { + const results: string[][] = [[]]; + + for await (let chunk of readable) { + while (chunk) { + const [c1, c2] = splitter(chunk); + results[results.length - 1].push(c1); + if (c2 !== undefined) { + chunk = c2; + results.push([]); + } else { + break; + } + } + } + + // Remove all trailing empty arrays + return trimRightIf( + results.map((b) => b.join('')), + (t) => !t.length, + ); +} + +export function* chunkify(chunk: string, length: number) { + let i = 0; + do { + yield chunk.substring(i * length, (i + 1) * length); + i++; + } while (i * length < chunk.length); +} diff --git a/packages/react-native/tests/_helpers/generators.ts b/packages/react-native/tests/_helpers/generators.ts new file mode 100644 index 00000000..ca581698 --- /dev/null +++ b/packages/react-native/tests/_helpers/generators.ts @@ -0,0 +1,82 @@ +import readline from 'readline'; +import { Readable } from 'stream'; +import { createRng } from './random'; + +const rng = createRng(); + +export function dataStream(data: W) { + return new ReadableStream({ + start(controller) { + controller.enqueue(data); + controller.close(); + }, + }); +} + +export function generatorStream(generator: Generator) { + return new ReadableStream({ + pull(controller) { + const { value, done } = generator.next(); + if (done) { + return controller.close(); + } + controller.enqueue(value); + }, + }); +} + +export function randomLines(count: number, minLineLength: number, maxLineLength: number) { + return [...new Array(count)].map(() => rng.string(rng.intBetween(minLineLength, maxLineLength))); +} + +export async function readLines(readable: ReadableStream, lines: number) { + const result: string[] = []; + const rl = readline.createInterface(Readable.from(readable)); + + let count = 0; + for await (const line of rl) { + result.push(line); + if (++count === lines) { + break; + } + } + + rl.close(); + return result; +} + +export function randomString(count: number) { + let generated = 0; + + return new ReadableStream({ + pull(controller) { + const remaining = count - generated; + if (remaining <= 0) { + return controller.close(); + } + const chunk = rng.string(Math.min(remaining, 16384)); + generated += chunk.length; + controller.enqueue(chunk); + }, + }); +} + +export async function readToEnd(readable: ReadableStream) { + const result: string[] = []; + const reader = readable.getReader(); + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (value === undefined) { + break; + } + + result.push(value); + if (done) { + break; + } + } + + return result.join(''); +} diff --git a/packages/react-native/tests/_helpers/memoryChunkSink.ts b/packages/react-native/tests/_helpers/memoryChunkSink.ts new file mode 100644 index 00000000..29ce0727 --- /dev/null +++ b/packages/react-native/tests/_helpers/memoryChunkSink.ts @@ -0,0 +1,22 @@ +import { ChunkSink } from '../../src/storage/Chunkifier'; + +export function memoryChunkSink() { + const results: string[][] = []; + + const sink: ChunkSink = () => { + const index = results.length; + results.push([]); + + return new WritableStream({ + write(chunk) { + results[index].push(chunk); + }, + }); + }; + + const getResults = () => { + return results.map((chunks) => chunks.join('')); + }; + + return { sink, getResults }; +} diff --git a/packages/react-native/tests/_helpers/random.ts b/packages/react-native/tests/_helpers/random.ts new file mode 100644 index 00000000..c30c71ed --- /dev/null +++ b/packages/react-native/tests/_helpers/random.ts @@ -0,0 +1,6 @@ +import randomSeed from 'random-seed'; + +const seed = process.env.TEST_SEED; +export function createRng() { + return randomSeed.create(seed); +} diff --git a/packages/react-native/tests/_mocks/fileSystem.ts b/packages/react-native/tests/_mocks/fileSystem.ts new file mode 100644 index 00000000..64518bb7 --- /dev/null +++ b/packages/react-native/tests/_mocks/fileSystem.ts @@ -0,0 +1,43 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore The following import fails due to missing extension, but it cannot have one (it imports a .ts file) +import { MockedFileSystem, mockFileSystem } from '@backtrace/sdk-core/tests/_mocks/fileSystem'; +import path from 'path'; +import { FileSystem } from '../../src/storage/FileSystem'; + +export function mockStreamFileSystem(files?: Record): MockedFileSystem { + const fs = mockFileSystem(files); + + return { + ...fs, + + copy: jest.fn().mockImplementation((sourceFile, destinationFile) => { + fs.files[path.resolve(destinationFile)] = fs.files[path.resolve(sourceFile)]; + return Promise.resolve(true); + }), + + copySync: jest.fn().mockImplementation((sourceFile, destinationFile) => { + fs.files[path.resolve(destinationFile)] = fs.files[path.resolve(sourceFile)]; + return true; + }), + + applicationDirectory: jest.fn().mockImplementation(() => { + return '/'; + }), + + createWriteStream: jest.fn().mockImplementation((p: string) => { + const writable = new WritableStream({ + write(str) { + const fullPath = path.resolve(p); + if (!fs.files[fullPath]) { + fs.files[fullPath] = str; + } else { + fs.files[fullPath] += str; + } + }, + }); + + (writable as { path?: string }).path = p; + return writable; + }), + }; +} diff --git a/packages/react-native/tests/storage/chunkifier.spec.ts b/packages/react-native/tests/storage/chunkifier.spec.ts new file mode 100644 index 00000000..2d34ff0d --- /dev/null +++ b/packages/react-native/tests/storage/chunkifier.spec.ts @@ -0,0 +1,174 @@ +import { ChunkifierSink, ChunkSplitter } from '../../src/storage/Chunkifier'; +import { blackholeChunkSink } from '../_helpers/blackholeChunkSink'; +import { splitToEnd } from '../_helpers/chunks'; +import { dataStream, randomString, readToEnd } from '../_helpers/generators'; +import { memoryChunkSink } from '../_helpers/memoryChunkSink'; + +function charSplitter(char: string): ChunkSplitter { + return (chunk) => { + const index = chunk.indexOf(char); + if (index === -1) { + return [chunk]; + } + return [chunk.substring(0, index), chunk.substring(index + 1)]; + }; +} + +function noopSplitter(): ChunkSplitter { + return (c: string) => [c]; +} + +describe('ChunkifierSink', () => { + it('should call splitter function with every chunk', async () => { + const data = randomString(16384 * 10); + const splitter = jest.fn(noopSplitter()); + + const instance = new WritableStream( + new ChunkifierSink({ + sink: blackholeChunkSink(), + splitter: () => splitter, + }), + ); + + await data.pipeTo(instance); + expect(splitter).toHaveBeenCalledTimes(10); + }); + + it('should call splitter factory with every new chunk', async () => { + const data = randomString(500); + + const splitCount = 10; + let split = 0; + const splitterFactory = jest.fn( + (): ChunkSplitter => (chunk) => { + if (split < splitCount) { + split++; + return [chunk.substring(0, 10), chunk.substring(10)]; + } + return [chunk]; + }, + ); + + const instance = new WritableStream( + new ChunkifierSink({ + sink: blackholeChunkSink(), + splitter: splitterFactory, + }), + ); + + await data.pipeTo(instance); + expect(splitterFactory).toHaveBeenCalledTimes(splitCount + 1); + }); + + it('should not call sink on ChunkifierSink creation', () => { + const splitter = noopSplitter(); + const sink = jest.fn(blackholeChunkSink()); + + new ChunkifierSink({ + sink, + splitter: () => splitter, + }); + + expect(sink).not.toHaveBeenCalled(); + }); + + it('should call sink on chunk split', async () => { + const data = dataStream('bAb'); + const splitter = charSplitter('A'); + const sink = jest.fn(blackholeChunkSink()); + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + }), + ); + + await data.pipeTo(instance); + expect(sink).toHaveBeenCalledTimes(2); + }); + + it('should split data into chunks', async () => { + const dataStr = await readToEnd(randomString(5000)); + const data = dataStream(dataStr); + const splitter = charSplitter(dataStr[3]); + const { sink, getResults } = memoryChunkSink(); + + const expected = await splitToEnd(dataStream(dataStr), splitter); + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: true, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); + + it('should split data into non-empty chunks', async () => { + const dataStr = await readToEnd(randomString(5000)); + const data = dataStream(dataStr); + const splitter = charSplitter(dataStr[3]); + const { sink, getResults } = memoryChunkSink(); + + const expected = (await splitToEnd(dataStream(dataStr), splitter)).filter((b) => b.length); + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: false, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); + + it('should create empty chunks if allowEmptyChunks is true', async () => { + const dataStr = 'abcaaaabcdaaa'; + const data = dataStream(dataStr); + const splitter = charSplitter('a'); + const { sink, getResults } = memoryChunkSink(); + + const expected = ['', 'bc', '', '', '', 'bcd', '', '']; + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: true, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); + + it('should not create empty chunks if allowEmptyChunks is false', async () => { + const dataStr = 'abcaaaabcdaaa'; + const data = dataStream(dataStr); + const splitter = charSplitter('a'); + const { sink, getResults } = memoryChunkSink(); + + const expected = ['bc', 'bcd']; + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: false, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); +}); diff --git a/packages/react-native/tests/storage/fileChunkSink.spec.ts b/packages/react-native/tests/storage/fileChunkSink.spec.ts new file mode 100644 index 00000000..2cbcfcc8 --- /dev/null +++ b/packages/react-native/tests/storage/fileChunkSink.spec.ts @@ -0,0 +1,56 @@ +import path from 'path'; +import { FileChunkSink } from '../../src/storage/FileChunkSink'; +import { mockStreamFileSystem } from '../_mocks/fileSystem'; + +async function writeAndClose(stream: WritableStream, value: string) { + const writer = stream.getWriter(); + await writer.write(value); + writer.releaseLock(); +} + +function sortString(a: string, b: string) { + return a.localeCompare(b); +} + +describe('fileChunkSink', () => { + it('should create a filestream with name from filename', async () => { + const fs = mockStreamFileSystem(); + const filename = 'abc'; + const sink = new FileChunkSink({ file: () => filename, maxFiles: Infinity, fs }); + + const stream = sink.getSink()(0); + expect(stream.path).toEqual(filename); + }); + + it('should create a filestream each time it is called', async () => { + const fs = mockStreamFileSystem(); + const dir = 'test'; + const sink = new FileChunkSink({ file: (n) => path.join(dir, n.toString()), maxFiles: Infinity, fs }); + const expected = [0, 2, 5]; + + for (const n of expected) { + const stream = sink.getSink()(n); + await writeAndClose(stream, 'a'); + } + + const actual = await fs.readDir(dir); + expect(actual.sort(sortString)).toEqual(expected.map((e) => e.toString()).sort(sortString)); + }); + + it('should remove previous files if count exceeds maxFiles', async () => { + const fs = mockStreamFileSystem(); + const dir = 'test'; + const maxFiles = 3; + const sink = new FileChunkSink({ file: (n) => path.join(dir, n.toString()), maxFiles, fs }); + const files = [0, 2, 5, 6, 79, 81, 38, -1, 3]; + const expected = files.slice(-maxFiles); + + for (const n of files) { + const stream = sink.getSink()(n); + await writeAndClose(stream, 'a'); + } + + const actual = await fs.readDir(dir); + expect(actual.sort(sortString)).toEqual(expected.map((e) => e.toString()).sort(sortString)); + }); +}); diff --git a/packages/react-native/tests/storage/lineChunkSplitter.spec.ts b/packages/react-native/tests/storage/lineChunkSplitter.spec.ts new file mode 100644 index 00000000..7939c03a --- /dev/null +++ b/packages/react-native/tests/storage/lineChunkSplitter.spec.ts @@ -0,0 +1,79 @@ +import { lineChunkSplitter } from '../../src/storage/lineChunkSplitter'; +import { chunkify, splitToEnd } from '../_helpers/chunks'; +import { generatorStream, randomLines } from '../_helpers/generators'; + +function countNewlines(buffer: string) { + return [...buffer.matchAll(/\n/g)].length; +} + +async function getData(lines: number) { + return randomLines(lines, 10, 20).join('\n') + '\n'; +} + +describe('lineChunkSplitter', () => { + it('should split chunk if it has more lines than maxLines', async () => { + const maxLines = 10; + const chunk = await getData(30); + const splitter = lineChunkSplitter(maxLines); + + const [c1, c2] = splitter(chunk); + expect(countNewlines(c1)).toEqual(maxLines); // include trailing newline + expect(c2 && countNewlines(c2)).toEqual(30 - maxLines); + }); + + it('should split chunk if total seen lines is more than maxLines', async () => { + const maxLines = 100; + const chunk = await getData(30); + const splitter = lineChunkSplitter(maxLines); + + splitter(chunk); + splitter(chunk); + splitter(chunk); + const [c1, c2] = splitter(chunk); + + expect(countNewlines(c1)).toEqual(100 - 30 * 3); + expect(c2 && countNewlines(c2)).toEqual(20); + }); + + it('should not split chunk if it has less lines than maxLines', async () => { + const maxLines = 100; + const chunk = await getData(30); + const splitter = lineChunkSplitter(maxLines); + const [c1, c2] = splitter(chunk); + + expect(countNewlines(c1)).toEqual(30); + expect(c2).toBeUndefined(); + }); + + it('should not split chunk if it has maxLines lines', async () => { + const maxLines = 100; + const chunk = await getData(maxLines); + const splitter = lineChunkSplitter(maxLines); + const [c1, c2] = splitter(chunk); + + expect(countNewlines(c1)).toEqual(maxLines); + expect(c2?.length).toEqual(0); + }); + + it('should split chunk by lines', async () => { + const maxLines = 123; + const data = await getData(1000); + const splitter = lineChunkSplitter(maxLines); + const actual = await splitToEnd(generatorStream(chunkify(data, 100)), splitter); + + let seen = 0; + for (let i = 0; i < actual.length; i++) { + const chunk = actual[i]; + const start = seen; + const end = seen + chunk.length; + expect(chunk).toEqual(data.substring(start, end)); + seen += chunk.length; + + if (i === actual.length - 1) { + expect(countNewlines(chunk)).toBeLessThanOrEqual(maxLines); + } else { + expect(countNewlines(chunk)).toEqual(maxLines); + } + } + }); +}); From 7c49b28cc8f12613ec2c2a19c1ebf80ec0754436 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Tue, 12 Nov 2024 12:29:18 +0100 Subject: [PATCH 04/11] react-native: make ChunkifierSink generic --- .../react-native/src/storage/Chunkifier.ts | 48 +++++++++++-------- .../react-native/src/storage/FileChunkSink.ts | 2 +- .../src/storage/lineChunkSplitter.ts | 2 +- .../tests/_helpers/blackholeChunkSink.ts | 2 +- .../react-native/tests/_helpers/chunks.ts | 6 +-- .../tests/storage/chunkifier.spec.ts | 10 ++-- 6 files changed, 40 insertions(+), 30 deletions(-) diff --git a/packages/react-native/src/storage/Chunkifier.ts b/packages/react-native/src/storage/Chunkifier.ts index 85af79e6..59300f8b 100644 --- a/packages/react-native/src/storage/Chunkifier.ts +++ b/packages/react-native/src/storage/Chunkifier.ts @@ -1,4 +1,4 @@ -export type ChunkSplitterFactory = () => ChunkSplitter; +export type ChunkSplitterFactory = () => ChunkSplitter; /** * Implementation of splitter should return either one or two `Buffer`s. @@ -7,50 +7,52 @@ export type ChunkSplitterFactory = () => ChunkSplitter; * If the second `Buffer` is returned, `chunkifier` will create a new chunk and write the * second buffer to the new chunk. */ -export type ChunkSplitter = (chunk: string) => [string, string?]; +export type ChunkSplitter = (chunk: W) => [W, W?]; /** * Implementation of chunk sink should return each time a new writable stream. * * `n` determines which stream it is in sequence. */ -export type ChunkSink = (n: number) => S; +export type ChunkSink = WritableStream> = (n: number) => S; -export interface ChunkifierOptions { +export type Chunk = { readonly length: number }; + +export interface ChunkifierOptions { /** * Chunk splitter factory. The factory will be called when creating a new chunk. */ - readonly splitter: ChunkSplitterFactory; + readonly splitter: ChunkSplitterFactory; /** * Chunk sink. The sink will be called when creating a new chunk. */ - readonly sink: ChunkSink; + readonly sink: ChunkSink; readonly allowEmptyChunks?: boolean; } -interface StreamContext { - readonly stream: WritableStream; - readonly streamWriter: WritableStreamDefaultWriter; +interface StreamContext { + readonly stream: WritableStream; + readonly streamWriter: WritableStreamDefaultWriter; isEmptyChunk: boolean; } -export class ChunkifierSink implements UnderlyingSink { - private _context?: StreamContext; - private _splitter?: ChunkSplitter; +export class ChunkifierSink implements UnderlyingSink { + private _context?: StreamContext; + private _splitter?: ChunkSplitter; private _chunkCount = 0; - constructor(private readonly _options: ChunkifierOptions) {} + constructor(private readonly _options: ChunkifierOptions) {} - public async write(data: string): Promise { + public async write(data: W): Promise { // If data is empty from the start, forward the write directly to current stream - if (!data.length) { + if (this.isEmpty(data)) { return await (this._context ??= this.createStreamContext()).streamWriter.write(data); } while (data) { - if (!data.length) { + if (this.isEmpty(data)) { break; } @@ -58,7 +60,7 @@ export class ChunkifierSink implements UnderlyingSink { const [currentChunk, nextChunk] = this._splitter(data); if (nextChunk === undefined) { const current = (this._context ??= this.createStreamContext()); - if (currentChunk.length) { + if (!this.isEmpty(currentChunk)) { current.isEmptyChunk = false; } @@ -66,7 +68,11 @@ export class ChunkifierSink implements UnderlyingSink { } data = nextChunk; - if (this._context ? this._context.isEmptyChunk : !currentChunk.length && !this._options.allowEmptyChunks) { + if ( + this._context + ? this._context.isEmptyChunk + : this.isEmpty(currentChunk) && !this._options.allowEmptyChunks + ) { continue; } @@ -84,9 +90,13 @@ export class ChunkifierSink implements UnderlyingSink { return await this._context?.streamWriter.close(); } - private createStreamContext(): StreamContext { + private createStreamContext(): StreamContext { const stream = this._options.sink(this._chunkCount++); const writer = stream.getWriter(); return { stream, streamWriter: writer, isEmptyChunk: true }; } + + private isEmpty(chunk: W) { + return !chunk.length; + } } diff --git a/packages/react-native/src/storage/FileChunkSink.ts b/packages/react-native/src/storage/FileChunkSink.ts index 70f68dd6..6dab1004 100644 --- a/packages/react-native/src/storage/FileChunkSink.ts +++ b/packages/react-native/src/storage/FileChunkSink.ts @@ -44,7 +44,7 @@ export class FileChunkSink { /** * Returns `ChunkSink`. Pass this to `chunkifier`. */ - public getSink(): ChunkSink { + public getSink(): ChunkSink { return (n) => { const stream = this.createStream(n); this._streamTracker.push(stream); diff --git a/packages/react-native/src/storage/lineChunkSplitter.ts b/packages/react-native/src/storage/lineChunkSplitter.ts index 6e8f7b6b..43a627e7 100644 --- a/packages/react-native/src/storage/lineChunkSplitter.ts +++ b/packages/react-native/src/storage/lineChunkSplitter.ts @@ -4,7 +4,7 @@ import type { ChunkSplitter } from './Chunkifier'; * Splits data into chunks with maximum lines. * @param maxLines Maximum lines in one chunk. */ -export function lineChunkSplitter(maxLines: number): ChunkSplitter { +export function lineChunkSplitter(maxLines: number): ChunkSplitter { let seen = 0; function findNthLine(data: string, remaining: number): [number, number] { diff --git a/packages/react-native/tests/_helpers/blackholeChunkSink.ts b/packages/react-native/tests/_helpers/blackholeChunkSink.ts index cbc8bcc9..472711ae 100644 --- a/packages/react-native/tests/_helpers/blackholeChunkSink.ts +++ b/packages/react-native/tests/_helpers/blackholeChunkSink.ts @@ -1,6 +1,6 @@ import { ChunkSink } from '../../src/storage/Chunkifier'; -export function blackholeChunkSink(): ChunkSink { +export function blackholeChunkSink(): ChunkSink { return () => { return new WritableStream({ write() { diff --git a/packages/react-native/tests/_helpers/chunks.ts b/packages/react-native/tests/_helpers/chunks.ts index 0043bd47..ffa7d1ea 100644 --- a/packages/react-native/tests/_helpers/chunks.ts +++ b/packages/react-native/tests/_helpers/chunks.ts @@ -1,4 +1,4 @@ -import { ChunkSplitter } from '../../src/storage/Chunkifier'; +import { Chunk, ChunkSplitter } from '../../src/storage/Chunkifier'; /** * Trims array from right until `predicate` returns `false`. @@ -16,8 +16,8 @@ function trimRightIf(t: T[], predicate: (t: T) => boolean) { return []; } -export async function splitToEnd(readable: ReadableStream, splitter: ChunkSplitter) { - const results: string[][] = [[]]; +export async function splitToEnd(readable: ReadableStream, splitter: ChunkSplitter) { + const results: W[][] = [[]]; for await (let chunk of readable) { while (chunk) { diff --git a/packages/react-native/tests/storage/chunkifier.spec.ts b/packages/react-native/tests/storage/chunkifier.spec.ts index 2d34ff0d..acf08dbc 100644 --- a/packages/react-native/tests/storage/chunkifier.spec.ts +++ b/packages/react-native/tests/storage/chunkifier.spec.ts @@ -1,10 +1,10 @@ -import { ChunkifierSink, ChunkSplitter } from '../../src/storage/Chunkifier'; +import { Chunk, ChunkifierSink, ChunkSplitter } from '../../src/storage/Chunkifier'; import { blackholeChunkSink } from '../_helpers/blackholeChunkSink'; import { splitToEnd } from '../_helpers/chunks'; import { dataStream, randomString, readToEnd } from '../_helpers/generators'; import { memoryChunkSink } from '../_helpers/memoryChunkSink'; -function charSplitter(char: string): ChunkSplitter { +function charSplitter(char: string): ChunkSplitter { return (chunk) => { const index = chunk.indexOf(char); if (index === -1) { @@ -14,8 +14,8 @@ function charSplitter(char: string): ChunkSplitter { }; } -function noopSplitter(): ChunkSplitter { - return (c: string) => [c]; +function noopSplitter(): ChunkSplitter { + return (c: W) => [c]; } describe('ChunkifierSink', () => { @@ -40,7 +40,7 @@ describe('ChunkifierSink', () => { const splitCount = 10; let split = 0; const splitterFactory = jest.fn( - (): ChunkSplitter => (chunk) => { + (): ChunkSplitter => (chunk) => { if (split < splitCount) { split++; return [chunk.substring(0, 10), chunk.substring(10)]; From 3ebf7c113cfeed0560081290836f21742afb6765 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Tue, 12 Nov 2024 12:51:19 +0100 Subject: [PATCH 05/11] react-native: fix formatting issues --- .../src/breadcrumbs/FileBreadcrumbsStorage.ts | 6 +++++- packages/react-native/src/storage/StreamWriter.ts | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts index 02da0890..695c9979 100644 --- a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts +++ b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts @@ -29,7 +29,11 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { private readonly _writer: WritableStreamDefaultWriter; private readonly _sink: FileChunkSink; - constructor(session: SessionFiles, private readonly _fileSystem: FileSystem, maximumBreadcrumbs: number) { + constructor( + session: SessionFiles, + private readonly _fileSystem: FileSystem, + maximumBreadcrumbs: number, + ) { this._sink = new FileChunkSink({ maxFiles: 2, fs: this._fileSystem, diff --git a/packages/react-native/src/storage/StreamWriter.ts b/packages/react-native/src/storage/StreamWriter.ts index 371de052..1e36bfdb 100644 --- a/packages/react-native/src/storage/StreamWriter.ts +++ b/packages/react-native/src/storage/StreamWriter.ts @@ -21,7 +21,10 @@ export interface StreamWriter { } export class FileWritableStream extends WritableStream { - constructor(public readonly path: string, streamWriter: StreamWriter) { + constructor( + public readonly path: string, + streamWriter: StreamWriter, + ) { super(new NativeUnderlyingSink(path, streamWriter)); } } @@ -29,7 +32,10 @@ export class FileWritableStream extends WritableStream { export class NativeUnderlyingSink implements UnderlyingSink { private _streamId?: string; - constructor(public readonly path: string, private readonly _streamWriter: StreamWriter) {} + constructor( + public readonly path: string, + private readonly _streamWriter: StreamWriter, + ) {} public async start() { this._streamId = this._streamWriter.create(this.path); From ad9f004a0265bcfbe730a967e2fade40c5d5dc04 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Tue, 12 Nov 2024 13:01:11 +0100 Subject: [PATCH 06/11] react-native: import ponyfills in tests --- packages/react-native/tests/_helpers/blackholeChunkSink.ts | 1 + packages/react-native/tests/_helpers/chunks.ts | 1 + packages/react-native/tests/_helpers/generators.ts | 1 + packages/react-native/tests/_helpers/memoryChunkSink.ts | 5 +++-- packages/react-native/tests/_mocks/fileSystem.ts | 3 +-- packages/react-native/tests/storage/chunkifier.spec.ts | 1 + packages/react-native/tests/storage/fileChunkSink.spec.ts | 1 + 7 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/react-native/tests/_helpers/blackholeChunkSink.ts b/packages/react-native/tests/_helpers/blackholeChunkSink.ts index 472711ae..3cd17a51 100644 --- a/packages/react-native/tests/_helpers/blackholeChunkSink.ts +++ b/packages/react-native/tests/_helpers/blackholeChunkSink.ts @@ -1,3 +1,4 @@ +import { WritableStream } from 'web-streams-polyfill'; import { ChunkSink } from '../../src/storage/Chunkifier'; export function blackholeChunkSink(): ChunkSink { diff --git a/packages/react-native/tests/_helpers/chunks.ts b/packages/react-native/tests/_helpers/chunks.ts index ffa7d1ea..d10e46bc 100644 --- a/packages/react-native/tests/_helpers/chunks.ts +++ b/packages/react-native/tests/_helpers/chunks.ts @@ -1,3 +1,4 @@ +import { ReadableStream } from 'web-streams-polyfill'; import { Chunk, ChunkSplitter } from '../../src/storage/Chunkifier'; /** diff --git a/packages/react-native/tests/_helpers/generators.ts b/packages/react-native/tests/_helpers/generators.ts index ca581698..b1a95736 100644 --- a/packages/react-native/tests/_helpers/generators.ts +++ b/packages/react-native/tests/_helpers/generators.ts @@ -1,5 +1,6 @@ import readline from 'readline'; import { Readable } from 'stream'; +import { ReadableStream } from 'web-streams-polyfill'; import { createRng } from './random'; const rng = createRng(); diff --git a/packages/react-native/tests/_helpers/memoryChunkSink.ts b/packages/react-native/tests/_helpers/memoryChunkSink.ts index 29ce0727..692bac85 100644 --- a/packages/react-native/tests/_helpers/memoryChunkSink.ts +++ b/packages/react-native/tests/_helpers/memoryChunkSink.ts @@ -1,13 +1,14 @@ +import { WritableStream } from 'web-streams-polyfill'; import { ChunkSink } from '../../src/storage/Chunkifier'; export function memoryChunkSink() { const results: string[][] = []; - const sink: ChunkSink = () => { + const sink: ChunkSink = () => { const index = results.length; results.push([]); - return new WritableStream({ + return new WritableStream({ write(chunk) { results[index].push(chunk); }, diff --git a/packages/react-native/tests/_mocks/fileSystem.ts b/packages/react-native/tests/_mocks/fileSystem.ts index 64518bb7..5a747f5d 100644 --- a/packages/react-native/tests/_mocks/fileSystem.ts +++ b/packages/react-native/tests/_mocks/fileSystem.ts @@ -1,7 +1,6 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore The following import fails due to missing extension, but it cannot have one (it imports a .ts file) import { MockedFileSystem, mockFileSystem } from '@backtrace/sdk-core/tests/_mocks/fileSystem'; import path from 'path'; +import { WritableStream } from 'web-streams-polyfill'; import { FileSystem } from '../../src/storage/FileSystem'; export function mockStreamFileSystem(files?: Record): MockedFileSystem { diff --git a/packages/react-native/tests/storage/chunkifier.spec.ts b/packages/react-native/tests/storage/chunkifier.spec.ts index acf08dbc..bbe361bb 100644 --- a/packages/react-native/tests/storage/chunkifier.spec.ts +++ b/packages/react-native/tests/storage/chunkifier.spec.ts @@ -1,3 +1,4 @@ +import { WritableStream } from 'web-streams-polyfill'; import { Chunk, ChunkifierSink, ChunkSplitter } from '../../src/storage/Chunkifier'; import { blackholeChunkSink } from '../_helpers/blackholeChunkSink'; import { splitToEnd } from '../_helpers/chunks'; diff --git a/packages/react-native/tests/storage/fileChunkSink.spec.ts b/packages/react-native/tests/storage/fileChunkSink.spec.ts index 2cbcfcc8..8fd302cf 100644 --- a/packages/react-native/tests/storage/fileChunkSink.spec.ts +++ b/packages/react-native/tests/storage/fileChunkSink.spec.ts @@ -1,4 +1,5 @@ import path from 'path'; +import { WritableStream } from 'web-streams-polyfill'; import { FileChunkSink } from '../../src/storage/FileChunkSink'; import { mockStreamFileSystem } from '../_mocks/fileSystem'; From 02d5fed819e656969a56bf6eb07fedf6a6fa9209 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Wed, 13 Nov 2024 10:44:55 +0100 Subject: [PATCH 07/11] react-native: replace null assignments with ensure functions --- .../react-native/src/storage/Chunkifier.ts | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/react-native/src/storage/Chunkifier.ts b/packages/react-native/src/storage/Chunkifier.ts index 59300f8b..944d9ce5 100644 --- a/packages/react-native/src/storage/Chunkifier.ts +++ b/packages/react-native/src/storage/Chunkifier.ts @@ -48,7 +48,7 @@ export class ChunkifierSink implements UnderlyingSink { public async write(data: W): Promise { // If data is empty from the start, forward the write directly to current stream if (this.isEmpty(data)) { - return await (this._context ??= this.createStreamContext()).streamWriter.write(data); + return await this.ensureStreamContext().streamWriter.write(data); } while (data) { @@ -56,10 +56,10 @@ export class ChunkifierSink implements UnderlyingSink { break; } - this._splitter ??= this._options.splitter(); - const [currentChunk, nextChunk] = this._splitter(data); + const splitter = this.ensureSplitter(); + const [currentChunk, nextChunk] = splitter(data); if (nextChunk === undefined) { - const current = (this._context ??= this.createStreamContext()); + const current = this.ensureStreamContext(); if (!this.isEmpty(currentChunk)) { current.isEmptyChunk = false; } @@ -76,13 +76,12 @@ export class ChunkifierSink implements UnderlyingSink { continue; } - const current = (this._context ??= this.createStreamContext()); + const current = this.ensureStreamContext(); await current.streamWriter.write(currentChunk); current.streamWriter.releaseLock(); // On next loop iteration, or write, create new stream again - this._context = undefined; - this._splitter = undefined; + this.reset(); } } @@ -90,12 +89,31 @@ export class ChunkifierSink implements UnderlyingSink { return await this._context?.streamWriter.close(); } + private ensureStreamContext() { + if (!this._context) { + return (this._context = this.createStreamContext()); + } + return this._context; + } + + private ensureSplitter() { + if (!this._splitter) { + return (this._splitter = this._options.splitter()); + } + return this._splitter; + } + private createStreamContext(): StreamContext { const stream = this._options.sink(this._chunkCount++); const writer = stream.getWriter(); return { stream, streamWriter: writer, isEmptyChunk: true }; } + private reset() { + this._context = undefined; + this._splitter = undefined; + } + private isEmpty(chunk: W) { return !chunk.length; } From 22d9778f32409a499acf4bfb3b2f2c53cc2a129b Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Fri, 15 Nov 2024 12:08:02 +0100 Subject: [PATCH 08/11] react-native: rename variables in FileBreadcrumbsStorage --- .../src/breadcrumbs/FileBreadcrumbsStorage.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts index 695c9979..0e6a9bdb 100644 --- a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts +++ b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts @@ -25,8 +25,8 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { } private _lastBreadcrumbId: number = TimeHelper.toTimestampInSec(TimeHelper.now()); - private readonly _dest: WritableStream; - private readonly _writer: WritableStreamDefaultWriter; + private readonly _destinationStream: WritableStream; + private readonly _destinationWriter: WritableStreamDefaultWriter; private readonly _sink: FileChunkSink; constructor( @@ -40,14 +40,14 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { file: (n) => session.getFileName(FileBreadcrumbsStorage.getFileName(n)), }); - this._dest = new WritableStream( + this._destinationStream = new WritableStream( new ChunkifierSink({ sink: this._sink.getSink(), splitter: () => lineChunkSplitter(Math.ceil(maximumBreadcrumbs / 2)), }), ); - this._writer = this._dest.getWriter(); + this._destinationWriter = this._destinationStream.getWriter(); } public static create(fileSystem: FileSystem, session: SessionFiles, maximumBreadcrumbs: number) { @@ -93,7 +93,7 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { }; const breadcrumbJson = JSON.stringify(breadcrumb, jsonEscaper()); - this._writer.write(breadcrumbJson + '\n'); + this._destinationWriter.write(breadcrumbJson + '\n'); return id; } From fed75297ca8b375d348b93871728e492f312648e Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Fri, 15 Nov 2024 12:10:16 +0100 Subject: [PATCH 09/11] react-native: catch breadcrumb errors silently --- .../src/breadcrumbs/FileBreadcrumbsStorage.ts | 4 +++- packages/react-native/src/storage/FileChunkSink.ts | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts index 0e6a9bdb..95750f40 100644 --- a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts +++ b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts @@ -93,7 +93,9 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { }; const breadcrumbJson = JSON.stringify(breadcrumb, jsonEscaper()); - this._destinationWriter.write(breadcrumbJson + '\n'); + this._destinationWriter.write(breadcrumbJson + '\n').catch(() => { + // Fail silently here, there's not much we can do about this + }); return id; } diff --git a/packages/react-native/src/storage/FileChunkSink.ts b/packages/react-native/src/storage/FileChunkSink.ts index 6dab1004..32d11b36 100644 --- a/packages/react-native/src/storage/FileChunkSink.ts +++ b/packages/react-native/src/storage/FileChunkSink.ts @@ -37,7 +37,16 @@ export class FileChunkSink { constructor(private readonly _options: FileChunkSinkOptions) { // Track files using a FIFO queue this._streamTracker = limitedFifo(_options.maxFiles, async (stream) => { - await stream.close().finally(() => _options.fs.unlink(stream.path)); + await stream + .close() + .catch(() => { + // Fail silently here, there's not much we can do about this + }) + .finally(() => + _options.fs.unlink(stream.path).catch(() => { + // Fail silently here, there's not much we can do about this + }), + ); }); } @@ -54,7 +63,6 @@ export class FileChunkSink { private createStream(n: number) { const path = this._options.file(n); - // TODO: What to do if this returns undefined? return this._options.fs.createWriteStream(path); } } From 92a243963da1bc5ca12554974255f31b21d41e17 Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Fri, 15 Nov 2024 12:10:39 +0100 Subject: [PATCH 10/11] react-native: add test case for chunkifier not calling splitter (NFC) --- .../react-native/tests/storage/chunkifier.spec.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/react-native/tests/storage/chunkifier.spec.ts b/packages/react-native/tests/storage/chunkifier.spec.ts index bbe361bb..f5a1e413 100644 --- a/packages/react-native/tests/storage/chunkifier.spec.ts +++ b/packages/react-native/tests/storage/chunkifier.spec.ts @@ -35,6 +35,21 @@ describe('ChunkifierSink', () => { expect(splitter).toHaveBeenCalledTimes(10); }); + it('should not call splitter function chunk if data is empty', async () => { + const data = dataStream(''); + const splitter = jest.fn(noopSplitter()); + + const instance = new WritableStream( + new ChunkifierSink({ + sink: blackholeChunkSink(), + splitter: () => splitter, + }), + ); + + await data.pipeTo(instance); + expect(splitter).not.toHaveBeenCalled(); + }); + it('should call splitter factory with every new chunk', async () => { const data = randomString(500); From 414e2a9f26452ff4b03c5b9ab268657b6532380f Mon Sep 17 00:00:00 2001 From: Sebastian Alex Date: Fri, 15 Nov 2024 12:13:06 +0100 Subject: [PATCH 11/11] react-native: add comments to ChunkifierSink.reset --- packages/react-native/src/storage/Chunkifier.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/react-native/src/storage/Chunkifier.ts b/packages/react-native/src/storage/Chunkifier.ts index 944d9ce5..a4d16883 100644 --- a/packages/react-native/src/storage/Chunkifier.ts +++ b/packages/react-native/src/storage/Chunkifier.ts @@ -109,8 +109,13 @@ export class ChunkifierSink implements UnderlyingSink { return { stream, streamWriter: writer, isEmptyChunk: true }; } + /** + * Resets the chunkifier to it's initial state. Use when switching streams. + */ private reset() { this._context = undefined; + + // Splitter may have an internal state which we need to recreate with new stream this._splitter = undefined; }