diff --git a/package-lock.json b/package-lock.json index ef1b9c3a..f83ed57a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24211,7 +24211,8 @@ "version": "0.1.1", "license": "MIT", "dependencies": { - "@backtrace/sdk-core": "^0.6.0" + "@backtrace/sdk-core": "^0.6.0", + "web-streams-polyfill": "^4.0.0" }, "devDependencies": { "@react-native-community/eslint-config": "^3.0.2", @@ -24250,6 +24251,14 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "packages/react-native/node_modules/web-streams-polyfill": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz", + "integrity": "sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw==", + "engines": { + "node": ">= 8" + } + }, "packages/react/node_modules/@testing-library/react": { "version": "14.3.1", "dev": true, diff --git a/packages/react-native/jest.config.js b/packages/react-native/jest.config.js index edb63f81..a6b2304b 100644 --- a/packages/react-native/jest.config.js +++ b/packages/react-native/jest.config.js @@ -3,4 +3,5 @@ module.exports = { preset: 'react-native', testEnvironment: 'node', setupFiles: ['./jest.setup.js'], + globalSetup: './jest.rng.mjs', }; diff --git a/packages/react-native/jest.rng.mjs b/packages/react-native/jest.rng.mjs new file mode 100644 index 00000000..8f685e8a --- /dev/null +++ b/packages/react-native/jest.rng.mjs @@ -0,0 +1,13 @@ +import crypto from 'crypto'; + +function getRandomSeed() { + return crypto.randomBytes(16).toString('hex'); +} + +export default function () { + if (!process.env.TEST_SEED) { + process.env.TEST_SEED = getRandomSeed(); + } + + console.log(`\n=== Using random seed ${process.env.TEST_SEED} ===`); +} diff --git a/packages/react-native/package.json b/packages/react-native/package.json index 1b194451..dad1cc63 100644 --- a/packages/react-native/package.json +++ b/packages/react-native/package.json @@ -28,6 +28,7 @@ ], "scripts": { "build": "bob build", + "watch": "tsc --noEmit --watch", "clean": "rimraf \"lib\"", "format:check": "eslint \"**/*.{js,ts,tsx}\"", "prepublishOnly": "npm run clean && cross-env NODE_ENV=production bob build", @@ -82,6 +83,7 @@ "jest": "^29.7.0", "pod-install": "^0.1.0", "prettier": "^2.0.5", + "random-seed": "^0.3.0", "react": "18.2.0", "react-native": "^0.72.4", "react-native-builder-bob": "^0.21.3", @@ -90,6 +92,7 @@ "typescript": "^5.0.2" }, "dependencies": { - "@backtrace/sdk-core": "^0.6.0" + "@backtrace/sdk-core": "^0.6.0", + "web-streams-polyfill": "^4.0.0" } } diff --git a/packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts b/packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts deleted file mode 100644 index 4dadccd9..00000000 --- a/packages/react-native/src/breadcrumbs/AlternatingFileWriter.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { type FileSystem } from '../storage'; -import { type StreamWriter } from '../storage/StreamWriter'; - -export class AlternatingFileWriter { - private _streamId?: string; - private _count = 0; - private _disposed = false; - - private readonly _streamWriter: StreamWriter; - - private readonly _logQueue: string[] = []; - - private _currentAppendedLog?: string; - - constructor( - private readonly _mainFile: string, - private readonly _fallbackFile: string, - private readonly _fileCapacity: number, - private readonly _fileSystem: FileSystem, - ) { - if (this._fileCapacity <= 0) { - throw new Error('File capacity may not be less or equal to 0.'); - } - this._streamWriter = this._fileSystem.streamWriter; - } - - public writeLine(value: string) { - if (this._disposed) { - throw new Error('This instance has been disposed.'); - } - - this._logQueue.push(value); - if (!this._currentAppendedLog) { - this.process(); - } - } - - private process() { - this._currentAppendedLog = this._logQueue.shift(); - - if (!this._currentAppendedLog) { - return; - } - - this.prepareBreadcrumbStream(); - - if (!this._streamId) { - this._logQueue.unshift(this._currentAppendedLog); - this._currentAppendedLog = undefined; - return; - } - - // if the queue is full and we can save more item in a batch - // try to save as much as possible to speed up potential native operations - this._count += 1; - const logsToAppend = [this._currentAppendedLog]; - - const restAppendingLogs = this._logQueue.splice(0, this._fileCapacity - this._count); - this._count = this._count + restAppendingLogs.length; - logsToAppend.push(...restAppendingLogs); - - this._streamWriter - .append(this._streamId, logsToAppend.join('\n') + '\n') - .catch(() => { - // handle potential issues with appending logs. - // we can't do really too much here other than retry - // logging the error might also cause a breadcrumb loop, that we should try to avoid - this._logQueue.unshift(...logsToAppend); - }) - .finally(() => { - if (this._logQueue.length !== 0) { - return this.process(); - } else { - this._currentAppendedLog = undefined; - } - }); - } - - private prepareBreadcrumbStream() { - if (!this._streamId) { - this._streamId = this._streamWriter.create(this._mainFile); - } else if (this._count >= this._fileCapacity) { - const closeResult = this._streamWriter.close(this._streamId); - if (!closeResult) { - return; - } - this._streamId = undefined; - - const renameResult = this._fileSystem.copySync(this._mainFile, this._fallbackFile); - if (!renameResult) { - return; - } - this._streamId = this._streamWriter.create(this._mainFile); - - this._count = 0; - } - } - - public dispose() { - if (this._streamId) { - this._streamWriter.close(this._streamId); - } - this._disposed = true; - } -} diff --git a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts index 6ca27320..95750f40 100644 --- a/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts +++ b/packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts @@ -5,13 +5,17 @@ import { SessionFiles, TimeHelper, type BacktraceAttachment, + type BacktraceAttachmentProvider, type Breadcrumb, type BreadcrumbsStorage, type RawBreadcrumb, } from '@backtrace/sdk-core'; +import { WritableStream } from 'web-streams-polyfill'; import { BacktraceFileAttachment } from '..'; import { type FileSystem } from '../storage'; -import { AlternatingFileWriter } from './AlternatingFileWriter'; +import { ChunkifierSink } from '../storage/Chunkifier'; +import { FileChunkSink } from '../storage/FileChunkSink'; +import { lineChunkSplitter } from '../storage/lineChunkSplitter'; const FILE_PREFIX = 'bt-breadcrumbs'; @@ -21,32 +25,48 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { } private _lastBreadcrumbId: number = TimeHelper.toTimestampInSec(TimeHelper.now()); - private readonly _writer: AlternatingFileWriter; + private readonly _destinationStream: WritableStream; + private readonly _destinationWriter: WritableStreamDefaultWriter; + private readonly _sink: FileChunkSink; constructor( + session: SessionFiles, private readonly _fileSystem: FileSystem, - private readonly _mainFile: string, - private readonly _fallbackFile: string, maximumBreadcrumbs: number, ) { - this._writer = new AlternatingFileWriter( - _mainFile, - _fallbackFile, - Math.floor(maximumBreadcrumbs / 2), - _fileSystem, + this._sink = new FileChunkSink({ + maxFiles: 2, + fs: this._fileSystem, + file: (n) => session.getFileName(FileBreadcrumbsStorage.getFileName(n)), + }); + + this._destinationStream = new WritableStream( + new ChunkifierSink({ + sink: this._sink.getSink(), + splitter: () => lineChunkSplitter(Math.ceil(maximumBreadcrumbs / 2)), + }), ); + + this._destinationWriter = this._destinationStream.getWriter(); } public static create(fileSystem: FileSystem, session: SessionFiles, maximumBreadcrumbs: number) { - const file1 = session.getFileName(this.getFileName(0)); - const file2 = session.getFileName(this.getFileName(1)); - return new FileBreadcrumbsStorage(fileSystem, file1, file2, maximumBreadcrumbs); + return new FileBreadcrumbsStorage(session, fileSystem, maximumBreadcrumbs); } public getAttachments(): BacktraceAttachment[] { + const files = [...this._sink.files].map((f) => f.path); + return files.map( + (f, i) => new BacktraceFileAttachment(this._fileSystem, f, `bt-breadcrumbs-${i}`, 'application/json'), + ); + } + + public getAttachmentProviders(): BacktraceAttachmentProvider[] { return [ - new BacktraceFileAttachment(this._fileSystem, this._mainFile, 'bt-breadcrumbs-0'), - new BacktraceFileAttachment(this._fileSystem, this._fallbackFile, 'bt-breadcrumbs-1'), + { + get: () => this.getAttachments(), + type: 'dynamic', + }, ]; } @@ -73,7 +93,9 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage { }; const breadcrumbJson = JSON.stringify(breadcrumb, jsonEscaper()); - this._writer.writeLine(breadcrumbJson); + this._destinationWriter.write(breadcrumbJson + '\n').catch(() => { + // Fail silently here, there's not much we can do about this + }); return id; } diff --git a/packages/react-native/src/storage/Chunkifier.ts b/packages/react-native/src/storage/Chunkifier.ts new file mode 100644 index 00000000..a4d16883 --- /dev/null +++ b/packages/react-native/src/storage/Chunkifier.ts @@ -0,0 +1,125 @@ +export type ChunkSplitterFactory = () => ChunkSplitter; + +/** + * Implementation of splitter should return either one or two `Buffer`s. + * + * The first `Buffer` will be written to the current chunk. + * If the second `Buffer` is returned, `chunkifier` will create a new chunk and write the + * second buffer to the new chunk. + */ +export type ChunkSplitter = (chunk: W) => [W, W?]; + +/** + * Implementation of chunk sink should return each time a new writable stream. + * + * `n` determines which stream it is in sequence. + */ +export type ChunkSink = WritableStream> = (n: number) => S; + +export type Chunk = { readonly length: number }; + +export interface ChunkifierOptions { + /** + * Chunk splitter factory. The factory will be called when creating a new chunk. + */ + readonly splitter: ChunkSplitterFactory; + + /** + * Chunk sink. The sink will be called when creating a new chunk. + */ + readonly sink: ChunkSink; + + readonly allowEmptyChunks?: boolean; +} + +interface StreamContext { + readonly stream: WritableStream; + readonly streamWriter: WritableStreamDefaultWriter; + isEmptyChunk: boolean; +} + +export class ChunkifierSink implements UnderlyingSink { + private _context?: StreamContext; + private _splitter?: ChunkSplitter; + private _chunkCount = 0; + + constructor(private readonly _options: ChunkifierOptions) {} + + public async write(data: W): Promise { + // If data is empty from the start, forward the write directly to current stream + if (this.isEmpty(data)) { + return await this.ensureStreamContext().streamWriter.write(data); + } + + while (data) { + if (this.isEmpty(data)) { + break; + } + + const splitter = this.ensureSplitter(); + const [currentChunk, nextChunk] = splitter(data); + if (nextChunk === undefined) { + const current = this.ensureStreamContext(); + if (!this.isEmpty(currentChunk)) { + current.isEmptyChunk = false; + } + + return await current.streamWriter.write(currentChunk); + } + + data = nextChunk; + if ( + this._context + ? this._context.isEmptyChunk + : this.isEmpty(currentChunk) && !this._options.allowEmptyChunks + ) { + continue; + } + + const current = this.ensureStreamContext(); + await current.streamWriter.write(currentChunk); + current.streamWriter.releaseLock(); + + // On next loop iteration, or write, create new stream again + this.reset(); + } + } + + public async close() { + return await this._context?.streamWriter.close(); + } + + private ensureStreamContext() { + if (!this._context) { + return (this._context = this.createStreamContext()); + } + return this._context; + } + + private ensureSplitter() { + if (!this._splitter) { + return (this._splitter = this._options.splitter()); + } + return this._splitter; + } + + private createStreamContext(): StreamContext { + const stream = this._options.sink(this._chunkCount++); + const writer = stream.getWriter(); + return { stream, streamWriter: writer, isEmptyChunk: true }; + } + + /** + * Resets the chunkifier to it's initial state. Use when switching streams. + */ + private reset() { + this._context = undefined; + + // Splitter may have an internal state which we need to recreate with new stream + this._splitter = undefined; + } + + private isEmpty(chunk: W) { + return !chunk.length; + } +} diff --git a/packages/react-native/src/storage/FileChunkSink.ts b/packages/react-native/src/storage/FileChunkSink.ts new file mode 100644 index 00000000..32d11b36 --- /dev/null +++ b/packages/react-native/src/storage/FileChunkSink.ts @@ -0,0 +1,91 @@ +import type { ChunkSink } from './Chunkifier'; +import type { FileSystem } from './FileSystem'; +import type { FileWritableStream } from './StreamWriter'; + +interface FileChunkSinkOptions { + /** + * Maximum number of files. + */ + readonly maxFiles: number; + + /** + * Full path to the chunk file. + */ + readonly file: (n: number) => string; + + /** + * File system to use. + */ + readonly fs: FileSystem; +} + +/** + * Chunk sink which writes data to disk. + * + * Each time a new chunk is created, a new stream is created with path provided from options. + */ +export class FileChunkSink { + private readonly _streamTracker: LimitedFifo; + + /** + * Returns all files that have been written to and are not deleted. + */ + public get files() { + return this._streamTracker.elements; + } + + constructor(private readonly _options: FileChunkSinkOptions) { + // Track files using a FIFO queue + this._streamTracker = limitedFifo(_options.maxFiles, async (stream) => { + await stream + .close() + .catch(() => { + // Fail silently here, there's not much we can do about this + }) + .finally(() => + _options.fs.unlink(stream.path).catch(() => { + // Fail silently here, there's not much we can do about this + }), + ); + }); + } + + /** + * Returns `ChunkSink`. Pass this to `chunkifier`. + */ + public getSink(): ChunkSink { + return (n) => { + const stream = this.createStream(n); + this._streamTracker.push(stream); + return stream; + }; + } + + private createStream(n: number) { + const path = this._options.file(n); + return this._options.fs.createWriteStream(path); + } +} + +/** + * Limited FIFO queue. Each time the capacity is exceeded, the first element is removed + * and `onShift` is called with the removed element. + * @param capacity Maximum capacity. + */ +function limitedFifo(capacity: number, onShift: (t: T) => void) { + const elements: T[] = []; + + function push(element: T) { + elements.push(element); + if (elements.length > capacity) { + const first = elements.shift(); + if (first) { + onShift(first); + } + } + } + + return { elements: elements as readonly T[], push }; +} + +type LimitedFifo = ReturnType>; diff --git a/packages/react-native/src/storage/FileSystem.ts b/packages/react-native/src/storage/FileSystem.ts index 02b09973..61329edc 100644 --- a/packages/react-native/src/storage/FileSystem.ts +++ b/packages/react-native/src/storage/FileSystem.ts @@ -1,8 +1,8 @@ import { type FileSystem as CoreFileSystem } from '@backtrace/sdk-core'; -import { type StreamWriter } from './StreamWriter'; +import type { FileWritableStream } from './StreamWriter'; export interface FileSystem extends CoreFileSystem { copy(sourceFile: string, destinationFile: string): Promise; copySync(sourceFile: string, destinationFile: string): boolean; applicationDirectory(): string; - streamWriter: StreamWriter; + createWriteStream(path: string): FileWritableStream; } diff --git a/packages/react-native/src/storage/ReactNativeFileSystem.ts b/packages/react-native/src/storage/ReactNativeFileSystem.ts index f8c8d6e2..f9c0b604 100644 --- a/packages/react-native/src/storage/ReactNativeFileSystem.ts +++ b/packages/react-native/src/storage/ReactNativeFileSystem.ts @@ -4,7 +4,7 @@ import { BacktraceFileAttachment } from '../attachment/BacktraceFileAttachment'; import { type FileSystem } from './FileSystem'; import { type ReactNativeDirectoryProvider } from './ReactNativeDirectoryProvider'; import { type ReactNativeFileProvider } from './ReactNativeFileProvider'; -import { type StreamWriter } from './StreamWriter'; +import { FileWritableStream, type StreamWriter } from './StreamWriter'; export class ReactNativeFileSystem implements FileSystem { private readonly _fileSystemProvider: ReactNativeFileProvider = NativeModules.BacktraceFileSystemProvider; private readonly _directoryProvider: ReactNativeDirectoryProvider = NativeModules.BacktraceDirectoryProvider; @@ -89,4 +89,8 @@ export class ReactNativeFileSystem implements FileSystem { public createAttachment(path: string, name?: string | undefined): BacktraceAttachment { return new BacktraceFileAttachment(this, path, name); } + + public createWriteStream(path: string): FileWritableStream { + return new FileWritableStream(path, this.streamWriter); + } } diff --git a/packages/react-native/src/storage/StreamWriter.ts b/packages/react-native/src/storage/StreamWriter.ts index c618f2e0..1e36bfdb 100644 --- a/packages/react-native/src/storage/StreamWriter.ts +++ b/packages/react-native/src/storage/StreamWriter.ts @@ -1,9 +1,11 @@ +import { WritableStream } from 'web-streams-polyfill'; + export interface StreamWriter { /** * Creates a new stream writer. Returns a key to stream writer. * @param source path to the file */ - create(source: string): string; + create(source: string): string | undefined; /** * Appends a string to a file using a stream writer pointed by the key * @param key stream writer key @@ -17,3 +19,56 @@ export interface StreamWriter { */ close(key: string): boolean; } + +export class FileWritableStream extends WritableStream { + constructor( + public readonly path: string, + streamWriter: StreamWriter, + ) { + super(new NativeUnderlyingSink(path, streamWriter)); + } +} + +export class NativeUnderlyingSink implements UnderlyingSink { + private _streamId?: string; + + constructor( + public readonly path: string, + private readonly _streamWriter: StreamWriter, + ) {} + + public async start() { + this._streamId = this._streamWriter.create(this.path); + if (!this._streamId) { + throw new Error(`Failed to open file ${this.path}.`); + } + } + + public close() { + if (!this._streamId) { + return; + } + + if (!this._streamWriter.close(this._streamId)) { + throw new Error(`Failed to close file ${this.path}.`); + } + } + + public async write(chunk: string) { + if (!this._streamId) { + throw new Error('File is not open.'); + } + + if (!(await this._streamWriter.append(this._streamId, chunk))) { + throw new Error(`Failed to write data to file ${this.path}.`); + } + } + + public abort() { + if (!this._streamId) { + return; + } + + this._streamWriter.close(this._streamId); + } +} diff --git a/packages/react-native/src/storage/lineChunkSplitter.ts b/packages/react-native/src/storage/lineChunkSplitter.ts new file mode 100644 index 00000000..43a627e7 --- /dev/null +++ b/packages/react-native/src/storage/lineChunkSplitter.ts @@ -0,0 +1,37 @@ +import type { ChunkSplitter } from './Chunkifier'; + +/** + * Splits data into chunks with maximum lines. + * @param maxLines Maximum lines in one chunk. + */ +export function lineChunkSplitter(maxLines: number): ChunkSplitter { + let seen = 0; + + function findNthLine(data: string, remaining: number): [number, number] { + let lastIndex = -1; + let count = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + lastIndex = data.indexOf('\n', lastIndex + 1); + if (lastIndex === -1) { + return [-1, count]; + } + + if (remaining === ++count) { + return [lastIndex + 1, count]; + } + } + } + + return function lineChunkSplitter(data) { + const remainingLines = maxLines - seen; + const [index, count] = findNthLine(data, remainingLines); + if (index === -1) { + seen += count; + return [data]; + } + + seen = 0; + return [data.substring(0, index), data.substring(index)]; + }; +} diff --git a/packages/react-native/tests/_helpers/blackholeChunkSink.ts b/packages/react-native/tests/_helpers/blackholeChunkSink.ts new file mode 100644 index 00000000..3cd17a51 --- /dev/null +++ b/packages/react-native/tests/_helpers/blackholeChunkSink.ts @@ -0,0 +1,12 @@ +import { WritableStream } from 'web-streams-polyfill'; +import { ChunkSink } from '../../src/storage/Chunkifier'; + +export function blackholeChunkSink(): ChunkSink { + return () => { + return new WritableStream({ + write() { + // Do nothing + }, + }); + }; +} diff --git a/packages/react-native/tests/_helpers/chunks.ts b/packages/react-native/tests/_helpers/chunks.ts new file mode 100644 index 00000000..d10e46bc --- /dev/null +++ b/packages/react-native/tests/_helpers/chunks.ts @@ -0,0 +1,49 @@ +import { ReadableStream } from 'web-streams-polyfill'; +import { Chunk, ChunkSplitter } from '../../src/storage/Chunkifier'; + +/** + * Trims array from right until `predicate` returns `false`. + * @returns Trimmed array. + */ +function trimRightIf(t: T[], predicate: (t: T) => boolean) { + for (let i = t.length - 1; i >= 0; i--) { + if (predicate(t[i])) { + continue; + } + + return t.slice(0, i + 1); + } + + return []; +} + +export async function splitToEnd(readable: ReadableStream, splitter: ChunkSplitter) { + const results: W[][] = [[]]; + + for await (let chunk of readable) { + while (chunk) { + const [c1, c2] = splitter(chunk); + results[results.length - 1].push(c1); + if (c2 !== undefined) { + chunk = c2; + results.push([]); + } else { + break; + } + } + } + + // Remove all trailing empty arrays + return trimRightIf( + results.map((b) => b.join('')), + (t) => !t.length, + ); +} + +export function* chunkify(chunk: string, length: number) { + let i = 0; + do { + yield chunk.substring(i * length, (i + 1) * length); + i++; + } while (i * length < chunk.length); +} diff --git a/packages/react-native/tests/_helpers/generators.ts b/packages/react-native/tests/_helpers/generators.ts new file mode 100644 index 00000000..b1a95736 --- /dev/null +++ b/packages/react-native/tests/_helpers/generators.ts @@ -0,0 +1,83 @@ +import readline from 'readline'; +import { Readable } from 'stream'; +import { ReadableStream } from 'web-streams-polyfill'; +import { createRng } from './random'; + +const rng = createRng(); + +export function dataStream(data: W) { + return new ReadableStream({ + start(controller) { + controller.enqueue(data); + controller.close(); + }, + }); +} + +export function generatorStream(generator: Generator) { + return new ReadableStream({ + pull(controller) { + const { value, done } = generator.next(); + if (done) { + return controller.close(); + } + controller.enqueue(value); + }, + }); +} + +export function randomLines(count: number, minLineLength: number, maxLineLength: number) { + return [...new Array(count)].map(() => rng.string(rng.intBetween(minLineLength, maxLineLength))); +} + +export async function readLines(readable: ReadableStream, lines: number) { + const result: string[] = []; + const rl = readline.createInterface(Readable.from(readable)); + + let count = 0; + for await (const line of rl) { + result.push(line); + if (++count === lines) { + break; + } + } + + rl.close(); + return result; +} + +export function randomString(count: number) { + let generated = 0; + + return new ReadableStream({ + pull(controller) { + const remaining = count - generated; + if (remaining <= 0) { + return controller.close(); + } + const chunk = rng.string(Math.min(remaining, 16384)); + generated += chunk.length; + controller.enqueue(chunk); + }, + }); +} + +export async function readToEnd(readable: ReadableStream) { + const result: string[] = []; + const reader = readable.getReader(); + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (value === undefined) { + break; + } + + result.push(value); + if (done) { + break; + } + } + + return result.join(''); +} diff --git a/packages/react-native/tests/_helpers/memoryChunkSink.ts b/packages/react-native/tests/_helpers/memoryChunkSink.ts new file mode 100644 index 00000000..692bac85 --- /dev/null +++ b/packages/react-native/tests/_helpers/memoryChunkSink.ts @@ -0,0 +1,23 @@ +import { WritableStream } from 'web-streams-polyfill'; +import { ChunkSink } from '../../src/storage/Chunkifier'; + +export function memoryChunkSink() { + const results: string[][] = []; + + const sink: ChunkSink = () => { + const index = results.length; + results.push([]); + + return new WritableStream({ + write(chunk) { + results[index].push(chunk); + }, + }); + }; + + const getResults = () => { + return results.map((chunks) => chunks.join('')); + }; + + return { sink, getResults }; +} diff --git a/packages/react-native/tests/_helpers/random.ts b/packages/react-native/tests/_helpers/random.ts new file mode 100644 index 00000000..c30c71ed --- /dev/null +++ b/packages/react-native/tests/_helpers/random.ts @@ -0,0 +1,6 @@ +import randomSeed from 'random-seed'; + +const seed = process.env.TEST_SEED; +export function createRng() { + return randomSeed.create(seed); +} diff --git a/packages/react-native/tests/_mocks/fileSystem.ts b/packages/react-native/tests/_mocks/fileSystem.ts new file mode 100644 index 00000000..5a747f5d --- /dev/null +++ b/packages/react-native/tests/_mocks/fileSystem.ts @@ -0,0 +1,42 @@ +import { MockedFileSystem, mockFileSystem } from '@backtrace/sdk-core/tests/_mocks/fileSystem'; +import path from 'path'; +import { WritableStream } from 'web-streams-polyfill'; +import { FileSystem } from '../../src/storage/FileSystem'; + +export function mockStreamFileSystem(files?: Record): MockedFileSystem { + const fs = mockFileSystem(files); + + return { + ...fs, + + copy: jest.fn().mockImplementation((sourceFile, destinationFile) => { + fs.files[path.resolve(destinationFile)] = fs.files[path.resolve(sourceFile)]; + return Promise.resolve(true); + }), + + copySync: jest.fn().mockImplementation((sourceFile, destinationFile) => { + fs.files[path.resolve(destinationFile)] = fs.files[path.resolve(sourceFile)]; + return true; + }), + + applicationDirectory: jest.fn().mockImplementation(() => { + return '/'; + }), + + createWriteStream: jest.fn().mockImplementation((p: string) => { + const writable = new WritableStream({ + write(str) { + const fullPath = path.resolve(p); + if (!fs.files[fullPath]) { + fs.files[fullPath] = str; + } else { + fs.files[fullPath] += str; + } + }, + }); + + (writable as { path?: string }).path = p; + return writable; + }), + }; +} diff --git a/packages/react-native/tests/storage/chunkifier.spec.ts b/packages/react-native/tests/storage/chunkifier.spec.ts new file mode 100644 index 00000000..f5a1e413 --- /dev/null +++ b/packages/react-native/tests/storage/chunkifier.spec.ts @@ -0,0 +1,190 @@ +import { WritableStream } from 'web-streams-polyfill'; +import { Chunk, ChunkifierSink, ChunkSplitter } from '../../src/storage/Chunkifier'; +import { blackholeChunkSink } from '../_helpers/blackholeChunkSink'; +import { splitToEnd } from '../_helpers/chunks'; +import { dataStream, randomString, readToEnd } from '../_helpers/generators'; +import { memoryChunkSink } from '../_helpers/memoryChunkSink'; + +function charSplitter(char: string): ChunkSplitter { + return (chunk) => { + const index = chunk.indexOf(char); + if (index === -1) { + return [chunk]; + } + return [chunk.substring(0, index), chunk.substring(index + 1)]; + }; +} + +function noopSplitter(): ChunkSplitter { + return (c: W) => [c]; +} + +describe('ChunkifierSink', () => { + it('should call splitter function with every chunk', async () => { + const data = randomString(16384 * 10); + const splitter = jest.fn(noopSplitter()); + + const instance = new WritableStream( + new ChunkifierSink({ + sink: blackholeChunkSink(), + splitter: () => splitter, + }), + ); + + await data.pipeTo(instance); + expect(splitter).toHaveBeenCalledTimes(10); + }); + + it('should not call splitter function chunk if data is empty', async () => { + const data = dataStream(''); + const splitter = jest.fn(noopSplitter()); + + const instance = new WritableStream( + new ChunkifierSink({ + sink: blackholeChunkSink(), + splitter: () => splitter, + }), + ); + + await data.pipeTo(instance); + expect(splitter).not.toHaveBeenCalled(); + }); + + it('should call splitter factory with every new chunk', async () => { + const data = randomString(500); + + const splitCount = 10; + let split = 0; + const splitterFactory = jest.fn( + (): ChunkSplitter => (chunk) => { + if (split < splitCount) { + split++; + return [chunk.substring(0, 10), chunk.substring(10)]; + } + return [chunk]; + }, + ); + + const instance = new WritableStream( + new ChunkifierSink({ + sink: blackholeChunkSink(), + splitter: splitterFactory, + }), + ); + + await data.pipeTo(instance); + expect(splitterFactory).toHaveBeenCalledTimes(splitCount + 1); + }); + + it('should not call sink on ChunkifierSink creation', () => { + const splitter = noopSplitter(); + const sink = jest.fn(blackholeChunkSink()); + + new ChunkifierSink({ + sink, + splitter: () => splitter, + }); + + expect(sink).not.toHaveBeenCalled(); + }); + + it('should call sink on chunk split', async () => { + const data = dataStream('bAb'); + const splitter = charSplitter('A'); + const sink = jest.fn(blackholeChunkSink()); + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + }), + ); + + await data.pipeTo(instance); + expect(sink).toHaveBeenCalledTimes(2); + }); + + it('should split data into chunks', async () => { + const dataStr = await readToEnd(randomString(5000)); + const data = dataStream(dataStr); + const splitter = charSplitter(dataStr[3]); + const { sink, getResults } = memoryChunkSink(); + + const expected = await splitToEnd(dataStream(dataStr), splitter); + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: true, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); + + it('should split data into non-empty chunks', async () => { + const dataStr = await readToEnd(randomString(5000)); + const data = dataStream(dataStr); + const splitter = charSplitter(dataStr[3]); + const { sink, getResults } = memoryChunkSink(); + + const expected = (await splitToEnd(dataStream(dataStr), splitter)).filter((b) => b.length); + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: false, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); + + it('should create empty chunks if allowEmptyChunks is true', async () => { + const dataStr = 'abcaaaabcdaaa'; + const data = dataStream(dataStr); + const splitter = charSplitter('a'); + const { sink, getResults } = memoryChunkSink(); + + const expected = ['', 'bc', '', '', '', 'bcd', '', '']; + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: true, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); + + it('should not create empty chunks if allowEmptyChunks is false', async () => { + const dataStr = 'abcaaaabcdaaa'; + const data = dataStream(dataStr); + const splitter = charSplitter('a'); + const { sink, getResults } = memoryChunkSink(); + + const expected = ['bc', 'bcd']; + + const instance = new WritableStream( + new ChunkifierSink({ + sink, + splitter: () => splitter, + allowEmptyChunks: false, + }), + ); + + await data.pipeTo(instance); + const results = getResults(); + expect(results).toEqual(expected); + }); +}); diff --git a/packages/react-native/tests/storage/fileChunkSink.spec.ts b/packages/react-native/tests/storage/fileChunkSink.spec.ts new file mode 100644 index 00000000..8fd302cf --- /dev/null +++ b/packages/react-native/tests/storage/fileChunkSink.spec.ts @@ -0,0 +1,57 @@ +import path from 'path'; +import { WritableStream } from 'web-streams-polyfill'; +import { FileChunkSink } from '../../src/storage/FileChunkSink'; +import { mockStreamFileSystem } from '../_mocks/fileSystem'; + +async function writeAndClose(stream: WritableStream, value: string) { + const writer = stream.getWriter(); + await writer.write(value); + writer.releaseLock(); +} + +function sortString(a: string, b: string) { + return a.localeCompare(b); +} + +describe('fileChunkSink', () => { + it('should create a filestream with name from filename', async () => { + const fs = mockStreamFileSystem(); + const filename = 'abc'; + const sink = new FileChunkSink({ file: () => filename, maxFiles: Infinity, fs }); + + const stream = sink.getSink()(0); + expect(stream.path).toEqual(filename); + }); + + it('should create a filestream each time it is called', async () => { + const fs = mockStreamFileSystem(); + const dir = 'test'; + const sink = new FileChunkSink({ file: (n) => path.join(dir, n.toString()), maxFiles: Infinity, fs }); + const expected = [0, 2, 5]; + + for (const n of expected) { + const stream = sink.getSink()(n); + await writeAndClose(stream, 'a'); + } + + const actual = await fs.readDir(dir); + expect(actual.sort(sortString)).toEqual(expected.map((e) => e.toString()).sort(sortString)); + }); + + it('should remove previous files if count exceeds maxFiles', async () => { + const fs = mockStreamFileSystem(); + const dir = 'test'; + const maxFiles = 3; + const sink = new FileChunkSink({ file: (n) => path.join(dir, n.toString()), maxFiles, fs }); + const files = [0, 2, 5, 6, 79, 81, 38, -1, 3]; + const expected = files.slice(-maxFiles); + + for (const n of files) { + const stream = sink.getSink()(n); + await writeAndClose(stream, 'a'); + } + + const actual = await fs.readDir(dir); + expect(actual.sort(sortString)).toEqual(expected.map((e) => e.toString()).sort(sortString)); + }); +}); diff --git a/packages/react-native/tests/storage/lineChunkSplitter.spec.ts b/packages/react-native/tests/storage/lineChunkSplitter.spec.ts new file mode 100644 index 00000000..7939c03a --- /dev/null +++ b/packages/react-native/tests/storage/lineChunkSplitter.spec.ts @@ -0,0 +1,79 @@ +import { lineChunkSplitter } from '../../src/storage/lineChunkSplitter'; +import { chunkify, splitToEnd } from '../_helpers/chunks'; +import { generatorStream, randomLines } from '../_helpers/generators'; + +function countNewlines(buffer: string) { + return [...buffer.matchAll(/\n/g)].length; +} + +async function getData(lines: number) { + return randomLines(lines, 10, 20).join('\n') + '\n'; +} + +describe('lineChunkSplitter', () => { + it('should split chunk if it has more lines than maxLines', async () => { + const maxLines = 10; + const chunk = await getData(30); + const splitter = lineChunkSplitter(maxLines); + + const [c1, c2] = splitter(chunk); + expect(countNewlines(c1)).toEqual(maxLines); // include trailing newline + expect(c2 && countNewlines(c2)).toEqual(30 - maxLines); + }); + + it('should split chunk if total seen lines is more than maxLines', async () => { + const maxLines = 100; + const chunk = await getData(30); + const splitter = lineChunkSplitter(maxLines); + + splitter(chunk); + splitter(chunk); + splitter(chunk); + const [c1, c2] = splitter(chunk); + + expect(countNewlines(c1)).toEqual(100 - 30 * 3); + expect(c2 && countNewlines(c2)).toEqual(20); + }); + + it('should not split chunk if it has less lines than maxLines', async () => { + const maxLines = 100; + const chunk = await getData(30); + const splitter = lineChunkSplitter(maxLines); + const [c1, c2] = splitter(chunk); + + expect(countNewlines(c1)).toEqual(30); + expect(c2).toBeUndefined(); + }); + + it('should not split chunk if it has maxLines lines', async () => { + const maxLines = 100; + const chunk = await getData(maxLines); + const splitter = lineChunkSplitter(maxLines); + const [c1, c2] = splitter(chunk); + + expect(countNewlines(c1)).toEqual(maxLines); + expect(c2?.length).toEqual(0); + }); + + it('should split chunk by lines', async () => { + const maxLines = 123; + const data = await getData(1000); + const splitter = lineChunkSplitter(maxLines); + const actual = await splitToEnd(generatorStream(chunkify(data, 100)), splitter); + + let seen = 0; + for (let i = 0; i < actual.length; i++) { + const chunk = actual[i]; + const start = seen; + const end = seen + chunk.length; + expect(chunk).toEqual(data.substring(start, end)); + seen += chunk.length; + + if (i === actual.length - 1) { + expect(countNewlines(chunk)).toBeLessThanOrEqual(maxLines); + } else { + expect(countNewlines(chunk)).toEqual(maxLines); + } + } + }); +});