react-native: replace AlternatingFileWriter with WritableStream and ChunkifierSink for breadcrumbs

packages/react-native/jest.config.js

@@ -3,4 +3,5 @@ module.exports = {
     preset: 'react-native',
     testEnvironment: 'node',
     setupFiles: ['./jest.setup.js'],
+    globalSetup: './jest.rng.mjs',
 };

packages/react-native/jest.rng.mjs

@@ -0,0 +1,13 @@
+import crypto from 'crypto';
+
+function getRandomSeed() {
+    return crypto.randomBytes(16).toString('hex');
+}
+
+export default function () {
+    if (!process.env.TEST_SEED) {
+        process.env.TEST_SEED = getRandomSeed();
+    }
+
+    console.log(`\n=== Using random seed ${process.env.TEST_SEED} ===`);
+}

packages/react-native/package.json

@@ -28,6 +28,7 @@
     ],
     "scripts": {
         "build": "bob build",
+        "watch": "tsc --noEmit --watch",
         "clean": "rimraf \"lib\"",
         "format:check": "eslint \"**/*.{js,ts,tsx}\"",
         "prepublishOnly": "npm run clean && cross-env NODE_ENV=production bob build",
@@ -82,6 +83,7 @@
         "jest": "^29.7.0",
         "pod-install": "^0.1.0",
         "prettier": "^2.0.5",
+        "random-seed": "^0.3.0",
         "react": "18.2.0",
         "react-native": "^0.72.4",
         "react-native-builder-bob": "^0.21.3",
@@ -90,6 +92,7 @@
         "typescript": "^5.0.2"
     },
     "dependencies": {
-        "@backtrace/sdk-core": "^0.6.0"
+        "@backtrace/sdk-core": "^0.6.0",
+        "web-streams-polyfill": "^4.0.0"
     }
 }

packages/react-native/src/breadcrumbs/FileBreadcrumbsStorage.ts

@@ -5,13 +5,17 @@ import {
     SessionFiles,
     TimeHelper,
     type BacktraceAttachment,
+    type BacktraceAttachmentProvider,
     type Breadcrumb,
     type BreadcrumbsStorage,
     type RawBreadcrumb,
 } from '@backtrace/sdk-core';
+import { WritableStream } from 'web-streams-polyfill';
 import { BacktraceFileAttachment } from '..';
 import { type FileSystem } from '../storage';
-import { AlternatingFileWriter } from './AlternatingFileWriter';
+import { ChunkifierSink } from '../storage/Chunkifier';
+import { FileChunkSink } from '../storage/FileChunkSink';
+import { lineChunkSplitter } from '../storage/lineChunkSplitter';
 
 const FILE_PREFIX = 'bt-breadcrumbs';
 
@@ -21,32 +25,48 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage {
     }
 
     private _lastBreadcrumbId: number = TimeHelper.toTimestampInSec(TimeHelper.now());
-    private readonly _writer: AlternatingFileWriter;
+    private readonly _destinationStream: WritableStream;
+    private readonly _destinationWriter: WritableStreamDefaultWriter;
+    private readonly _sink: FileChunkSink;
 
     constructor(
+        session: SessionFiles,
         private readonly _fileSystem: FileSystem,
-        private readonly _mainFile: string,
-        private readonly _fallbackFile: string,
         maximumBreadcrumbs: number,
     ) {
-        this._writer = new AlternatingFileWriter(
-            _mainFile,
-            _fallbackFile,
-            Math.floor(maximumBreadcrumbs / 2),
-            _fileSystem,
+        this._sink = new FileChunkSink({
+            maxFiles: 2,
+            fs: this._fileSystem,
+            file: (n) => session.getFileName(FileBreadcrumbsStorage.getFileName(n)),
+        });
+
+        this._destinationStream = new WritableStream(
+            new ChunkifierSink({
+                sink: this._sink.getSink(),
+                splitter: () => lineChunkSplitter(Math.ceil(maximumBreadcrumbs / 2)),
+            }),
         );
+
+        this._destinationWriter = this._destinationStream.getWriter();
     }
 
     public static create(fileSystem: FileSystem, session: SessionFiles, maximumBreadcrumbs: number) {
-        const file1 = session.getFileName(this.getFileName(0));
-        const file2 = session.getFileName(this.getFileName(1));
-        return new FileBreadcrumbsStorage(fileSystem, file1, file2, maximumBreadcrumbs);
+        return new FileBreadcrumbsStorage(session, fileSystem, maximumBreadcrumbs);
     }
 
     public getAttachments(): BacktraceAttachment<unknown>[] {
+        const files = [...this._sink.files].map((f) => f.path);
+        return files.map(
+            (f, i) => new BacktraceFileAttachment(this._fileSystem, f, `bt-breadcrumbs-${i}`, 'application/json'),
+        );
+    }
+
+    public getAttachmentProviders(): BacktraceAttachmentProvider[] {
         return [
-            new BacktraceFileAttachment(this._fileSystem, this._mainFile, 'bt-breadcrumbs-0'),
-            new BacktraceFileAttachment(this._fileSystem, this._fallbackFile, 'bt-breadcrumbs-1'),
+            {
+                get: () => this.getAttachments(),
+                type: 'dynamic',
+            },
         ];
     }
 
@@ -73,7 +93,9 @@ export class FileBreadcrumbsStorage implements BreadcrumbsStorage {
         };
 
         const breadcrumbJson = JSON.stringify(breadcrumb, jsonEscaper());
-        this._writer.writeLine(breadcrumbJson);
+        this._destinationWriter.write(breadcrumbJson + '\n').catch(() => {
+            // Fail silently here, there's not much we can do about this
+        });
 
         return id;
     }

packages/react-native/src/storage/Chunkifier.ts

@@ -0,0 +1,125 @@
+export type ChunkSplitterFactory<W extends Chunk> = () => ChunkSplitter<W>;
+
+/**
+ * Implementation of splitter should return either one or two `Buffer`s.
+ *
+ * The first `Buffer` will be written to the current chunk.
+ * If the second `Buffer` is returned, `chunkifier` will create a new chunk and write the
+ * second buffer to the new chunk.
+ */
+export type ChunkSplitter<W extends Chunk> = (chunk: W) => [W, W?];
+
+/**
+ * Implementation of chunk sink should return each time a new writable stream.
+ *
+ * `n` determines which stream it is in sequence.
+ */
+export type ChunkSink<W extends Chunk, S extends WritableStream<W> = WritableStream<W>> = (n: number) => S;
+
+export type Chunk = { readonly length: number };
+
+export interface ChunkifierOptions<W extends Chunk> {
+    /**
+     * Chunk splitter factory. The factory will be called when creating a new chunk.
+     */
+    readonly splitter: ChunkSplitterFactory<W>;
+
+    /**
+     * Chunk sink. The sink will be called when creating a new chunk.
+     */
+    readonly sink: ChunkSink<W>;
+
+    readonly allowEmptyChunks?: boolean;
+}
+
+interface StreamContext<W extends Chunk> {
+    readonly stream: WritableStream<W>;
+    readonly streamWriter: WritableStreamDefaultWriter<W>;
+    isEmptyChunk: boolean;
+}
+
+export class ChunkifierSink<W extends Chunk> implements UnderlyingSink<W> {
+    private _context?: StreamContext<W>;
+    private _splitter?: ChunkSplitter<W>;
+    private _chunkCount = 0;
+
+    constructor(private readonly _options: ChunkifierOptions<W>) {}
+
+    public async write(data: W): Promise<void> {
+        // If data is empty from the start, forward the write directly to current stream
+        if (this.isEmpty(data)) {
+            return await this.ensureStreamContext().streamWriter.write(data);
+        }
+
+        while (data) {
+            if (this.isEmpty(data)) {
+                break;
+            }
+
+            const splitter = this.ensureSplitter();
+            const [currentChunk, nextChunk] = splitter(data);
+            if (nextChunk === undefined) {
+                const current = this.ensureStreamContext();
+                if (!this.isEmpty(currentChunk)) {
+                    current.isEmptyChunk = false;
+                }
+
+                return await current.streamWriter.write(currentChunk);
+            }
+
+            data = nextChunk;
+            if (
+                this._context
+                    ? this._context.isEmptyChunk
+                    : this.isEmpty(currentChunk) && !this._options.allowEmptyChunks
+            ) {
+                continue;
+            }
+
+            const current = this.ensureStreamContext();
+            await current.streamWriter.write(currentChunk);
+            current.streamWriter.releaseLock();
+
+            // On next loop iteration, or write, create new stream again
+            this.reset();
+        }
+    }
+
+    public async close() {
+        return await this._context?.streamWriter.close();
+    }
+
+    private ensureStreamContext() {
+        if (!this._context) {
+            return (this._context = this.createStreamContext());
+        }
+        return this._context;
+    }
+
+    private ensureSplitter() {
+        if (!this._splitter) {
+            return (this._splitter = this._options.splitter());
+        }
+        return this._splitter;
+    }
+
+    private createStreamContext(): StreamContext<W> {
+        const stream = this._options.sink(this._chunkCount++);
+        const writer = stream.getWriter();
+        return { stream, streamWriter: writer, isEmptyChunk: true };
+    }
+
+    /**
+     * Resets the chunkifier to it's initial state. Use when switching streams.
+     */
+    private reset() {
+        this._context = undefined;
+
+        // Splitter may have an internal state which we need to recreate with new stream
+        this._splitter = undefined;
+    }
+
+    private isEmpty(chunk: W) {
+        return !chunk.length;
+    }
+}