Skip to content

Commit 165f48d

Browse files
james-prejames-pre
committed
Added growBuffer
Changed from `ArrayBuffer.slice` to `ArrayBuffer.subarray` (performance) Fixed `LazyFile.write` and `.writeSync` using an incorrect position
1 parent 6aa29f5 commit 165f48d

File tree

7 files changed

+77
-54
lines changed

7 files changed

+77
-54
lines changed

‎eslint.shared.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ export default [
4545
'@typescript-eslint/no-unsafe-call': 'off',
4646
'@typescript-eslint/restrict-plus-operands': 'off',
4747
'@typescript-eslint/no-base-to-string': 'off',
48+
'@typescript-eslint/no-unused-expressions': 'warn',
4849
},
4950
},
5051
{

‎src/backends/store/fs.ts

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import { randomInt, serialize } from 'utilium';
22
import { Errno, ErrnoError } from '../../error.js';
33
import type { File } from '../../file.js';
4-
import { PreloadFile } from '../../file.js';
4+
import { LazyFile, PreloadFile } from '../../file.js';
55
import type { CreationOptions, FileSystemMetadata, PureCreationOptions } from '../../filesystem.js';
66
import { FileSystem } from '../../filesystem.js';
77
import type { FileType, Stats } from '../../stats.js';
8-
import { _throw, canary, decodeDirListing, encodeDirListing, encodeUTF8 } from '../../utils.js';
8+
import { _throw, canary, decodeDirListing, encodeDirListing, encodeUTF8, growBuffer } from '../../utils.js';
99
import { S_IFDIR, S_IFREG, S_ISGID, S_ISUID, size_max } from '../../vfs/constants.js';
1010
import { basename, dirname, join, parse, resolve } from '../../vfs/path.js';
1111
import { Index } from './file_index.js';
@@ -267,28 +267,28 @@ export class StoreFS<T extends Store = Store> extends FileSystem {
267267

268268
public async createFile(path: string, flag: string, mode: number, options: CreationOptions): Promise<File> {
269269
const node = await this.commitNew(path, S_IFREG, { mode, ...options }, new Uint8Array(), 'createFile');
270-
return new PreloadFile(this, path, flag, node.toStats(), new Uint8Array());
270+
return new LazyFile(this, path, flag, node.toStats());
271271
}
272272

273273
public createFileSync(path: string, flag: string, mode: number, options: CreationOptions): File {
274274
const node = this.commitNewSync(path, S_IFREG, { mode, ...options }, new Uint8Array(), 'createFile');
275-
return new PreloadFile(this, path, flag, node.toStats(), new Uint8Array());
275+
return new LazyFile(this, path, flag, node.toStats());
276276
}
277277

278278
public async openFile(path: string, flag: string): Promise<File> {
279279
await using tx = this.store.transaction();
280280
const node = await this.findInode(tx, path, 'openFile');
281-
const data = (await tx.get(node.data)) ?? _throw(ErrnoError.With('ENODATA', path, 'openFile'));
281+
//const data = (await tx.get(node.data)) ?? _throw(ErrnoError.With('ENODATA', path, 'openFile'));
282282

283-
return new PreloadFile(this, path, flag, node.toStats(), data);
283+
return new LazyFile(this, path, flag, node.toStats());
284284
}
285285

286286
public openFileSync(path: string, flag: string): File {
287287
using tx = this.store.transaction();
288288
const node = this.findInodeSync(tx, path, 'openFile');
289-
const data = tx.getSync(node.data) ?? _throw(ErrnoError.With('ENOENT', path, 'openFile'));
289+
//const data = tx.getSync(node.data) ?? _throw(ErrnoError.With('ENODATA', path, 'openFile'));
290290

291-
return new PreloadFile(this, path, flag, node.toStats(), data);
291+
return new LazyFile(this, path, flag, node.toStats());
292292
}
293293

294294
public async unlink(path: string): Promise<void> {
@@ -420,22 +420,27 @@ export class StoreFS<T extends Store = Store> extends FileSystem {
420420

421421
const inode = await this.findInode(tx, path, 'write');
422422

423-
const buffer = await tx.get(inode.data);
423+
const buffer = growBuffer(await tx.get(inode.data), offset + data.byteLength);
424424
buffer.set(data, offset);
425425

426-
await this.sync(path, buffer, inode);
426+
inode.update({ mtimeMs: Date.now(), size: buffer.byteLength });
427+
428+
await tx.set(inode.ino, serialize(inode));
429+
await tx.set(inode.data, buffer);
430+
431+
await tx.commit();
427432
}
428433

429434
public writeSync(path: string, data: Uint8Array, offset: number): void {
430435
using tx = this.store.transaction();
431436

432437
const inode = this.findInodeSync(tx, path, 'write');
433438

434-
inode.update({ mtimeMs: Date.now() });
435-
436-
const buffer = tx.getSync(inode.data);
439+
const buffer = growBuffer(tx.getSync(inode.data), offset + data.byteLength);
437440
buffer.set(data, offset);
438441

442+
inode.update({ mtimeMs: Date.now(), size: buffer.byteLength });
443+
439444
tx.setSync(inode.ino, serialize(inode));
440445
tx.setSync(inode.data, buffer);
441446

‎src/file.ts

Lines changed: 10 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { Errno, ErrnoError } from './error.js';
22
import type { FileSystem } from './filesystem.js';
33
import './polyfills.js';
44
import { _chown, Stats, type StatsLike } from './stats.js';
5+
import { growBuffer } from './utils.js';
56
import { config } from './vfs/config.js';
67
import * as c from './vfs/constants.js';
78

@@ -377,7 +378,7 @@ export class PreloadFile<FS extends FileSystem> extends File<FS> {
377378
}
378379
this.stats.size = length;
379380
// Truncate.
380-
this._buffer = length ? this._buffer.slice(0, length) : new Uint8Array();
381+
this._buffer = length ? this._buffer.subarray(0, length) : new Uint8Array();
381382
}
382383

383384
public async truncate(length: number): Promise<void> {
@@ -399,26 +400,10 @@ export class PreloadFile<FS extends FileSystem> extends File<FS> {
399400

400401
this.dirty = true;
401402
const end = position + length;
402-
const slice = buffer.slice(offset, offset + length);
403+
const slice = buffer.subarray(offset, offset + length);
403404

404-
if (end > this.stats.size) {
405-
this.stats.size = end;
406-
if (end > this._buffer.byteLength) {
407-
const { buffer } = this._buffer;
408-
if ('resizable' in buffer && buffer.resizable && buffer.maxByteLength <= end) {
409-
buffer.resize(end);
410-
} else if ('growable' in buffer && buffer.growable && buffer.maxByteLength <= end) {
411-
buffer.grow(end);
412-
} else if (config.unsafeBufferReplace) {
413-
this._buffer = slice;
414-
} else {
415-
// Extend the buffer!
416-
const newBuffer = new Uint8Array(new ArrayBuffer(end, this.fs.metadata().noResizableBuffers ? {} : { maxByteLength }));
417-
newBuffer.set(this._buffer);
418-
this._buffer = newBuffer;
419-
}
420-
}
421-
}
405+
this._buffer = growBuffer(this._buffer, end);
406+
if (end > this.stats.size) this.stats.size = end;
422407

423408
this._buffer.set(slice, position);
424409
this.stats.mtimeMs = Date.now();
@@ -479,7 +464,7 @@ export class PreloadFile<FS extends FileSystem> extends File<FS> {
479464
// No copy/read. Return immediately for better performance
480465
return bytesRead;
481466
}
482-
const slice = this._buffer.slice(position, end);
467+
const slice = this._buffer.subarray(position, end);
483468
new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength).set(slice, offset);
484469
return bytesRead;
485470
}
@@ -717,11 +702,9 @@ export class LazyFile<FS extends FileSystem> extends File<FS> {
717702

718703
this.dirty = true;
719704
const end = position + length;
720-
const slice = buffer.slice(offset, offset + length);
705+
const slice = buffer.subarray(offset, offset + length);
721706

722-
if (end > this.stats.size) {
723-
this.stats.size = end;
724-
}
707+
if (end > this.stats.size) this.stats.size = end;
725708

726709
this.stats.mtimeMs = Date.now();
727710
this._position = position + slice.byteLength;
@@ -738,7 +721,7 @@ export class LazyFile<FS extends FileSystem> extends File<FS> {
738721
*/
739722
public async write(buffer: Uint8Array, offset: number = 0, length: number = buffer.byteLength - offset, position: number = this.position): Promise<number> {
740723
const slice = this.prepareWrite(buffer, offset, length, position);
741-
await this.fs.write(this.path, slice, offset);
724+
await this.fs.write(this.path, slice, position);
742725
if (config.syncImmediately) await this.sync();
743726
return slice.byteLength;
744727
}
@@ -754,7 +737,7 @@ export class LazyFile<FS extends FileSystem> extends File<FS> {
754737
*/
755738
public writeSync(buffer: Uint8Array, offset: number = 0, length: number = buffer.byteLength - offset, position: number = this.position): number {
756739
const slice = this.prepareWrite(buffer, offset, length, position);
757-
this.fs.writeSync(this.path, slice, offset);
740+
this.fs.writeSync(this.path, slice, position);
758741
if (config.syncImmediately) this.syncSync();
759742
return slice.byteLength;
760743
}

‎src/utils.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,3 +198,42 @@ export function canary(path?: string, syscall?: string) {
198198
export function _throw(e: unknown): never {
199199
throw e;
200200
}
201+
202+
interface ArrayBufferViewConstructor {
203+
readonly prototype: ArrayBufferView<ArrayBufferLike>;
204+
new (length: number): ArrayBufferView<ArrayBuffer>;
205+
new (array: ArrayLike<number>): ArrayBufferView<ArrayBuffer>;
206+
new <TArrayBuffer extends ArrayBufferLike = ArrayBuffer>(buffer: TArrayBuffer, byteOffset?: number, length?: number): ArrayBufferView<TArrayBuffer>;
207+
new (array: ArrayLike<number> | ArrayBuffer): ArrayBufferView<ArrayBuffer>;
208+
}
209+
210+
/**
211+
* Grows a buffer if it isn't large enough
212+
* @returns The original buffer if resized successfully, or a newly created buffer
213+
* @internal Not for external use!
214+
*/
215+
export function growBuffer<T extends ArrayBufferLike | ArrayBufferView>(buffer: T, newByteLength: number): T {
216+
if (buffer.byteLength >= newByteLength) return buffer;
217+
218+
if (ArrayBuffer.isView(buffer)) {
219+
const newBuffer = growBuffer(buffer.buffer, newByteLength);
220+
return new (buffer.constructor as ArrayBufferViewConstructor)(newBuffer, buffer.byteOffset, newByteLength) as T;
221+
}
222+
223+
const isShared = buffer instanceof SharedArrayBuffer;
224+
225+
// Note: If true, the buffer must be resizable/growable because of the first check.
226+
if (buffer.maxByteLength > newByteLength) {
227+
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
228+
isShared ? buffer.grow(newByteLength) : buffer.resize(newByteLength);
229+
return buffer;
230+
}
231+
232+
if (!isShared) {
233+
return buffer.transfer(newByteLength) as T;
234+
}
235+
236+
const newBuffer = new SharedArrayBuffer(newByteLength) as T & SharedArrayBuffer;
237+
new Uint8Array(newBuffer).set(new Uint8Array(buffer));
238+
return newBuffer;
239+
}

‎src/vfs/async.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -696,7 +696,7 @@ export function createReadStream(this: V_Context, path: fs.PathLike, options?: B
696696
try {
697697
handle ||= await promises.open.call(context, path, 'r', options?.mode);
698698
const result = await handle.read(new Uint8Array(size), 0, size, handle.file.position);
699-
stream.push(!result.bytesRead ? null : result.buffer.slice(0, result.bytesRead));
699+
stream.push(!result.bytesRead ? null : result.buffer.subarray(0, result.bytesRead));
700700
handle.file.position += result.bytesRead;
701701
if (!result.bytesRead) {
702702
await handle.close();

‎src/vfs/promises.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ export class FileHandle implements promises.FileHandle {
212212
controller.close();
213213
return;
214214
}
215-
controller.enqueue(result.buffer.slice(0, result.bytesRead));
215+
controller.enqueue(result.buffer.subarray(0, result.bytesRead));
216216
position += result.bytesRead;
217217
if (++i >= maxChunks) {
218218
throw new ErrnoError(Errno.EFBIG, 'Too many iterations on readable stream', this.file.path, 'FileHandle.readableWebStream');
@@ -371,7 +371,7 @@ export class FileHandle implements promises.FileHandle {
371371
read: async (size: number) => {
372372
try {
373373
const result = await this.read(new Uint8Array(size), 0, size, this.file.position);
374-
stream.push(!result.bytesRead ? null : result.buffer.slice(0, result.bytesRead)); // Push data or null for EOF
374+
stream.push(!result.bytesRead ? null : result.buffer.subarray(0, result.bytesRead)); // Push data or null for EOF
375375
this.file.position += result.bytesRead;
376376
} catch (error) {
377377
stream.destroy(error as Error);

‎tests/fs/write.test.ts

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import assert from 'node:assert';
22
import { suite, test } from 'node:test';
33
import { fs } from '../common.js';
4-
4+
const fn = 'write.txt';
55
suite('write', () => {
66
test('write file with specified content', async () => {
7-
const fn = 'write.txt';
87
const expected = 'ümlaut.';
98

109
const handle = await fs.promises.open(fn, 'w', 0o644);
@@ -20,26 +19,22 @@ suite('write', () => {
2019
});
2120

2221
test('write a buffer to a file', async () => {
23-
const filename = 'write.txt';
2422
const expected = Buffer.from('hello');
2523

26-
const handle = await fs.promises.open(filename, 'w', 0o644);
24+
const handle = await fs.promises.open(fn, 'w', 0o644);
2725

2826
const written = await handle.write(expected, 0, expected.length, null);
2927

3028
assert.strictEqual(expected.length, written.bytesWritten);
3129

3230
await handle.close();
3331

34-
assert((await fs.promises.readFile(filename)).equals(expected));
32+
assert((await fs.promises.readFile(fn)).equals(expected));
3533

36-
await fs.promises.unlink(filename);
34+
await fs.promises.unlink(fn);
3735
});
38-
});
3936

40-
suite('writeSync', () => {
41-
test('write file with specified content', () => {
42-
const fn = 'write.txt';
37+
test('writeSync file with specified content', () => {
4338
const fd = fs.openSync(fn, 'w');
4439

4540
let written = fs.writeSync(fd, '');
@@ -53,6 +48,6 @@ suite('writeSync', () => {
5348

5449
fs.closeSync(fd);
5550

56-
assert(fs.readFileSync(fn, 'utf8') === 'foobár');
51+
assert.strictEqual(fs.readFileSync(fn, 'utf8'), 'foobár');
5752
});
5853
});

0 commit comments

Comments
 (0)