Moved over LazyFile

`OverlayFS` no longer uses a mutex
Improved `FSRequest` types in `Port`
Fixed `PortFS.read` buffer transfer issues
Implemented `readSync` and `writeSync` on `Async`

Author: james-pre

src/backends/overlay.ts

@@ -5,9 +5,8 @@ import type { Backend } from './backend.js';
 import type { InodeLike } from './store/inode.js';
 
 import { Errno, ErrnoError } from '../error.js';
-import { PreloadFile, parseFlag } from '../file.js';
+import { LazyFile, parseFlag } from '../file.js';
 import { FileSystem } from '../filesystem.js';
-import { Mutexed } from '../mixins/mutexed.js';
 import { canary, decodeUTF8, encodeUTF8 } from '../utils.js';
 import { dirname, join } from '../vfs/path.js';
 
@@ -36,7 +35,7 @@ export interface OverlayOptions {
  *
  * @internal
  */
-export class UnmutexedOverlayFS extends FileSystem {
+export class OverlayFS extends FileSystem {
 	async ready(): Promise<void> {
 		await this.readable.ready();
 		await this.writable.ready();
@@ -203,23 +202,16 @@ export class UnmutexedOverlayFS extends FileSystem {
 		if (await this.writable.exists(path)) {
 			return this.writable.openFile(path, flag);
 		}
-		// Create an OverlayFile.
-		const file = await this.readable.openFile(path, parseFlag('r'));
-		const stats = await file.stat();
-		const { buffer } = await file.read(new Uint8Array(stats.size));
-		return new PreloadFile(this, path, flag, stats, buffer);
+		const stats = await this.readable.stat(path);
+		return new LazyFile(this, path, flag, stats);
 	}
 
 	public openFileSync(path: string, flag: string): File {
 		if (this.writable.existsSync(path)) {
 			return this.writable.openFileSync(path, flag);
 		}
-		// Create an OverlayFile.
-		const file = this.readable.openFileSync(path, parseFlag('r'));
-		const stats = file.statSync();
-		const data = new Uint8Array(stats.size);
-		file.readSync(data);
-		return new PreloadFile(this, path, flag, stats, data);
+		const stats = this.readable.statSync(path);
+		return new LazyFile(this, path, flag, stats);
 	}
 
 	public async createFile(path: string, flag: string, mode: number, options: CreationOptions): Promise<File> {
@@ -555,14 +547,6 @@ export class UnmutexedOverlayFS extends FileSystem {
 	}
 }
 
-/**
- * OverlayFS makes a read-only filesystem writable by storing writes on a second,
- * writable file system. Deletes are persisted via metadata stored on the writable
- * file system.
- * @internal
- */
-export class OverlayFS extends Mutexed(UnmutexedOverlayFS) {}
-
 const _Overlay = {
 	name: 'Overlay',
 	options: {

src/backends/port/fs.ts

@@ -1,27 +1,31 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { pick, type ExtractProperties } from 'utilium';
+import type { ExtractProperties } from 'utilium';
 import type { MountConfiguration } from '../../config.js';
 import type { CreationOptions, FileSystemMetadata } from '../../filesystem.js';
 import type { Backend, FilesystemOf } from '../backend.js';
+import type { Inode, InodeLike } from '../store/inode.js';
+import type { File } from '../../file.js';
 
-import type { TransferListItem } from 'node:worker_threads';
+import { pick } from 'utilium';
 import { resolveMountConfig } from '../../config.js';
 import { Errno, ErrnoError } from '../../error.js';
-import { File } from '../../file.js';
 import { FileSystem } from '../../filesystem.js';
 import { Async } from '../../mixins/async.js';
 import { Stats } from '../../stats.js';
+import { decodeUTF8 } from '../../utils.js';
 import { InMemory } from '../memory.js';
-import type { Inode, InodeLike } from '../store/inode.js';
 import * as RPC from './rpc.js';
 
 type FSMethods = ExtractProperties<FileSystem, (...args: any[]) => Promise<any> | FileSystemMetadata>;
 type FSMethod = keyof FSMethods;
-/** @internal */
-export interface FSRequest<TMethod extends FSMethod = FSMethod> extends RPC.Request {
-	method: TMethod;
-	args: Parameters<FSMethods[TMethod]>;
-}
+
+export type FSRequest<TMethod extends FSMethod = FSMethod> = RPC.Message &
+	{
+		[M in TMethod]: {
+			method: M;
+			args: Parameters<FSMethods[M]>;
+		};
+	}[TMethod];
 
 /**
  * PortFS lets you access an FS instance that is running in a port, or the other way around.
@@ -54,7 +58,7 @@ export class PortFS extends Async(FileSystem) {
 	}
 
 	protected rpc<const T extends FSMethod>(method: T, ...args: Parameters<FSMethods[T]>): Promise<Awaited<ReturnType<FSMethods[T]>>> {
-		return RPC.request<FSRequest<T>, Awaited<ReturnType<FSMethods[T]>>>({ method, args }, { ...this.options, fs: this });
+		return RPC.request<FSRequest<T>, Awaited<ReturnType<FSMethods[T]>>>({ method, args } as Omit<FSRequest<T>, 'id' | 'stack' | '_zenfs'>, { ...this.options, fs: this });
 	}
 
 	public async ready(): Promise<void> {
@@ -107,8 +111,9 @@ export class PortFS extends Async(FileSystem) {
 		return this.rpc('link', srcpath, dstpath);
 	}
 
-	public read(path: string, buffer: Uint8Array, offset: number, length: number): Promise<void> {
-		return this.rpc('read', path, buffer, offset, length);
+	public async read(path: string, buffer: Uint8Array, offset: number, length: number): Promise<void> {
+		const _buf = (await this.rpc('read', path, buffer, offset, length)) as unknown as Uint8Array;
+		buffer.set(_buf);
 	}
 
 	public write(path: string, buffer: Uint8Array, offset: number): Promise<void> {
@@ -125,31 +130,29 @@ export async function handleRequest(port: RPC.Port, fs: FileSystem & { _descript
 	let value,
 		error: boolean = false;
 
-	const transfer: TransferListItem[] = [];
-
 	try {
 		// @ts-expect-error 2556
 		value = await fs[method](...args);
-		if (value instanceof File) {
-			await using file = await fs.openFile(args[0] as string, 'r+');
-			const stats = await file.stat();
-			const data = new Uint8Array(stats.size);
-
-			await file.read(data);
-			value = {
-				path: value.path,
-				flag: args[1] as string,
-				stats,
-				buffer: data.buffer,
-			} satisfies RPC.FileData;
-			transfer.push(data.buffer);
+		switch (method) {
+			case 'openFile':
+			case 'createFile': {
+				value = {
+					path: args[0],
+					flag: args[1],
+					stats: await fs.stat(args[0]),
+				} satisfies RPC.FileData;
+				break;
+			}
+			case 'read':
+				value = args[1];
+				break;
 		}
 	} catch (e: any) {
 		value = e instanceof ErrnoError ? e.toJSON() : pick(e, 'message', 'stack');
 		error = true;
 	}
 
-	port.postMessage({ _zenfs: true, id, error, method, stack, value }, transfer);
+	port.postMessage({ _zenfs: true, id, error, method, stack, value });
 }
 
 export function attachFS(port: RPC.Port, fs: FileSystem): void {

src/backends/port/rpc.ts

@@ -7,7 +7,7 @@ import type { Backend, FilesystemOf } from '../backend.js';
 import type { PortFS } from './fs.js';
 
 import { Errno, ErrnoError } from '../../error.js';
-import { PreloadFile } from '../../file.js';
+import { LazyFile } from '../../file.js';
 import { Stats, type StatsLike } from '../../stats.js';
 import { handleRequest } from './fs.js';
 
@@ -57,13 +57,18 @@ interface _ResponseWithValue<T> extends Message {
 	value: Awaited<T> extends File ? FileData : Awaited<T>;
 }
 
-export type Response<T = unknown> = _ResponseWithError | _ResponseWithValue<T>;
+interface _ResponseRead extends Message {
+	error: false;
+	method: 'read';
+	value: Uint8Array;
+}
+
+export type Response<T = unknown> = _ResponseWithError | _ResponseWithValue<T> | _ResponseRead;
 
 export interface FileData {
 	path: string;
 	flag: string;
 	stats: StatsLike<number>;
-	buffer: ArrayBuffer;
 }
 
 function isFileData(value: unknown): value is FileData {
@@ -129,8 +134,8 @@ export function handleResponse<const TResponse extends Response>(response: TResp
 	}
 
 	if (isFileData(value)) {
-		const { path, flag, stats, buffer } = value;
-		const file = new PreloadFile(fs!, path, flag, new Stats(stats), new Uint8Array(buffer));
+		const { path, flag, stats } = value;
+		const file = new LazyFile(fs!, path, flag, new Stats(stats));
 		resolve(file);
 		executors.delete(id);
 		return;

src/backends/store/fs.ts

@@ -1,7 +1,7 @@
 import { randomInt, serialize } from 'utilium';
 import { Errno, ErrnoError } from '../../error.js';
 import type { File } from '../../file.js';
-import { LazyFile, PreloadFile } from '../../file.js';
+import { LazyFile } from '../../file.js';
 import type { CreationOptions, FileSystemMetadata, PureCreationOptions } from '../../filesystem.js';
 import { FileSystem } from '../../filesystem.js';
 import type { FileType, Stats } from '../../stats.js';

src/backends/store/index_fs.ts

@@ -1,6 +1,6 @@
 import { ErrnoError } from '../../error.js';
 import type { File } from '../../file.js';
-import { PreloadFile } from '../../file.js';
+import { LazyFile } from '../../file.js';
 import type { CreationOptions } from '../../filesystem.js';
 import { Stats } from '../../stats.js';
 import { S_IFREG } from '../../vfs/constants.js';
@@ -58,14 +58,14 @@ export abstract class IndexFS<T extends Store> extends StoreFS<T> {
 
 	public override async createFile(path: string, flag: string, mode: number, options: CreationOptions): Promise<File> {
 		const node = await this.commitNew(path, S_IFREG, { mode, ...options }, new Uint8Array(), 'createFile');
-		const file = new PreloadFile(this, path, flag, node.toStats(), new Uint8Array());
+		const file = new LazyFile(this, path, flag, node.toStats());
 		this.index.set(path, node);
 		return file;
 	}
 
 	public createFileSync(path: string, flag: string, mode: number, options: CreationOptions): File {
 		const node = this.commitNewSync(path, S_IFREG, { mode, ...options }, new Uint8Array(), 'createFile');
-		const file = new PreloadFile(this, path, flag, node.toStats(), new Uint8Array());
+		const file = new LazyFile(this, path, flag, node.toStats());
 		this.index.set(path, node);
 		return file;
 	}

src/mixins/async.ts

@@ -5,7 +5,7 @@ import type { _SyncFSKeys, AsyncFSMethods, Mixin } from './shared.js';
 
 import { StoreFS } from '../backends/store/fs.js';
 import { Errno, ErrnoError } from '../error.js';
-import { parseFlag, PreloadFile } from '../file.js';
+import { LazyFile, parseFlag } from '../file.js';
 import { join } from '../vfs/path.js';
 
 /** @internal */
@@ -36,7 +36,7 @@ export interface AsyncMixin extends Pick<FileSystem, Exclude<_SyncFSKeys, 'exist
  *
  */
 export function Async<const T extends typeof FileSystem>(FS: T): Mixin<T, AsyncMixin> {
-	abstract class AsyncFS extends FS {
+	abstract class AsyncFS extends FS implements AsyncMixin {
 		/**
 		 * Queue of pending asynchronous operations.
 		 */
@@ -116,20 +116,17 @@ export function Async<const T extends typeof FileSystem>(FS: T): Mixin<T, AsyncM
 			return this._sync.statSync(path);
 		}
 
-		public createFileSync(path: string, flag: string, mode: number, options: CreationOptions): PreloadFile<this> {
+		public createFileSync(path: string, flag: string, mode: number, options: CreationOptions): LazyFile<this> {
 			this.checkSync(path, 'createFile');
 			this._sync.createFileSync(path, flag, mode, options);
 			this.queue('createFile', path, flag, mode, options);
 			return this.openFileSync(path, flag);
 		}
 
-		public openFileSync(path: string, flag: string): PreloadFile<this> {
+		public openFileSync(path: string, flag: string): LazyFile<this> {
 			this.checkSync(path, 'openFile');
-			const file = this._sync.openFileSync(path, flag + '+');
-			const stats = file.statSync();
-			const buffer = new Uint8Array(stats.size);
-			file.readSync(buffer);
-			return new PreloadFile(this, path, flag, stats, buffer);
+			const stats = this._sync.statSync(path);
+			return new LazyFile(this, path, flag, stats);
 		}
 
 		public unlinkSync(path: string): void {
@@ -172,6 +169,17 @@ export function Async<const T extends typeof FileSystem>(FS: T): Mixin<T, AsyncM
 			return this._sync.existsSync(path);
 		}
 
+		public readSync(path: string, buffer: Uint8Array, offset: number, end: number): void {
+			this.checkSync(path, 'read');
+			this._sync.readSync(path, buffer, offset, end);
+		}
+
+		public writeSync(path: string, buffer: Uint8Array, offset: number): void {
+			this.checkSync(path, 'write');
+			this._sync.writeSync(path, buffer, offset);
+			this.queue('write', path, buffer, offset);
+		}
+
 		/**
 		 * @internal
 		 */

src/mixins/mutexed.ts

@@ -251,7 +251,7 @@ export class _MutexedFS<T extends FileSystem> implements FileSystem {
  * For example, on an OverlayFS instance with an async lower
  * directory operations like rename and rmdir may involve multiple
  * requests involving both the upper and lower file systems -- they
- * are not executed in a single atomic step. OverlayFS uses this
+ * are not executed in a single atomic step. OverlayFS used to use this
  * to avoid having to reason about the correctness of
  * multiple requests interleaving.
  *