From 6f1604efb2ba8cdb1f4c3299bc3e0e2f083470f5 Mon Sep 17 00:00:00 2001 From: James Prevett Date: Tue, 14 Jan 2025 20:20:31 -0600 Subject: [PATCH] Moved over `LazyFile` `OverlayFS` no longer uses a mutex Improved `FSRequest` types in `Port` Fixed `PortFS.read` buffer transfer issues Implemented `readSync` and `writeSync` on `Async` --- src/backends/overlay.ts | 28 ++++------------ src/backends/port/fs.ts | 59 ++++++++++++++++++---------------- src/backends/port/rpc.ts | 15 ++++++--- src/backends/store/fs.ts | 2 +- src/backends/store/index_fs.ts | 6 ++-- src/mixins/async.ts | 26 +++++++++------ src/mixins/mutexed.ts | 2 +- 7 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/backends/overlay.ts b/src/backends/overlay.ts index ecd78ba2..330dd10b 100644 --- a/src/backends/overlay.ts +++ b/src/backends/overlay.ts @@ -5,9 +5,8 @@ import type { Backend } from './backend.js'; import type { InodeLike } from './store/inode.js'; import { Errno, ErrnoError } from '../error.js'; -import { PreloadFile, parseFlag } from '../file.js'; +import { LazyFile, parseFlag } from '../file.js'; import { FileSystem } from '../filesystem.js'; -import { Mutexed } from '../mixins/mutexed.js'; import { canary, decodeUTF8, encodeUTF8 } from '../utils.js'; import { dirname, join } from '../vfs/path.js'; @@ -36,7 +35,7 @@ export interface OverlayOptions { * * @internal */ -export class UnmutexedOverlayFS extends FileSystem { +export class OverlayFS extends FileSystem { async ready(): Promise { await this.readable.ready(); await this.writable.ready(); @@ -203,23 +202,16 @@ export class UnmutexedOverlayFS extends FileSystem { if (await this.writable.exists(path)) { return this.writable.openFile(path, flag); } - // Create an OverlayFile. - const file = await this.readable.openFile(path, parseFlag('r')); - const stats = await file.stat(); - const { buffer } = await file.read(new Uint8Array(stats.size)); - return new PreloadFile(this, path, flag, stats, buffer); + const stats = await this.readable.stat(path); + return new LazyFile(this, path, flag, stats); } public openFileSync(path: string, flag: string): File { if (this.writable.existsSync(path)) { return this.writable.openFileSync(path, flag); } - // Create an OverlayFile. - const file = this.readable.openFileSync(path, parseFlag('r')); - const stats = file.statSync(); - const data = new Uint8Array(stats.size); - file.readSync(data); - return new PreloadFile(this, path, flag, stats, data); + const stats = this.readable.statSync(path); + return new LazyFile(this, path, flag, stats); } public async createFile(path: string, flag: string, mode: number, options: CreationOptions): Promise { @@ -555,14 +547,6 @@ export class UnmutexedOverlayFS extends FileSystem { } } -/** - * OverlayFS makes a read-only filesystem writable by storing writes on a second, - * writable file system. Deletes are persisted via metadata stored on the writable - * file system. - * @internal - */ -export class OverlayFS extends Mutexed(UnmutexedOverlayFS) {} - const _Overlay = { name: 'Overlay', options: { diff --git a/src/backends/port/fs.ts b/src/backends/port/fs.ts index 11cdd18f..52e8e3d0 100644 --- a/src/backends/port/fs.ts +++ b/src/backends/port/fs.ts @@ -1,27 +1,31 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { pick, type ExtractProperties } from 'utilium'; +import type { ExtractProperties } from 'utilium'; import type { MountConfiguration } from '../../config.js'; import type { CreationOptions, FileSystemMetadata } from '../../filesystem.js'; import type { Backend, FilesystemOf } from '../backend.js'; +import type { Inode, InodeLike } from '../store/inode.js'; +import type { File } from '../../file.js'; -import type { TransferListItem } from 'node:worker_threads'; +import { pick } from 'utilium'; import { resolveMountConfig } from '../../config.js'; import { Errno, ErrnoError } from '../../error.js'; -import { File } from '../../file.js'; import { FileSystem } from '../../filesystem.js'; import { Async } from '../../mixins/async.js'; import { Stats } from '../../stats.js'; +import { decodeUTF8 } from '../../utils.js'; import { InMemory } from '../memory.js'; -import type { Inode, InodeLike } from '../store/inode.js'; import * as RPC from './rpc.js'; type FSMethods = ExtractProperties Promise | FileSystemMetadata>; type FSMethod = keyof FSMethods; -/** @internal */ -export interface FSRequest extends RPC.Request { - method: TMethod; - args: Parameters; -} + +export type FSRequest = RPC.Message & + { + [M in TMethod]: { + method: M; + args: Parameters; + }; + }[TMethod]; /** * PortFS lets you access an FS instance that is running in a port, or the other way around. @@ -54,7 +58,7 @@ export class PortFS extends Async(FileSystem) { } protected rpc(method: T, ...args: Parameters): Promise>> { - return RPC.request, Awaited>>({ method, args }, { ...this.options, fs: this }); + return RPC.request, Awaited>>({ method, args } as Omit, 'id' | 'stack' | '_zenfs'>, { ...this.options, fs: this }); } public async ready(): Promise { @@ -107,8 +111,9 @@ export class PortFS extends Async(FileSystem) { return this.rpc('link', srcpath, dstpath); } - public read(path: string, buffer: Uint8Array, offset: number, length: number): Promise { - return this.rpc('read', path, buffer, offset, length); + public async read(path: string, buffer: Uint8Array, offset: number, length: number): Promise { + const _buf = (await this.rpc('read', path, buffer, offset, length)) as unknown as Uint8Array; + buffer.set(_buf); } public write(path: string, buffer: Uint8Array, offset: number): Promise { @@ -125,31 +130,29 @@ export async function handleRequest(port: RPC.Port, fs: FileSystem & { _descript let value, error: boolean = false; - const transfer: TransferListItem[] = []; - try { // @ts-expect-error 2556 value = await fs[method](...args); - if (value instanceof File) { - await using file = await fs.openFile(args[0] as string, 'r+'); - const stats = await file.stat(); - const data = new Uint8Array(stats.size); - - await file.read(data); - value = { - path: value.path, - flag: args[1] as string, - stats, - buffer: data.buffer, - } satisfies RPC.FileData; - transfer.push(data.buffer); + switch (method) { + case 'openFile': + case 'createFile': { + value = { + path: args[0], + flag: args[1], + stats: await fs.stat(args[0]), + } satisfies RPC.FileData; + break; + } + case 'read': + value = args[1]; + break; } } catch (e: any) { value = e instanceof ErrnoError ? e.toJSON() : pick(e, 'message', 'stack'); error = true; } - port.postMessage({ _zenfs: true, id, error, method, stack, value }, transfer); + port.postMessage({ _zenfs: true, id, error, method, stack, value }); } export function attachFS(port: RPC.Port, fs: FileSystem): void { diff --git a/src/backends/port/rpc.ts b/src/backends/port/rpc.ts index 410fffb1..f333b1e8 100644 --- a/src/backends/port/rpc.ts +++ b/src/backends/port/rpc.ts @@ -7,7 +7,7 @@ import type { Backend, FilesystemOf } from '../backend.js'; import type { PortFS } from './fs.js'; import { Errno, ErrnoError } from '../../error.js'; -import { PreloadFile } from '../../file.js'; +import { LazyFile } from '../../file.js'; import { Stats, type StatsLike } from '../../stats.js'; import { handleRequest } from './fs.js'; @@ -57,13 +57,18 @@ interface _ResponseWithValue extends Message { value: Awaited extends File ? FileData : Awaited; } -export type Response = _ResponseWithError | _ResponseWithValue; +interface _ResponseRead extends Message { + error: false; + method: 'read'; + value: Uint8Array; +} + +export type Response = _ResponseWithError | _ResponseWithValue | _ResponseRead; export interface FileData { path: string; flag: string; stats: StatsLike; - buffer: ArrayBuffer; } function isFileData(value: unknown): value is FileData { @@ -129,8 +134,8 @@ export function handleResponse(response: TResp } if (isFileData(value)) { - const { path, flag, stats, buffer } = value; - const file = new PreloadFile(fs!, path, flag, new Stats(stats), new Uint8Array(buffer)); + const { path, flag, stats } = value; + const file = new LazyFile(fs!, path, flag, new Stats(stats)); resolve(file); executors.delete(id); return; diff --git a/src/backends/store/fs.ts b/src/backends/store/fs.ts index dee48eea..9292d11c 100644 --- a/src/backends/store/fs.ts +++ b/src/backends/store/fs.ts @@ -1,7 +1,7 @@ import { randomInt, serialize } from 'utilium'; import { Errno, ErrnoError } from '../../error.js'; import type { File } from '../../file.js'; -import { LazyFile, PreloadFile } from '../../file.js'; +import { LazyFile } from '../../file.js'; import type { CreationOptions, FileSystemMetadata, PureCreationOptions } from '../../filesystem.js'; import { FileSystem } from '../../filesystem.js'; import type { FileType, Stats } from '../../stats.js'; diff --git a/src/backends/store/index_fs.ts b/src/backends/store/index_fs.ts index 18e36020..8cfdaf21 100644 --- a/src/backends/store/index_fs.ts +++ b/src/backends/store/index_fs.ts @@ -1,6 +1,6 @@ import { ErrnoError } from '../../error.js'; import type { File } from '../../file.js'; -import { PreloadFile } from '../../file.js'; +import { LazyFile } from '../../file.js'; import type { CreationOptions } from '../../filesystem.js'; import { Stats } from '../../stats.js'; import { S_IFREG } from '../../vfs/constants.js'; @@ -58,14 +58,14 @@ export abstract class IndexFS extends StoreFS { public override async createFile(path: string, flag: string, mode: number, options: CreationOptions): Promise { const node = await this.commitNew(path, S_IFREG, { mode, ...options }, new Uint8Array(), 'createFile'); - const file = new PreloadFile(this, path, flag, node.toStats(), new Uint8Array()); + const file = new LazyFile(this, path, flag, node.toStats()); this.index.set(path, node); return file; } public createFileSync(path: string, flag: string, mode: number, options: CreationOptions): File { const node = this.commitNewSync(path, S_IFREG, { mode, ...options }, new Uint8Array(), 'createFile'); - const file = new PreloadFile(this, path, flag, node.toStats(), new Uint8Array()); + const file = new LazyFile(this, path, flag, node.toStats()); this.index.set(path, node); return file; } diff --git a/src/mixins/async.ts b/src/mixins/async.ts index e2871f54..8446dafe 100644 --- a/src/mixins/async.ts +++ b/src/mixins/async.ts @@ -5,7 +5,7 @@ import type { _SyncFSKeys, AsyncFSMethods, Mixin } from './shared.js'; import { StoreFS } from '../backends/store/fs.js'; import { Errno, ErrnoError } from '../error.js'; -import { parseFlag, PreloadFile } from '../file.js'; +import { LazyFile, parseFlag } from '../file.js'; import { join } from '../vfs/path.js'; /** @internal */ @@ -36,7 +36,7 @@ export interface AsyncMixin extends Pick(FS: T): Mixin { - abstract class AsyncFS extends FS { + abstract class AsyncFS extends FS implements AsyncMixin { /** * Queue of pending asynchronous operations. */ @@ -116,20 +116,17 @@ export function Async(FS: T): Mixin { + public createFileSync(path: string, flag: string, mode: number, options: CreationOptions): LazyFile { this.checkSync(path, 'createFile'); this._sync.createFileSync(path, flag, mode, options); this.queue('createFile', path, flag, mode, options); return this.openFileSync(path, flag); } - public openFileSync(path: string, flag: string): PreloadFile { + public openFileSync(path: string, flag: string): LazyFile { this.checkSync(path, 'openFile'); - const file = this._sync.openFileSync(path, flag + '+'); - const stats = file.statSync(); - const buffer = new Uint8Array(stats.size); - file.readSync(buffer); - return new PreloadFile(this, path, flag, stats, buffer); + const stats = this._sync.statSync(path); + return new LazyFile(this, path, flag, stats); } public unlinkSync(path: string): void { @@ -172,6 +169,17 @@ export function Async(FS: T): Mixin implements FileSystem { * For example, on an OverlayFS instance with an async lower * directory operations like rename and rmdir may involve multiple * requests involving both the upper and lower file systems -- they - * are not executed in a single atomic step. OverlayFS uses this + * are not executed in a single atomic step. OverlayFS used to use this * to avoid having to reason about the correctness of * multiple requests interleaving. *