From 434f20eabb8cb0ee8ac5b9d088c3bc310383cd06 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:21:01 +0200 Subject: [PATCH] Pass Uint8Array from/to fd_read/fd_write This way individual Fd implementations don't have to handle vectorized reads/writes and don't need direct access to the wasm linear memory. --- examples/rustc.html | 12 +-- src/fd.ts | 23 ++--- src/fs_mem.ts | 162 +++++++++----------------------- src/fs_opfs.ts | 42 +++------ src/wasi.ts | 74 ++++++++++++--- test/adapters/node/run-wasi.mjs | 11 +-- 6 files changed, 134 insertions(+), 190 deletions(-) diff --git a/examples/rustc.html b/examples/rustc.html index b6cfe1f..3e46b63 100644 --- a/examples/rustc.html +++ b/examples/rustc.html @@ -47,15 +47,11 @@ super(); this.term = term; } - fd_write(view8/*: Uint8Array*/, iovs/*: [wasi.Iovec]*/)/*: {ret: number, nwritten: number}*/ { + fd_write(data/*: Uint8Array*/)/*: {ret: number, nwritten: number}*/ { let nwritten = 0; - for (let iovec of iovs) { - console.log(iovec.buf_len, iovec.buf_len, view8.slice(iovec.buf, iovec.buf + iovec.buf_len)); - let buffer = view8.slice(iovec.buf, iovec.buf + iovec.buf_len); - this.term.writeUtf8(buffer); - nwritten += iovec.buf_len; - } - return { ret: 0, nwritten }; + console.log(data); + this.term.writeUtf8(data); + return { ret: 0, nwritten: data.byteLength }; } } diff --git a/src/fd.ts b/src/fd.ts index b192ce9..2227ab6 100644 --- a/src/fd.ts +++ b/src/fd.ts @@ -35,12 +35,8 @@ export abstract class Fd { fd_filestat_set_times(atim: bigint, mtim: bigint, fst_flags: number): number { return wasi.ERRNO_NOTSUP; } - fd_pread( - view8: Uint8Array, - iovs: Array, - offset: bigint, - ): { ret: number; nread: number } { - return { ret: wasi.ERRNO_NOTSUP, nread: 0 }; + fd_pread(size: number, offset: bigint): { ret: number; data: Uint8Array } { + return { ret: wasi.ERRNO_NOTSUP, data: new Uint8Array() }; } fd_prestat_get(): { ret: number; prestat: wasi.Prestat | null } { return { ret: wasi.ERRNO_NOTSUP, prestat: null }; @@ -49,17 +45,13 @@ export abstract class Fd { return { ret: wasi.ERRNO_NOTSUP, prestat_dir_name: null }; } fd_pwrite( - view8: Uint8Array, - iovs: Array, + data: Uint8Array, offset: bigint, ): { ret: number; nwritten: number } { return { ret: wasi.ERRNO_NOTSUP, nwritten: 0 }; } - fd_read( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nread: number } { - return { ret: wasi.ERRNO_NOTSUP, nread: 0 }; + fd_read(size: number): { ret: number; data: Uint8Array } { + return { ret: wasi.ERRNO_NOTSUP, data: new Uint8Array() }; } fd_readdir_single(cookie: bigint): { ret: number; @@ -76,10 +68,7 @@ export abstract class Fd { fd_tell(): { ret: number; offset: bigint } { return { ret: wasi.ERRNO_NOTSUP, offset: 0n }; } - fd_write( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nwritten: number } { + fd_write(data: Uint8Array): { ret: number; nwritten: number } { return { ret: wasi.ERRNO_NOTSUP, nwritten: 0 }; } path_create_directory(path: string): number { diff --git a/src/fs_mem.ts b/src/fs_mem.ts index 4cea8f0..d089bb6 100644 --- a/src/fs_mem.ts +++ b/src/fs_mem.ts @@ -42,47 +42,21 @@ export class OpenFile extends Fd { return wasi.ERRNO_SUCCESS; } - fd_read( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nread: number } { - let nread = 0; - for (const iovec of iovs) { - if (this.file_pos < this.file.data.byteLength) { - const slice = this.file.data.slice( - Number(this.file_pos), - Number(this.file_pos + BigInt(iovec.buf_len)), - ); - view8.set(slice, iovec.buf); - this.file_pos += BigInt(slice.length); - nread += slice.length; - } else { - break; - } - } - return { ret: 0, nread }; + fd_read(size: number): { ret: number; data: Uint8Array } { + const slice = this.file.data.slice( + Number(this.file_pos), + Number(this.file_pos + BigInt(size)), + ); + this.file_pos += BigInt(slice.length); + return { ret: 0, data: slice }; } - fd_pread( - view8: Uint8Array, - iovs: Array, - offset: bigint, - ): { ret: number; nread: number } { - let nread = 0; - for (const iovec of iovs) { - if (offset < this.file.data.byteLength) { - const slice = this.file.data.slice( - Number(offset), - Number(offset + BigInt(iovec.buf_len)), - ); - view8.set(slice, iovec.buf); - offset += BigInt(slice.length); - nread += slice.length; - } else { - break; - } - } - return { ret: 0, nread }; + fd_pread(size: number, offset: bigint): { ret: number; data: Uint8Array } { + const slice = this.file.data.slice( + Number(offset), + Number(offset + BigInt(size)), + ); + return { ret: 0, data: slice }; } fd_seek(offset: bigint, whence: number): { ret: number; offset: bigint } { @@ -113,51 +87,33 @@ export class OpenFile extends Fd { return { ret: 0, offset: this.file_pos }; } - fd_write( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nwritten: number } { - let nwritten = 0; - if (this.file.readonly) return { ret: wasi.ERRNO_BADF, nwritten }; - for (const iovec of iovs) { - const buffer = view8.slice(iovec.buf, iovec.buf + iovec.buf_len); - if (this.file_pos + BigInt(buffer.byteLength) > this.file.size) { - const old = this.file.data; - this.file.data = new Uint8Array( - Number(this.file_pos + BigInt(buffer.byteLength)), - ); - this.file.data.set(old); - } - this.file.data.set( - buffer.slice(0, Number(this.file.size - this.file_pos)), - Number(this.file_pos), + fd_write(data: Uint8Array): { ret: number; nwritten: number } { + if (this.file.readonly) return { ret: wasi.ERRNO_BADF, nwritten: 0 }; + + if (this.file_pos + BigInt(data.byteLength) > this.file.size) { + const old = this.file.data; + this.file.data = new Uint8Array( + Number(this.file_pos + BigInt(data.byteLength)), ); - this.file_pos += BigInt(buffer.byteLength); - nwritten += iovec.buf_len; + this.file.data.set(old); } - return { ret: 0, nwritten }; + + this.file.data.set(data, Number(this.file_pos)); + this.file_pos += BigInt(data.byteLength); + return { ret: 0, nwritten: data.byteLength }; } - fd_pwrite(view8: Uint8Array, iovs: Array, offset: bigint) { - let nwritten = 0; - if (this.file.readonly) return { ret: wasi.ERRNO_BADF, nwritten }; - for (const iovec of iovs) { - const buffer = view8.slice(iovec.buf, iovec.buf + iovec.buf_len); - if (offset + BigInt(buffer.byteLength) > this.file.size) { - const old = this.file.data; - this.file.data = new Uint8Array( - Number(offset + BigInt(buffer.byteLength)), - ); - this.file.data.set(old); - } - this.file.data.set( - buffer.slice(0, Number(this.file.size - offset)), - Number(offset), - ); - offset += BigInt(buffer.byteLength); - nwritten += iovec.buf_len; + fd_pwrite(data: Uint8Array, offset: bigint) { + if (this.file.readonly) return { ret: wasi.ERRNO_BADF, nwritten: 0 }; + + if (offset + BigInt(data.byteLength) > this.file.size) { + const old = this.file.data; + this.file.data = new Uint8Array(Number(offset + BigInt(data.byteLength))); + this.file.data.set(old); } - return { ret: 0, nwritten }; + + this.file.data.set(data, Number(offset)); + return { ret: 0, nwritten: data.byteLength }; } fd_filestat_get(): { ret: number; filestat: wasi.Filestat } { @@ -464,40 +420,24 @@ export class OpenDirectory extends Fd { return wasi.ERRNO_BADF; } - fd_read( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - view8: Uint8Array, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - iovs: wasi.Iovec[], - ): { ret: number; nread: number } { - return { ret: wasi.ERRNO_BADF, nread: 0 }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + fd_read(size: number): { ret: number; data: Uint8Array } { + return { ret: wasi.ERRNO_BADF, data: new Uint8Array() }; } - fd_pread( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - view8: Uint8Array, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - iovs: wasi.Iovec[], - // eslint-disable-next-line @typescript-eslint/no-unused-vars - offset: bigint, - ): { ret: number; nread: number } { - return { ret: wasi.ERRNO_BADF, nread: 0 }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + fd_pread(size: number, offset: bigint): { ret: number; data: Uint8Array } { + return { ret: wasi.ERRNO_BADF, data: new Uint8Array() }; } - fd_write( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - view8: Uint8Array, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - iovs: wasi.Ciovec[], - ): { ret: number; nwritten: number } { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + fd_write(data: Uint8Array): { ret: number; nwritten: number } { return { ret: wasi.ERRNO_BADF, nwritten: 0 }; } fd_pwrite( // eslint-disable-next-line @typescript-eslint/no-unused-vars - view8: Uint8Array, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - iovs: wasi.Ciovec[], + data: Uint8Array, // eslint-disable-next-line @typescript-eslint/no-unused-vars offset: bigint, ): { ret: number; nwritten: number } { @@ -785,17 +725,9 @@ export class ConsoleStdout extends Fd { return { ret: 0, fdstat }; } - fd_write( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nwritten: number } { - let nwritten = 0; - for (const iovec of iovs) { - const buffer = view8.slice(iovec.buf, iovec.buf + iovec.buf_len); - this.write(buffer); - nwritten += iovec.buf_len; - } - return { ret: 0, nwritten }; + fd_write(data: Uint8Array): { ret: number; nwritten: number } { + this.write(data); + return { ret: 0, nwritten: data.byteLength }; } static lineBuffered(write: (line: string) => void): ConsoleStdout { diff --git a/src/fs_opfs.ts b/src/fs_opfs.ts index df43876..d1fe42a 100644 --- a/src/fs_opfs.ts +++ b/src/fs_opfs.ts @@ -101,22 +101,11 @@ export class OpenSyncOPFSFile extends Fd { return wasi.ERRNO_SUCCESS; } - fd_read( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nread: number } { - let nread = 0; - for (const iovec of iovs) { - if (this.position < this.file.handle.getSize()) { - const buf = new Uint8Array(view8.buffer, iovec.buf, iovec.buf_len); - const n = this.file.handle.read(buf, { at: Number(this.position) }); - this.position += BigInt(n); - nread += n; - } else { - break; - } - } - return { ret: 0, nread }; + fd_read(size: number): { ret: number; data: Uint8Array } { + const buf = new Uint8Array(size); + const n = this.file.handle.read(buf, { at: Number(this.position) }); + this.position += BigInt(n); + return { ret: 0, data: buf.slice(0, n) }; } fd_seek( @@ -144,20 +133,13 @@ export class OpenSyncOPFSFile extends Fd { return { ret: wasi.ERRNO_SUCCESS, offset: this.position }; } - fd_write( - view8: Uint8Array, - iovs: Array, - ): { ret: number; nwritten: number } { - let nwritten = 0; - if (this.file.readonly) return { ret: wasi.ERRNO_BADF, nwritten }; - for (const iovec of iovs) { - const buf = new Uint8Array(view8.buffer, iovec.buf, iovec.buf_len); - // don't need to extend file manually, just write - const n = this.file.handle.write(buf, { at: Number(this.position) }); - this.position += BigInt(n); - nwritten += n; - } - return { ret: wasi.ERRNO_SUCCESS, nwritten }; + fd_write(data: Uint8Array): { ret: number; nwritten: number } { + if (this.file.readonly) return { ret: wasi.ERRNO_BADF, nwritten: 0 }; + + // don't need to extend file manually, just write + const n = this.file.handle.write(data, { at: Number(this.position) }); + this.position += BigInt(n); + return { ret: wasi.ERRNO_SUCCESS, nwritten: n }; } fd_datasync(): number { diff --git a/src/wasi.ts b/src/wasi.ts index 9da80f9..9c906b0 100644 --- a/src/wasi.ts +++ b/src/wasi.ts @@ -300,9 +300,22 @@ export default class WASI { iovs_ptr, iovs_len, ); - const { ret, nread } = self.fds[fd].fd_pread(buffer8, iovecs, offset); + let nread = 0; + for (const iovec of iovecs) { + const { ret, data } = self.fds[fd].fd_pread(iovec.buf_len, offset); + if (ret != wasi.ERRNO_SUCCESS) { + buffer.setUint32(nread_ptr, nread, true); + return ret; + } + buffer8.set(data, iovec.buf); + nread += data.length; + offset += BigInt(data.length); + if (data.length != iovec.buf_len) { + break; + } + } buffer.setUint32(nread_ptr, nread, true); - return ret; + return wasi.ERRNO_SUCCESS; } else { return wasi.ERRNO_BADF; } @@ -358,13 +371,25 @@ export default class WASI { iovs_ptr, iovs_len, ); - const { ret, nwritten } = self.fds[fd].fd_pwrite( - buffer8, - iovecs, - offset, - ); + let nwritten = 0; + for (const iovec of iovecs) { + const data = buffer8.slice(iovec.buf, iovec.buf + iovec.buf_len); + const { ret, nwritten: nwritten_part } = self.fds[fd].fd_pwrite( + data, + offset, + ); + if (ret != wasi.ERRNO_SUCCESS) { + buffer.setUint32(nwritten_ptr, nwritten, true); + return ret; + } + nwritten += nwritten_part; + offset += BigInt(nwritten_part); + if (nwritten_part != data.byteLength) { + break; + } + } buffer.setUint32(nwritten_ptr, nwritten, true); - return ret; + return wasi.ERRNO_SUCCESS; } else { return wasi.ERRNO_BADF; } @@ -383,9 +408,21 @@ export default class WASI { iovs_ptr, iovs_len, ); - const { ret, nread } = self.fds[fd].fd_read(buffer8, iovecs); + let nread = 0; + for (const iovec of iovecs) { + const { ret, data } = self.fds[fd].fd_read(iovec.buf_len); + if (ret != wasi.ERRNO_SUCCESS) { + buffer.setUint32(nread_ptr, nread, true); + return ret; + } + buffer8.set(data, iovec.buf); + nread += data.length; + if (data.length != iovec.buf_len) { + break; + } + } buffer.setUint32(nread_ptr, nread, true); - return ret; + return wasi.ERRNO_SUCCESS; } else { return wasi.ERRNO_BADF; } @@ -510,9 +547,22 @@ export default class WASI { iovs_ptr, iovs_len, ); - const { ret, nwritten } = self.fds[fd].fd_write(buffer8, iovecs); + let nwritten = 0; + for (const iovec of iovecs) { + const data = buffer8.slice(iovec.buf, iovec.buf + iovec.buf_len); + const { ret, nwritten: nwritten_part } = + self.fds[fd].fd_write(data); + if (ret != wasi.ERRNO_SUCCESS) { + buffer.setUint32(nwritten_ptr, nwritten, true); + return ret; + } + nwritten += nwritten_part; + if (nwritten_part != data.byteLength) { + break; + } + } buffer.setUint32(nwritten_ptr, nwritten, true); - return ret; + return wasi.ERRNO_SUCCESS; } else { return wasi.ERRNO_BADF; } diff --git a/test/adapters/node/run-wasi.mjs b/test/adapters/node/run-wasi.mjs index ac31367..4a82d0d 100644 --- a/test/adapters/node/run-wasi.mjs +++ b/test/adapters/node/run-wasi.mjs @@ -25,14 +25,9 @@ class NodeStdout extends Fd { return { ret: 0, fdstat }; } - fd_write(view8, iovs) { - let nwritten = 0; - for (let iovec of iovs) { - let buffer = view8.slice(iovec.buf, iovec.buf + iovec.buf_len); - this.out.write(buffer); - nwritten += iovec.buf_len; - } - return { ret: 0, nwritten }; + fd_write(data) { + this.out.write(data); + return { ret: 0, nwritten: data.byteLength }; } }