Skip to content

Commit

Permalink
Add API to choose non-default preopens (#1317)
Browse files Browse the repository at this point in the history
I need to find a proper document explaining how this all works.
As is it works but is not following a proper spec.
  • Loading branch information
squarejesse authored Jul 30, 2023
1 parent 0aac566 commit c7e7ad3
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 35 deletions.
12 changes: 9 additions & 3 deletions okio-wasifilesystem/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,12 @@ val injectWasiInit by tasks.creating {
outputs.file(entryPointMjs)

doLast {
val tmpdir = File(System.getProperty("java.io.tmpdir"), "okio-wasifilesystem-test")
tmpdir.mkdirs()
val base = File(System.getProperty("java.io.tmpdir"), "okio-wasifilesystem-test")
val baseA = File(base, "a")
val baseB = File(base, "b")
base.mkdirs()
baseA.mkdirs()
baseB.mkdirs()

entryPointMjs.writeText(
"""
Expand All @@ -75,7 +79,9 @@ val injectWasiInit by tasks.creating {
export const wasi = new WASI({
version: 'preview1',
preopens: {
'/tmp': '$tmpdir'
'/tmp': '$base',
'/a': '$baseA',
'/b': '$baseB'
}
});
Expand Down
76 changes: 57 additions & 19 deletions okio-wasifilesystem/src/wasmMain/kotlin/okio/WasiFileSystem.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import okio.Path.Companion.toPath
import okio.internal.ErrnoException
import okio.internal.fdClose
import okio.internal.preview1.Errno
import okio.internal.preview1.FirstPreopenDirectoryTmp
import okio.internal.preview1.dirnamelen
import okio.internal.preview1.fd
import okio.internal.preview1.fd_readdir
Expand Down Expand Up @@ -60,7 +59,18 @@ import okio.internal.write
*
* [WASI]: https://wasi.dev/
*/
object WasiFileSystem : FileSystem() {
class WasiFileSystem(
private val relativePathPreopen: Int = DEFAULT_FIRST_PREOPEN,
pathToPreopen: Map<Path, Int> = mapOf("/".toPath() to DEFAULT_FIRST_PREOPEN),
) : FileSystem() {
private val pathSegmentsToPreopen = pathToPreopen.mapKeys { (key, _) -> key.segmentsBytes }

init {
require(pathSegmentsToPreopen.isNotEmpty()) {
"pathToPreopen must be non-empty"
}
}

override fun canonicalize(path: Path): Path {
// There's no APIs in preview1 to canonicalize a path. We give it a best effort by resolving
// all symlinks, but this could result in a relative path.
Expand Down Expand Up @@ -108,7 +118,7 @@ object WasiFileSystem : FileSystem() {
val (pathAddress, pathSize) = allocator.write(path.toString())

val errno = path_filestat_get(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd(path) ?: return null,
flags = 0,
path = pathAddress.address.toInt(),
pathSize = pathSize,
Expand Down Expand Up @@ -144,7 +154,7 @@ object WasiFileSystem : FileSystem() {
val bufPointer = allocator.allocate(bufLen)
val readlinkReturnPointer = allocator.allocate(4) // `size` is u32, 4 bytes.
val readlinkErrno = path_readlink(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd(path) ?: return null,
path = pathAddress.address.toInt(),
pathSize = pathSize,
buf = bufPointer.address.toInt(),
Expand Down Expand Up @@ -174,7 +184,7 @@ object WasiFileSystem : FileSystem() {

override fun list(dir: Path): List<Path> {
val fd = pathOpen(
path = dir.toString(),
path = dir,
oflags = oflag_directory,
rightsBase = right_fd_readdir,
)
Expand Down Expand Up @@ -252,7 +262,7 @@ object WasiFileSystem : FileSystem() {
right_fd_seek or
right_fd_sync
val fd = pathOpen(
path = file.toString(),
path = file,
oflags = 0,
rightsBase = rightsBase,
)
Expand All @@ -275,7 +285,7 @@ object WasiFileSystem : FileSystem() {
right_fd_sync or
right_fd_write
val fd = pathOpen(
path = file.toString(),
path = file,
oflags = oflags,
rightsBase = rightsBase,
)
Expand All @@ -285,7 +295,7 @@ object WasiFileSystem : FileSystem() {
override fun source(file: Path): Source {
return FileSource(
fd = pathOpen(
path = file.toString(),
path = file,
oflags = 0,
rightsBase = right_fd_read,
),
Expand All @@ -300,7 +310,7 @@ object WasiFileSystem : FileSystem() {

return FileSink(
fd = pathOpen(
path = file.toString(),
path = file,
oflags = oflags,
rightsBase = right_fd_write or right_fd_sync,
),
Expand All @@ -315,7 +325,7 @@ object WasiFileSystem : FileSystem() {

return FileSink(
fd = pathOpen(
path = file.toString(),
path = file,
oflags = oflags,
rightsBase = right_fd_write,
fdflags = fdflags_append,
Expand All @@ -328,7 +338,7 @@ object WasiFileSystem : FileSystem() {
val (pathAddress, pathSize) = allocator.write(dir.toString())

val errno = path_create_directory(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd(dir) ?: throw FileNotFoundException("no preopen: $dir"),
path = pathAddress.address.toInt(),
pathSize = pathSize,
)
Expand All @@ -347,10 +357,10 @@ object WasiFileSystem : FileSystem() {
val (targetPathAddress, targetPathSize) = allocator.write(target.toString())

val errno = path_rename(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd(source) ?: throw FileNotFoundException("no preopen: $source"),
old_path = sourcePathAddress.address.toInt(),
old_pathSize = sourcePathSize,
new_fd = FirstPreopenDirectoryTmp,
new_fd = preopenFd(target) ?: throw FileNotFoundException("no preopen: $target"),
new_path = targetPathAddress.address.toInt(),
new_pathSize = targetPathSize,
)
Expand All @@ -365,9 +375,10 @@ object WasiFileSystem : FileSystem() {
override fun delete(path: Path, mustExist: Boolean) {
withScopedMemoryAllocator { allocator ->
val (pathAddress, pathSize) = allocator.write(path.toString())
val preopenFd = preopenFd(path) ?: throw FileNotFoundException("no preopen: $path")

var errno = path_unlink_file(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd,
path = pathAddress.address.toInt(),
pathSize = pathSize,
)
Expand All @@ -382,7 +393,7 @@ object WasiFileSystem : FileSystem() {
Errno.isdir.ordinal,
-> {
errno = path_remove_directory(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd,
path = pathAddress.address.toInt(),
pathSize = pathSize,
)
Expand All @@ -400,7 +411,7 @@ object WasiFileSystem : FileSystem() {
val errno = path_symlink(
old_path = targetPathAddress.address.toInt(),
old_pathSize = targetPathSize,
fd = FirstPreopenDirectoryTmp,
fd = preopenFd(source) ?: throw FileNotFoundException("no preopen: $source"),
new_path = sourcePathAddress.address.toInt(),
new_pathSize = sourcePathSize,
)
Expand All @@ -409,17 +420,18 @@ object WasiFileSystem : FileSystem() {
}

private fun pathOpen(
path: String,
path: Path,
oflags: oflags,
rightsBase: rights,
fdflags: fdflags = 0,
): fd {
withScopedMemoryAllocator { allocator ->
val (pathAddress, pathSize) = allocator.write(path)
val preopenFd = preopenFd(path) ?: throw FileNotFoundException("no preopen: $path")
val (pathAddress, pathSize) = allocator.write(path.toString())

val returnPointer: Pointer = allocator.allocate(4) // fd is u32.
val errno = path_open(
fd = FirstPreopenDirectoryTmp,
fd = preopenFd,
dirflags = 0,
path = pathAddress.address.toInt(),
pathSize = pathSize,
Expand All @@ -437,5 +449,31 @@ object WasiFileSystem : FileSystem() {
}
}

/**
* Returns the file descriptor of the preopened path that is an ancestor of [path]. Returns null
* if there is no such file descriptor.
*/
private fun preopenFd(path: Path): fd? {
if (path.isRelative) return relativePathPreopen

val pathSegmentsBytes = path.segmentsBytes
for ((candidate, fd) in pathSegmentsToPreopen) {
if (pathSegmentsBytes.size < candidate.size) continue
if (pathSegmentsBytes.subList(0, candidate.size) != candidate) continue
return fd
}
return null
}

override fun toString() = "okio.WasiFileSystem"

companion object {
/**
* File descriptor of the first preopen in the `WASI` instance's configured `preopens` property.
* This is 3 by default, assuming `stdin` is 0, `stdout` is 1, and `stderr` is 2.
*
* Other preopens are assigned sequentially starting at this value.
*/
val DEFAULT_FIRST_PREOPEN = 3
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,6 @@ typealias dirnamelen = Int
*/
typealias PointerU8 = Int

val Stdin: fd = 0
val Stdout: fd = 1
val Stderr: fd = 2

/**
* Assume the /tmp directory is fd 3.
*
* TODO: look this up at runtime from whatever parent directory is requested.
*/
val FirstPreopenDirectoryTmp: fd = 3

/**
* path_create_directory(fd: fd, path: string) -> Result<(), errno>
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* Copyright (C) 2023 Square, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package okio

import kotlin.test.BeforeTest
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertNull
import okio.Path.Companion.toPath
import okio.WasiFileSystem.Companion.DEFAULT_FIRST_PREOPEN

/**
* Confirm the [WasiFileSystem] can operate on different preopened directories independently.
*
* This tracks the `preopens` attribute in `.mjs` script in `okio-wasifilesystem/build.gradle.kts`.
*/
class WasiFileSystemPreopensTest {
private val fileSystem = WasiFileSystem(
relativePathPreopen = DEFAULT_FIRST_PREOPEN,
pathToPreopen = mapOf(
"/tmp".toPath() to DEFAULT_FIRST_PREOPEN,
"/a".toPath() to DEFAULT_FIRST_PREOPEN + 1,
"/b".toPath() to DEFAULT_FIRST_PREOPEN + 2,
),
)

private val testId = "${this::class.simpleName}-${randomToken(16)}"
private val baseA: Path = "/a".toPath() / testId
private val baseB: Path = "/b".toPath() / testId

@BeforeTest
fun setUp() {
fileSystem.createDirectory(baseA)
fileSystem.createDirectory(baseB)
}

@Test
fun operateOnPreopens() {
fileSystem.write(baseA / "a.txt") {
writeUtf8("hello world a")
}
fileSystem.write(baseB / "b.txt") {
writeUtf8("bello burld")
}
assertEquals(
"hello world a".length.toLong(),
fileSystem.metadata(baseA / "a.txt").size,
)
assertEquals(
"bello burld".length.toLong(),
fileSystem.metadata(baseB / "b.txt").size,
)
}

@Test
fun operateAcrossPreopens() {
fileSystem.write(baseA / "a.txt") {
writeUtf8("hello world")
}

fileSystem.atomicMove(baseA / "a.txt", baseB / "b.txt")

assertEquals(
"hello world",
fileSystem.read(baseB / "b.txt") {
readUtf8()
},
)
}

@Test
fun cannotOperateOutsideOfPreopens() {
val noPreopen = "/c".toPath() / testId
assertFailsWith<FileNotFoundException> {
fileSystem.createDirectory(noPreopen)
}
assertFailsWith<FileNotFoundException> {
fileSystem.sink(noPreopen)
}
assertNull(fileSystem.metadataOrNull(noPreopen))
assertFailsWith<FileNotFoundException> {
fileSystem.metadata(noPreopen)
}
assertNull(fileSystem.listOrNull(noPreopen))
assertFailsWith<FileNotFoundException> {
fileSystem.list(noPreopen)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import okio.Path.Companion.toPath

class WasiFileSystemTest : AbstractFileSystemTest(
clock = WasiClock,
fileSystem = WasiFileSystem,
fileSystem = WasiFileSystem(),
windowsLimitations = Path.DIRECTORY_SEPARATOR == "\\",
allowClobberingEmptyDirectories = Path.DIRECTORY_SEPARATOR == "\\",
allowAtomicMoveFromFileToDirectory = false,
Expand Down
2 changes: 1 addition & 1 deletion okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import okio.ByteString.Companion.encodeUtf8
import okio.Path.Companion.toPath

class WasiTest {
private val fileSystem = WasiFileSystem
private val fileSystem = WasiFileSystem()
private val base: Path = "/tmp".toPath() / "${this::class.simpleName}-${randomToken(16)}"

@BeforeTest
Expand Down

0 comments on commit c7e7ad3

Please sign in to comment.