feat: host I/O access using a Hybrid VFS

packages/cli/src/main/kotlin/elide/tool/cli/cmd/repl/ToolShellCommand.kt

@@ -67,6 +67,7 @@ import elide.runtime.Logging
 import elide.runtime.core.PolyglotContext
 import elide.runtime.core.PolyglotEngine
 import elide.runtime.core.PolyglotEngineConfiguration
+import elide.runtime.core.PolyglotEngineConfiguration.HostAccess
 import elide.runtime.core.extensions.attach
 import elide.runtime.gvm.internals.GraalVMGuest
 import elide.runtime.gvm.internals.IntrinsicsManager
@@ -1488,6 +1489,14 @@ import elide.tool.project.ProjectManager
     if (debug) debugger.apply(this)
     inspector.apply(this)
 
+    // configure host access rules
+    hostAccess = when {
+      accessControl.allowAll -> HostAccess.ALLOW_ALL
+      accessControl.allowIo -> HostAccess.ALLOW_IO
+      accessControl.allowEnv -> HostAccess.ALLOW_ENV
+      else -> HostAccess.ALLOW_NONE
+    }
+
     // configure environment variables
     appEnvironment.apply(project, this)
 

packages/graalvm/src/main/kotlin/elide/runtime/gvm/internals/vfs/HybridVfs.kt

@@ -0,0 +1,137 @@
+package elide.runtime.gvm.internals.vfs
+
+import org.graalvm.polyglot.io.FileSystem
+import java.net.URI
+import java.nio.channels.SeekableByteChannel
+import java.nio.file.*
+import java.nio.file.DirectoryStream.Filter
+import java.nio.file.attribute.FileAttribute
+import kotlin.io.path.pathString
+
+/**
+ * A hybrid [FileSystem] implementation using two layers: an in-memory [overlay], which takes priority for reads but
+ * ignores writes, and a [backing] layer that can be written to, and which handles any requests not satisfied by the
+ * overlay.
+ *
+ * Note that the current implementation is designed with a specific combination in mind: a JIMFS-backed embedded VFS
+ * as the overlay (using [EmbeddedGuestVFSImpl]), and a host-backed VFS as the base layer (using [HostVFSImpl]).
+ *
+ * Instances of this class can be acquired using [HybridVfs.acquire].
+ */
+internal class HybridVfs private constructor(
+  private val backing: FileSystem,
+  private val overlay: FileSystem,
+) : FileSystem {
+  /**
+   * Convert this path to one that can be used by the [overlay] vfs.
+   *
+   * Because of incompatible types used by the underlying JIMFS and the platform-default file system, paths created
+   * using, for example, [Path.of], will not be recognized properly, and they must be transformed before use.
+   */
+  private fun Path.forEmbedded(): Path {
+    return overlay.parsePath(pathString)
+  }
+
+  override fun parsePath(uri: URI?): Path {
+    return backing.parsePath(uri)
+  }
+
+  override fun parsePath(path: String?): Path {
+    return backing.parsePath(path)
+  }
+
+  override fun toAbsolutePath(path: Path?): Path {
+    return backing.toAbsolutePath(path)
+  }
+
+  override fun toRealPath(path: Path?, vararg linkOptions: LinkOption?): Path {
+    return backing.toRealPath(path, *linkOptions)
+  }
+
+  override fun createDirectory(dir: Path?, vararg attrs: FileAttribute<*>?) {
+    return backing.createDirectory(dir, *attrs)
+  }
+
+  override fun delete(path: Path?) {
+    backing.delete(path)
+  }
+
+  override fun checkAccess(path: Path, modes: MutableSet<out AccessMode>, vararg linkOptions: LinkOption) {
+    // if only READ is requested, try the in-memory vfs first
+    if (modes.size == 0 || (modes.size == 1 && modes.contains(AccessMode.READ))) runCatching {
+      // ensure the path is compatible with the embedded vfs before passing it
+      overlay.checkAccess(path.forEmbedded(), modes, *linkOptions)
+      return
+    }
+
+    // if WRITE or EXECUTE were requested, or if the in-memory vfs denied access,
+    // try using the host instead
+    backing.checkAccess(path, modes, *linkOptions)
+  }
+
+  override fun newByteChannel(
+    path: Path,
+    options: MutableSet<out OpenOption>,
+    vararg attrs: FileAttribute<*>,
+  ): SeekableByteChannel {
+    // if only READ is requested, try the in-memory vfs first
+    if (options.size == 0 || (options.size == 1 && options.contains(StandardOpenOption.READ))) runCatching {
+      // ensure the path is compatible with the embedded vfs before passing it
+      return overlay.newByteChannel(path.forEmbedded(), options, *attrs)
+    }
+
+    // if write-related options were set, or the in-memory vfs failed to open the file
+    // (e.g. because it doesn't exist in the bundle), try using the host instead
+    return backing.newByteChannel(path, options, *attrs)
+  }
+
+  override fun newDirectoryStream(dir: Path, filter: Filter<in Path>?): DirectoryStream<Path> {
+    // try the in-memory vfs first
+    runCatching {
+      // ensure the path is compatible with the embedded vfs before passing it
+      return overlay.newDirectoryStream(dir.forEmbedded(), filter)
+    }
+
+    // if the in-memory vfs failed to open the directory, try using the host instead
+    return backing.newDirectoryStream(dir, filter)
+  }
+
+  override fun readAttributes(path: Path, attributes: String?, vararg options: LinkOption): MutableMap<String, Any> {
+    // try the in-memory vfs first
+    runCatching {
+      // ensure the path is compatible with the embedded vfs before passing it
+      return overlay.readAttributes(path.forEmbedded(), attributes, *options)
+    }
+
+    // if the in-memory vfs failed to read the file attributes, try using the host instead
+    return backing.readAttributes(path, attributes, *options)
+  }
+
+  companion object {
+    /**
+     * Configures a new [HybridVfs] using an in-memory VFS containing the provided [overlay] as [overlay], and the
+     * host file system as [backing] layer.
+     *
+     * @param overlay A list of bundles to be unpacked into the in-memory fs.
+     * @param writable Whether to allow writes to the backing layer.
+     * @return A new [HybridVfs] instance.
+     */
+    fun acquire(writable: Boolean, overlay: List<URI>): HybridVfs {
+      // configure an in-memory vfs with the provided bundles as overlay
+      val inMemory = EmbeddedGuestVFSImpl.Builder.newBuilder()
+        .setBundlePaths(overlay)
+        .setReadOnly(true)
+        .build()
+
+      // use the host fs as backing layer
+      val host = HostVFSImpl.Builder.newBuilder()
+        .setReadOnly(!writable)
+        .build()
+
+      return HybridVfs(
+        backing = host,
+        overlay = inMemory,
+      )
+    }
+  }
+}

packages/graalvm/src/main/kotlin/elide/runtime/plugins/vfs/VfsConfig.kt

@@ -16,24 +16,30 @@ package elide.runtime.plugins.vfs
 import java.net.URI
 import java.net.URL
 import elide.runtime.core.DelicateElideApi
+import elide.runtime.core.PolyglotEngineConfiguration
+import elide.runtime.core.PolyglotEngineConfiguration.HostAccess.ALLOW_ALL
+import elide.runtime.core.PolyglotEngineConfiguration.HostAccess.ALLOW_IO
 
 /** Configuration DSL for the [Vfs] plugin. */
-@DelicateElideApi public class VfsConfig internal constructor() {
+@DelicateElideApi public class VfsConfig internal constructor(configuration: PolyglotEngineConfiguration) {
   /** Private mutable list of registered bundles. */
   private val bundles: MutableList<URI> = mutableListOf()
-  
+
   /** Internal list of bundles registered for use in the VFS. */
   internal val registeredBundles: List<URI> get() = bundles
 
   /** Whether the file system is writable. If false, write operations will throw an exception. */
   public var writable: Boolean = false
-  
+
   /**
    * Whether to use the host's file system instead of an embedded VFS. If true, bundles registered using [include] will
    * not be applied.
+   *
+   * Enabled by default if the engine's [hostAccess][PolyglotEngineConfiguration.hostAccess] is set to [ALLOW_ALL] or
+   * [ALLOW_IO], otherwise false.
    */
-  internal var useHost: Boolean = false
-  
+  internal var useHost: Boolean = configuration.hostAccess == ALLOW_ALL || configuration.hostAccess == ALLOW_IO
+
   /** Register a [bundle] to be added to the VFS on creation. */
   public fun include(bundle: URI) {
     bundles.add(bundle)

packages/graalvm/src/main/kotlin/elide/runtime/plugins/vfs/VfsPlugin.kt

@@ -15,16 +15,15 @@ package elide.runtime.plugins.vfs
 
 import org.graalvm.polyglot.io.FileSystem
 import org.graalvm.polyglot.io.IOAccess
+import java.net.URI
+import elide.runtime.Logging
 import elide.runtime.core.*
 import elide.runtime.core.EngineLifecycleEvent.ContextCreated
 import elide.runtime.core.EngineLifecycleEvent.EngineCreated
 import elide.runtime.core.EnginePlugin.InstallationScope
 import elide.runtime.core.EnginePlugin.Key
-import elide.runtime.core.PolyglotEngineConfiguration.HostAccess
-import elide.runtime.core.PolyglotEngineConfiguration.HostAccess.ALLOW_ALL
-import elide.runtime.core.PolyglotEngineConfiguration.HostAccess.ALLOW_IO
-import elide.runtime.gvm.vfs.EmbeddedGuestVFS
-import elide.runtime.gvm.vfs.HostVFS
+import elide.runtime.gvm.internals.vfs.EmbeddedGuestVFSImpl
+import elide.runtime.gvm.internals.vfs.HybridVfs
 
 /**
  * Engine plugin providing configurable VFS support for polyglot contexts. Both embedded and host VFS implementations
@@ -43,21 +42,22 @@ import elide.runtime.gvm.vfs.HostVFS
  * ```
  */
 @DelicateElideApi public class Vfs private constructor(public val config: VfsConfig) {
+  /** Plugin logger instance. */
+  private val logging by lazy { Logging.of(Vfs::class) }
+
   /** Pre-configured VFS, created when the engine is initialized. */
   private lateinit var fileSystem: FileSystem
 
   internal fun onEngineCreated(@Suppress("unused_parameter") builder: PolyglotEngineBuilder) {
     // select the VFS implementation depending on the configuration
-    fileSystem = when (config.useHost) {
-      true -> when (config.writable) {
-        true -> HostVFS.acquireWritable()
-        false -> HostVFS.acquire()
-      }
-
-      false -> when (config.writable) {
-        true -> EmbeddedGuestVFS.writable(config.registeredBundles)
-        else -> EmbeddedGuestVFS.forBundles(config.registeredBundles)
-      }
+    // if no host access is requested, use an embedded in-memory vfs
+    fileSystem = if (!config.useHost) {
+      logging.debug("No host access requested, using in-memory vfs")
+      acquireEmbeddedVfs(config.writable, config.registeredBundles)
+    } else {
+      // if the configuration requires host access, we use a hybrid vfs
+      logging.debug("Host access requested, using hybrid vfs")
+      HybridVfs.acquire(config.writable, config.registeredBundles)
     }
   }
 
@@ -73,11 +73,7 @@ import elide.runtime.gvm.vfs.HostVFS
 
     override fun install(scope: InstallationScope, configuration: VfsConfig.() -> Unit): Vfs {
       // apply the configuration and create the plugin instance
-      val config = VfsConfig().apply(configuration)
-
-      // switch to the host's FS if requested in the general configuration
-      config.useHost = scope.configuration.hostAccess.useHostFs
-
+      val config = VfsConfig(scope.configuration).apply(configuration)
       val instance = Vfs(config)
 
       // subscribe to lifecycle events
@@ -87,7 +83,13 @@ import elide.runtime.gvm.vfs.HostVFS
       return instance
     }
 
-    private val HostAccess.useHostFs get() = this == ALLOW_IO || this == ALLOW_ALL
+    /** Build a new embedded [FileSystem], optionally [writable], using the specified [bundles]. */
+    private fun acquireEmbeddedVfs(writable: Boolean, bundles: List<URI>): FileSystem {
+      return EmbeddedGuestVFSImpl.Builder.newBuilder()
+        .setBundlePaths(bundles)
+        .setReadOnly(!writable)
+        .build()
+    }
   }
 }
 

packages/graalvm/src/test/kotlin/elide/runtime/gvm/internals/vfs/HybridVfsTest.kt

@@ -0,0 +1,112 @@
+package elide.runtime.gvm.internals.vfs
+
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.junit.jupiter.api.io.TempDir
+import java.nio.channels.Channels
+import java.nio.channels.SeekableByteChannel
+import java.nio.channels.WritableByteChannel
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import kotlin.io.path.createFile
+import kotlin.io.path.readText
+import kotlin.io.path.writeText
+import kotlin.test.assertEquals
+
+internal class HybridVfsTest {
+  /** Temporary directory used for host-related test cases. */
+  @TempDir lateinit var tempDirectory: Path
+
+  /** Read all data from this channel as a UTF-8 string. */
+  private fun SeekableByteChannel.readText(): String {
+    return Channels.newReader(this, Charsets.UTF_8).readText()
+  }
+
+  /** Write a [text] message into the channel using the UTF-8 charset and immediately flush the stream. */
+  private fun WritableByteChannel.writeText(text: String) {
+    Channels.newWriter(this, Charsets.UTF_8).run {
+      write(text)
+      flush()
+    }
+  }
+
+  /**
+   * Create and configure a new [HybridVfs] for use in tests, using an embedded bundle from the test resources for
+   * the in-memory layer.
+   *
+   * @see useVfs
+   */
+  private fun acquireVfs(): HybridVfs {
+    val bundles = listOf(HybridVfsTest::class.java.getResource("/sample-vfs.tar")!!.toURI())
+    return HybridVfs.acquire(writable = true, overlay = bundles)
+  }
+
+  /** Convenience method used to [acquire][acquireVfs] a [HybridVfs] instance and use it in a test. */
+  private inline fun useVfs(block: (HybridVfs) -> Unit) {
+    return block(acquireVfs())
+  }
+
+  @Test fun testReadFromEmbeddedBundles(): Unit = useVfs { vfs ->
+    val path = vfs.parsePath("/hello.txt")
+
+    val channel = assertDoesNotThrow("should allow reading known good bundled file") {
+      vfs.newByteChannel(path, mutableSetOf(StandardOpenOption.READ))
+    }
+
+    assertEquals(
+      expected = "hello",
+      actual = channel.readText().trim(),
+      message = "should read file contents from embedded bundle",
+    )
+  }
+
+  @Test fun testReadFromHost() = useVfs { vfs ->
+    val data = "host"
+    val file = tempDirectory.resolve("hello.txt").createFile().apply { writeText(data) }
+
+    val channel = assertDoesNotThrow("should allow reading from host") {
+      vfs.newByteChannel(file, mutableSetOf(StandardOpenOption.READ))
+    }
+
+    assertEquals(
+      expected = data,
+      actual = channel.readText().trim(),
+      message = "should read file contents written to host file",
+    )
+  }
+
+  @Test fun testReadWithGenericPath() = useVfs { vfs ->
+    // points to a file in the embedded bundle, but is constructed using the current
+    // file system provider (not by calling vfs.parsePath)
+    val inMemoryPath = Path.of("/hello.txt")
+    assertDoesNotThrow("should accept generic path object when reading from memory") {
+      vfs.newByteChannel(inMemoryPath, mutableSetOf(StandardOpenOption.READ))
+    }
+
+    // same as the previous case, but using a host path instead (ensure the file exists first)
+    val hostPath = tempDirectory.resolve("hello.txt").createFile()
+    assertDoesNotThrow("should accept generic path object when reading from host") {
+      vfs.newByteChannel(hostPath, mutableSetOf(StandardOpenOption.READ))
+    }
+  }
+
+  @Test fun testWrite() = useVfs { vfs ->
+    val hostPath = tempDirectory.resolve("hello.txt")
+    val data = "Hello"
+
+    // force the creation of a new file, to avoid collisions with other tests
+    val channel = assertDoesNotThrow("should open channel to file in host file system") {
+      vfs.newByteChannel(hostPath, mutableSetOf(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW))
+    }
+
+    assertDoesNotThrow("should allow writing to file in host file system") {
+      channel.writeText(data)
+    }
+
+    assertEquals(
+      expected = data,
+      actual = hostPath.readText(),
+      message = "file should contain the written data",
+    )
+  }
+}