Add support for virtual files (eg, not backed by an OS file).

Virtual files are implemented through trait objects, with a default
implementation that tries to behave like on-disk files, but entirely
backed by in-memory structures.

.github/workflows/main.yml

@@ -181,6 +181,7 @@ jobs:
     # Build and test all features except for lightbeam
     - run: cargo test --features test_programs --all --exclude lightbeam -- --nocapture
       env:
+        RUST_LOG: "wasi_common=trace"
         RUST_BACKTRACE: 1
         RUSTFLAGS: "-D warnings"
 

crates/wasi-common/src/ctx.rs

@@ -1,4 +1,6 @@
-use crate::fdentry::FdEntry;
+use crate::fdentry::{Descriptor, FdEntry};
+use crate::sys::fdentry_impl::OsHandle;
+use crate::virtfs::VirtualFile;
 use crate::{wasi, Error, Result};
 use std::borrow::Borrow;
 use std::collections::HashMap;
@@ -61,7 +63,7 @@ impl PendingCString {
 /// A builder allowing customizable construction of `WasiCtx` instances.
 pub struct WasiCtxBuilder {
     fds: HashMap<wasi::__wasi_fd_t, PendingFdEntry>,
-    preopens: Vec<(PathBuf, File)>,
+    preopens: Vec<(PathBuf, Descriptor)>,
     args: Vec<PendingCString>,
     env: HashMap<PendingCString, PendingCString>,
 }
@@ -182,9 +184,23 @@ impl WasiCtxBuilder {
         self
     }
 
-    /// Add a preopened directory.
+    /// Add a preopened OS directory.
     pub fn preopened_dir<P: AsRef<Path>>(mut self, dir: File, guest_path: P) -> Self {
-        self.preopens.push((guest_path.as_ref().to_owned(), dir));
+        self.preopens.push((
+            guest_path.as_ref().to_owned(),
+            Descriptor::OsHandle(OsHandle::from(dir)),
+        ));
+        self
+    }
+
+    /// Add a preopened virtual directory.
+    pub fn preopened_virt<P: AsRef<Path>>(
+        mut self,
+        dir: Box<dyn VirtualFile>,
+        guest_path: P,
+    ) -> Self {
+        self.preopens
+            .push((guest_path.as_ref().to_owned(), Descriptor::VirtualFile(dir)));
         self
     }
 
@@ -226,7 +242,7 @@ impl WasiCtxBuilder {
                     fds.insert(fd, f()?);
                 }
                 PendingFdEntry::File(f) => {
-                    fds.insert(fd, FdEntry::from(f)?);
+                    fds.insert(fd, FdEntry::from(Descriptor::OsHandle(OsHandle::from(f)))?);
                 }
             }
         }
@@ -239,8 +255,20 @@ impl WasiCtxBuilder {
             // unnecessarily if we have exactly the maximum number of file descriptors.
             preopen_fd = preopen_fd.checked_add(1).ok_or(Error::ENFILE)?;
 
-            if !dir.metadata()?.is_dir() {
-                return Err(Error::EBADF);
+            match &dir {
+                Descriptor::OsHandle(handle) => {
+                    if !handle.metadata()?.is_dir() {
+                        return Err(Error::EBADF);
+                    }
+                }
+                Descriptor::VirtualFile(virt) => {
+                    if virt.get_file_type() != wasi::__WASI_FILETYPE_DIRECTORY {
+                        return Err(Error::EBADF);
+                    }
+                }
+                Descriptor::Stdin | Descriptor::Stdout | Descriptor::Stderr => {
+                    panic!("implementation error, stdin/stdout/stderr shouldn't be in the list of preopens");
+                }
             }
 
             // We don't currently allow setting file descriptors other than 0-2, but this will avoid

crates/wasi-common/src/fdentry.rs

@@ -2,40 +2,118 @@ use crate::sys::dev_null;
 use crate::sys::fdentry_impl::{
     descriptor_as_oshandle, determine_type_and_access_rights, OsHandle,
 };
+use crate::virtfs::VirtualFile;
 use crate::{wasi, Error, Result};
 use std::marker::PhantomData;
 use std::mem::ManuallyDrop;
 use std::ops::{Deref, DerefMut};
 use std::path::PathBuf;
-use std::{fs, io};
+use std::{fmt, fs, io};
+
+pub(crate) enum HandleMut<'handle> {
+    OsHandle(OsHandleRef<'handle>),
+    VirtualFile(&'handle mut dyn VirtualFile),
+}
+
+impl<'descriptor> fmt::Debug for HandleMut<'descriptor> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            HandleMut::OsHandle(file) => {
+                // coerce to the target debug-printable type
+                let file: &fs::File = file;
+                write!(f, "{:?}", file)
+            }
+            HandleMut::VirtualFile(_) => write!(f, "VirtualFile"),
+        }
+    }
+}
+
+pub(crate) enum Handle<'handle> {
+    OsHandle(OsHandleRef<'handle>),
+    VirtualFile(&'handle dyn VirtualFile),
+}
+
+impl<'descriptor> Handle<'descriptor> {
+    pub(crate) fn try_clone(&self) -> io::Result<Descriptor> {
+        match self {
+            Handle::OsHandle(file) => file
+                .try_clone()
+                .map(|f| Descriptor::OsHandle(OsHandle::from(f))),
+            Handle::VirtualFile(virt) => virt.try_clone().map(|f| Descriptor::VirtualFile(f)),
+        }
+    }
+}
+
+impl<'descriptor> fmt::Debug for Handle<'descriptor> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Handle::OsHandle(file) => {
+                // coerce to the target debug-printable type
+                let file: &fs::File = file;
+                write!(f, "{:?}", file)
+            }
+            Handle::VirtualFile(_) => write!(f, "VirtualFile"),
+        }
+    }
+}
 
-#[derive(Debug)]
 pub(crate) enum Descriptor {
     OsHandle(OsHandle),
+    VirtualFile(Box<dyn VirtualFile>),
     Stdin,
     Stdout,
     Stderr,
 }
 
+impl fmt::Debug for Descriptor {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Descriptor::OsHandle(handle) => write!(f, "{:?}", handle),
+            Descriptor::VirtualFile(_) => write!(f, "VirtualFile"),
+            Descriptor::Stdin => write!(f, "Stdin"),
+            Descriptor::Stdout => write!(f, "Stdout"),
+            Descriptor::Stderr => write!(f, "Stderr"),
+        }
+    }
+}
+
 impl Descriptor {
-    /// Return a reference to the `OsHandle` treating it as an actual file/dir, and
-    /// allowing operations which require an actual file and not just a stream or
-    /// socket file descriptor.
-    pub(crate) fn as_file(&self) -> Result<&OsHandle> {
+    /// Return a reference to the `OsHandle` or `VirtualFile` treating it as an
+    /// actual file/dir, and allowing operations which require an actual file and
+    /// not just a stream or socket file descriptor.
+    pub(crate) fn as_file<'descriptor>(&'descriptor self) -> Result<Handle<'descriptor>> {
         match self {
-            Self::OsHandle(file) => Ok(file),
+            Self::OsHandle(_) => Ok(Handle::OsHandle(descriptor_as_oshandle(self))),
+            Self::VirtualFile(virt) => Ok(Handle::VirtualFile(virt.as_ref())),
             _ => Err(Error::EBADF),
         }
     }
 
     /// Like `as_file`, but return a mutable reference.
-    pub(crate) fn as_file_mut(&mut self) -> Result<&mut OsHandle> {
+    pub(crate) fn as_file_mut<'descriptor>(
+        &'descriptor mut self,
+    ) -> Result<HandleMut<'descriptor>> {
         match self {
-            Self::OsHandle(file) => Ok(file),
+            Self::OsHandle(_) => Ok(HandleMut::OsHandle(descriptor_as_oshandle(self))),
+            Self::VirtualFile(virt) => Ok(HandleMut::VirtualFile(virt.as_mut())),
             _ => Err(Error::EBADF),
         }
     }
 
+    pub(crate) fn as_handle<'descriptor>(&'descriptor self) -> Handle<'descriptor> {
+        match self {
+            Self::VirtualFile(virt) => Handle::VirtualFile(virt.as_ref()),
+            other => Handle::OsHandle(other.as_os_handle()),
+        }
+    }
+
+    pub(crate) fn as_handle_mut<'descriptor>(&'descriptor mut self) -> HandleMut<'descriptor> {
+        match self {
+            Self::VirtualFile(virt) => HandleMut::VirtualFile(virt.as_mut()),
+            other => HandleMut::OsHandle(other.as_os_handle()),
+        }
+    }
+
     /// Return an `OsHandle`, which may be a stream or socket file descriptor.
     pub(crate) fn as_os_handle<'descriptor>(&'descriptor self) -> OsHandleRef<'descriptor> {
         descriptor_as_oshandle(self)
@@ -61,16 +139,33 @@ pub(crate) struct FdEntry {
 }
 
 impl FdEntry {
-    pub(crate) fn from(file: fs::File) -> Result<Self> {
-        unsafe { determine_type_and_access_rights(&file) }.map(
-            |(file_type, rights_base, rights_inheriting)| Self {
-                file_type,
-                descriptor: Descriptor::OsHandle(OsHandle::from(file)),
-                rights_base,
-                rights_inheriting,
-                preopen_path: None,
-            },
-        )
+    pub(crate) fn from(file: Descriptor) -> Result<Self> {
+        match file {
+            Descriptor::OsHandle(handle) => unsafe { determine_type_and_access_rights(&handle) }
+                .map(|(file_type, rights_base, rights_inheriting)| Self {
+                    file_type,
+                    descriptor: Descriptor::OsHandle(handle),
+                    rights_base,
+                    rights_inheriting,
+                    preopen_path: None,
+                }),
+            Descriptor::VirtualFile(virt) => {
+                let file_type = virt.get_file_type();
+                let rights_base = virt.get_rights_base();
+                let rights_inheriting = virt.get_rights_inheriting();
+
+                Ok(Self {
+                    file_type,
+                    descriptor: Descriptor::VirtualFile(virt),
+                    rights_base,
+                    rights_inheriting,
+                    preopen_path: None,
+                })
+            }
+            Descriptor::Stdin | Descriptor::Stdout | Descriptor::Stderr => {
+                panic!("implementation error, stdin/stdout/stderr FdEntry must not be constructed from FdEntry::from");
+            }
+        }
     }
 
     pub(crate) fn duplicate_stdin() -> Result<Self> {
@@ -110,7 +205,7 @@ impl FdEntry {
     }
 
     pub(crate) fn null() -> Result<Self> {
-        Self::from(dev_null()?)
+        Self::from(Descriptor::OsHandle(OsHandle::from(dev_null()?)))
     }
 
     /// Convert this `FdEntry` into a host `Descriptor` object provided the specified
@@ -201,6 +296,15 @@ impl<'descriptor> OsHandleRef<'descriptor> {
             _ref: PhantomData,
         }
     }
+
+    #[allow(dead_code)]
+    pub(crate) fn handle(&self) -> &OsHandle {
+        &self.handle
+    }
+
+    pub(crate) fn handle_mut(&mut self) -> &mut OsHandle {
+        &mut self.handle
+    }
 }
 
 impl<'descriptor> Deref for OsHandleRef<'descriptor> {

crates/wasi-common/src/host.rs

@@ -24,7 +24,7 @@ pub(crate) unsafe fn iovec_to_host_mut(iovec: &mut __wasi_iovec_t) -> io::IoSlic
 #[allow(dead_code)] // trouble with sockets
 #[derive(Clone, Copy, Debug)]
 #[repr(u8)]
-pub(crate) enum FileType {
+pub enum FileType {
     Unknown = __WASI_FILETYPE_UNKNOWN,
     BlockDevice = __WASI_FILETYPE_BLOCK_DEVICE,
     CharacterDevice = __WASI_FILETYPE_CHARACTER_DEVICE,
@@ -42,7 +42,7 @@ impl FileType {
 }
 
 #[derive(Debug, Clone)]
-pub(crate) struct Dirent {
+pub struct Dirent {
     pub name: String,
     pub ftype: FileType,
     pub ino: u64,