wasi: moves system resource management to the importing module (#401)

The prior design had a problem, where multiple imports of WASI would end
up having different file descriptors for the same file. Moreover, there
was no means to close any of these when a module importing WASI was
closed.

This moves all existing functionality to a new type SystemContext, which
is owned by the importing module, similar to how it owns its memory.
While this PR doesn't fix some problems such as unclosed files, the code
is now organized in a way it can be, and these issues will be resolved
by #394.

In order to fix scope, `WASISnapshotPreview1WithConfig` had to be
removed.

Signed-off-by: Adrian Cole <adrian@tetrate.io>

README.md

@@ -36,7 +36,7 @@ language that compiles to (targets) WebAssembly, such as AssemblyScript, C, C++,
 
 The former example is a pure function. While a good start, you probably are
 wondering how to do something more realistic, like read a file. WebAssembly
-Modules (Wasm) are sand-boxed similar to containers. They can't read anything
+Modules (Wasm) are sandboxed similar to containers. They can't read anything
 on your machine unless you explicitly allow it.
 
 System access is defined by an emerging specification called WebAssembly

examples/file_system_test.go

@@ -31,7 +31,7 @@ func Test_Cat(t *testing.T) {
 	stdoutBuf := bytes.NewBuffer(nil)
 	wasiConfig := wazero.NewWASIConfig().WithStdout(stdoutBuf)
 
-	// Next, configure a sand-boxed filesystem to include one file.
+	// Next, configure a sandboxed filesystem to include one file.
 	file := "cat.go" // arbitrary file
 	memFS := wazero.WASIMemFS()
 	err := writeFile(memFS, file, catGo)
@@ -42,13 +42,18 @@ func Test_Cat(t *testing.T) {
 	// Remember, arg[0] is the program name!
 	wasiConfig.WithArgs("cat", file)
 
-	// Now, instantiate WASI with the above configuration.
-	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1WithConfig(wasiConfig))
+	// Compile the `cat` module.
+	compiled, err := r.CompileModule(catWasm)
+	require.NoError(t, err)
+
+	// Instantiate WASI, which implements system I/O such as console output.
+	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1())
 	require.NoError(t, err)
 	defer wasi.Close()
 
-	// Finally, start the `cat` program's main function (in WASI, this is named `_start`).
-	cat, err := wazero.StartWASICommandFromSource(r, catWasm)
+	// StartWASICommand runs the "_start" function which is what TinyGo compiles "main" to.
+	cat, err := wazero.StartWASICommandWithConfig(r, compiled, wasiConfig)
+
 	require.NoError(t, err)
 	defer cat.Close()
 

examples/host_func_test.go

@@ -59,15 +59,23 @@ func Test_hostFunc(t *testing.T) {
 	_, err := r.NewModuleBuilder("env").ExportFunction("get_random_bytes", getRandomBytes).Instantiate()
 	require.NoError(t, err)
 
-	// Note: host_func.go doesn't directly use WASI, but TinyGo needs to be initialized as a WASI Command.
+	// Compile the `hostFunc` module.
+	compiled, err := r.CompileModule(hostFuncWasm)
+	require.NoError(t, err)
+
+	// Configure stdout (console) to write to a buffer.
 	stdout := bytes.NewBuffer(nil)
-	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1WithConfig(wazero.NewWASIConfig().WithStdout(stdout)))
+	config := wazero.NewWASIConfig().WithStdout(stdout)
+
+	// Instantiate WASI, which implements system I/O such as console output.
+	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1())
 	require.NoError(t, err)
 	defer wasi.Close()
 
-	module, err := wazero.StartWASICommandFromSource(r, hostFuncWasm)
+	// StartWASICommand runs the "_start" function which is what TinyGo compiles "main" to.
+	module, err := wazero.StartWASICommandWithConfig(r, compiled, config)
 	require.NoError(t, err)
-	defer module.Close()
+	defer wasi.Close()
 
 	allocateInWasmBufferFn := module.ExportedFunction("allocate_buffer")
 	require.NotNil(t, allocateInWasmBuffer)

examples/stdio_test.go

@@ -18,20 +18,25 @@ var stdioWasm []byte
 func Test_stdio(t *testing.T) {
 	r := wazero.NewRuntime()
 
+	// Compile the `stdioWasm` module.
+	compiled, err := r.CompileModule(stdioWasm)
+	require.NoError(t, err)
+
+	// Configure standard I/O (ex stdout) to write to buffers instead of no-op.
 	stdinBuf := bytes.NewBuffer([]byte("WASI\n"))
 	stdoutBuf := bytes.NewBuffer(nil)
 	stderrBuf := bytes.NewBuffer(nil)
+	config := wazero.NewWASIConfig().WithStdin(stdinBuf).WithStdout(stdoutBuf).WithStderr(stderrBuf)
 
-	// Configure WASI host functions with the IO buffers
-	wasiConfig := wazero.NewWASIConfig().WithStdin(stdinBuf).WithStdout(stdoutBuf).WithStderr(stderrBuf)
-	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1WithConfig(wasiConfig))
+	// Instantiate WASI, which implements system I/O such as console output.
+	wasi, err := r.InstantiateModule(wazero.WASISnapshotPreview1())
 	require.NoError(t, err)
 	defer wasi.Close()
 
-	// StartWASICommand runs the "_start" function which is what TinyGo compiles "main" to
-	mod, err := wazero.StartWASICommandFromSource(r, stdioWasm)
+	// StartWASICommand runs the "_start" function which is what TinyGo compiles "main" to.
+	module, err := wazero.StartWASICommandWithConfig(r, compiled, config)
 	require.NoError(t, err)
-	defer mod.Close()
+	defer module.Close()
 
 	require.Equal(t, "Hello, WASI!", strings.TrimSpace(stdoutBuf.String()))
 	require.Equal(t, "Error Message", strings.TrimSpace(stderrBuf.String()))

internal/wasi/strings.go → internal/cstring/cstring.go

@@ -1,27 +1,31 @@
-package internalwasi
+// Package cstring is named cstring because null-terminated strings are also known as CString and that avoids using
+// clashing package names like "strings" or a really long one like "null-terminated-strings"
+package cstring
 
 import (
 	"fmt"
 	"unicode/utf8"
 )
 
-// nullTerminatedStrings holds null-terminated strings. It ensures that
+// NullTerminatedStrings holds null-terminated strings. It ensures that
 // its length and total buffer size don't exceed the max of uint32.
-// nullTerminatedStrings are convenience struct for args_get and environ_get. (environ_get is not implemented yet)
+// NullTerminatedStrings are convenience struct for args_get and environ_get. (environ_get is not implemented yet)
 //
 // A Null-terminated string is a byte string with a NULL suffix ("\x00").
 // See https://en.wikipedia.org/wiki/Null-terminated_string
-type nullTerminatedStrings struct {
-	// nullTerminatedValues are null-terminated values with a NULL suffix.
-	nullTerminatedValues [][]byte
-	totalBufSize         uint32
+type NullTerminatedStrings struct {
+	// NullTerminatedValues are null-terminated values with a NULL suffix.
+	NullTerminatedValues [][]byte
+	TotalBufSize         uint32
 }
 
-// newNullTerminatedStrings creates a nullTerminatedStrings from the given string slice. It returns an error
-// if the length or the total buffer size of the result nullTerminatedStrings exceeds the maxBufSize
-func newNullTerminatedStrings(maxBufSize uint32, valName string, vals ...string) (*nullTerminatedStrings, error) {
+var EmptyNullTerminatedStrings = &NullTerminatedStrings{NullTerminatedValues: [][]byte{}}
+
+// NewNullTerminatedStrings creates a NullTerminatedStrings from the given string slice. It returns an error
+// if the length or the total buffer size of the result NullTerminatedStrings exceeds the maxBufSize
+func NewNullTerminatedStrings(maxBufSize uint32, valName string, vals ...string) (*NullTerminatedStrings, error) {
 	if len(vals) == 0 {
-		return &nullTerminatedStrings{nullTerminatedValues: [][]byte{}}, nil
+		return EmptyNullTerminatedStrings, nil
 	}
 	var strings [][]byte // don't pre-allocate as this function is size bound
 	totalBufSize := uint32(0)
@@ -37,5 +41,5 @@ func newNullTerminatedStrings(maxBufSize uint32, valName string, vals ...string)
 		totalBufSize = uint32(nextSize)
 		strings = append(strings, append([]byte(arg), 0))
 	}
-	return &nullTerminatedStrings{nullTerminatedValues: strings, totalBufSize: totalBufSize}, nil
+	return &NullTerminatedStrings{NullTerminatedValues: strings, TotalBufSize: totalBufSize}, nil
 }

internal/wasi/strings_test.go → internal/cstring/cstring_test.go

@@ -1,4 +1,4 @@
-package internalwasi
+package cstring
 
 import (
 	_ "embed"
@@ -8,11 +8,11 @@ import (
 )
 
 func TestNewNullTerminatedStrings(t *testing.T) {
-	emptyWASIStringArray := &nullTerminatedStrings{nullTerminatedValues: [][]byte{}}
+	emptyWASIStringArray := &NullTerminatedStrings{NullTerminatedValues: [][]byte{}}
 	tests := []struct {
 		name     string
 		input    []string
-		expected *nullTerminatedStrings
+		expected *NullTerminatedStrings
 	}{
 		{
 			name:     "nil",
@@ -26,37 +26,37 @@ func TestNewNullTerminatedStrings(t *testing.T) {
 		{
 			name:  "two",
 			input: []string{"a", "bc"},
-			expected: &nullTerminatedStrings{
-				nullTerminatedValues: [][]byte{
+			expected: &NullTerminatedStrings{
+				NullTerminatedValues: [][]byte{
 					{'a', 0},
 					{'b', 'c', 0},
 				},
-				totalBufSize: 5,
+				TotalBufSize: 5,
 			},
 		},
 		{
 			name:  "two and empty string",
 			input: []string{"a", "", "bc"},
-			expected: &nullTerminatedStrings{
-				nullTerminatedValues: [][]byte{
+			expected: &NullTerminatedStrings{
+				NullTerminatedValues: [][]byte{
 					{'a', 0},
 					{0},
 					{'b', 'c', 0},
 				},
-				totalBufSize: 6,
+				TotalBufSize: 6,
 			},
 		},
 		{
 			name: "utf-8",
 			// "😨", "🤣", and "️🏃‍♀️" have 4, 4, and 13 bytes respectively
 			input: []string{"😨🤣🏃\u200d♀️", "foo", "bar"},
-			expected: &nullTerminatedStrings{
-				nullTerminatedValues: [][]byte{
+			expected: &NullTerminatedStrings{
+				NullTerminatedValues: [][]byte{
 					[]byte("😨🤣🏃\u200d♀️\x00"),
 					{'f', 'o', 'o', 0},
 					{'b', 'a', 'r', 0},
 				},
-				totalBufSize: 30,
+				TotalBufSize: 30,
 			},
 		},
 	}
@@ -65,7 +65,7 @@ func TestNewNullTerminatedStrings(t *testing.T) {
 		tc := tt
 
 		t.Run(tc.name, func(t *testing.T) {
-			s, err := newNullTerminatedStrings(100, "", tc.input...)
+			s, err := NewNullTerminatedStrings(100, "", tc.input...)
 			require.NoError(t, err)
 			require.Equal(t, tc.expected, s)
 		})
@@ -74,15 +74,15 @@ func TestNewNullTerminatedStrings(t *testing.T) {
 
 func TestNewNullTerminatedStrings_Errors(t *testing.T) {
 	t.Run("invalid utf-8", func(t *testing.T) {
-		_, err := newNullTerminatedStrings(100, "arg", "\xff\xfe\xfd", "foo", "bar")
+		_, err := NewNullTerminatedStrings(100, "arg", "\xff\xfe\xfd", "foo", "bar")
 		require.EqualError(t, err, "arg[0] is not a valid UTF-8 string")
 	})
 	t.Run("arg[0] too large", func(t *testing.T) {
-		_, err := newNullTerminatedStrings(1, "arg", "a", "bc")
+		_, err := NewNullTerminatedStrings(1, "arg", "a", "bc")
 		require.EqualError(t, err, "arg[0] will exceed max buffer size 1")
 	})
 	t.Run("empty arg too large due to null terminator", func(t *testing.T) {
-		_, err := newNullTerminatedStrings(2, "arg", "a", "", "bc")
+		_, err := NewNullTerminatedStrings(2, "arg", "a", "", "bc")
 		require.EqualError(t, err, "arg[1] will exceed max buffer size 2")
 	})
 }