Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(go): remove experimental FS API usage in Wasm #3299

Merged
merged 3 commits into from
Dec 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions pkg/module/memfs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package module

import (
"io"
"io/fs"
"path/filepath"

"golang.org/x/xerrors"

dio "github.com/aquasecurity/go-dep-parser/pkg/io"
"github.com/aquasecurity/memoryfs"
)

// memFS is a wrapper of memoryfs.FS and can change its underlying file system
// at runtime. This implements fs.FS.
type memFS struct {
current *memoryfs.FS
}

// Open implements fs.FS.
func (m *memFS) Open(name string) (fs.File, error) {
return m.current.Open(name)
}

// initialize changes the underlying memory file system with the given file path and contents.
//
// Note: it is always to safe swap the underlying FS with this API since this is called only at the beginning of
mathetake marked this conversation as resolved.
Show resolved Hide resolved
// Analyze interface call, which is not concurrently called per module instance.
func (m *memFS) initialize(filePath string, content dio.ReadSeekerAt) (err error) {
memfs := memoryfs.New()
if err = memfs.MkdirAll(filepath.Dir(filePath), fs.ModePerm); err != nil {
return xerrors.Errorf("memory fs mkdir error: %w", err)
}
err = memfs.WriteLazyFile(filePath, func() (io.Reader, error) {
return content, nil
}, fs.ModePerm)
if err != nil {
return xerrors.Errorf("memory fs write error: %w", err)
}

m.current = memfs
return
}
33 changes: 33 additions & 0 deletions pkg/module/memfs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package module

import (
"io"
"os"
"strings"
"testing"

"github.com/stretchr/testify/require"
)

func TestMemFS(t *testing.T) {
m := &memFS{}
require.Nil(t, m.current)

const path, content = "/usr/foo/bar.txt", "my-content"
err := m.initialize(path, strings.NewReader(content))
require.NoError(t, err)
require.NotNil(t, m.current)

t.Run("happy", func(t *testing.T) {
f, err := m.Open(path)
require.NoError(t, err)
actual, err := io.ReadAll(f)
require.NoError(t, err)
require.Equal(t, content, string(actual))
})

t.Run("not found", func(t *testing.T) {
_, err = m.Open(path + "tmp")
require.ErrorIs(t, err, os.ErrNotExist)
})
}
34 changes: 14 additions & 20 deletions pkg/module/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,20 @@ package module
import (
"context"
"encoding/json"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"sync"

"github.com/mailru/easyjson"
"github.com/samber/lo"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
wasi "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
"golang.org/x/exp/slices"
"golang.org/x/xerrors"

"github.com/aquasecurity/memoryfs"

"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/log"
tapi "github.com/aquasecurity/trivy/pkg/module/api"
Expand Down Expand Up @@ -217,7 +214,9 @@ func marshal(ctx context.Context, m api.Module, malloc api.Function, v easyjson.
}

type wasmModule struct {
mod api.Module
mod api.Module
memFS *memFS
mux sync.Mutex

name string
version int
Expand All @@ -235,8 +234,8 @@ type wasmModule struct {
}

func newWASMPlugin(ctx context.Context, r wazero.Runtime, code []byte) (*wasmModule, error) {
// Combine the above into our baseline config, overriding defaults (which discard stdout and have no file system).
config := wazero.NewModuleConfig().WithStdout(os.Stdout).WithFS(memoryfs.New())
mf := &memFS{}
config := wazero.NewModuleConfig().WithStdout(os.Stdout).WithFS(mf)

// Create an empty namespace so that multiple modules will not conflict
ns := r.NewNamespace(ctx)
Expand Down Expand Up @@ -334,6 +333,7 @@ func newWASMPlugin(ctx context.Context, r wazero.Runtime, code []byte) (*wasmMod

return &wasmModule{
mod: mod,
memFS: mf,
name: name,
version: version,
requiredFiles: requiredFiles,
Expand Down Expand Up @@ -390,20 +390,14 @@ func (m *wasmModule) Analyze(ctx context.Context, input analyzer.AnalysisInput)
filePath := "/" + filepath.ToSlash(input.FilePath)
log.Logger.Debugf("Module %s: analyzing %s...", m.name, filePath)

memfs := memoryfs.New()
if err := memfs.MkdirAll(filepath.Dir(filePath), fs.ModePerm); err != nil {
return nil, xerrors.Errorf("memory fs mkdir error: %w", err)
}
err := memfs.WriteLazyFile(filePath, func() (io.Reader, error) {
return input.Content, nil
}, fs.ModePerm)
if err != nil {
return nil, xerrors.Errorf("memory fs write error: %w", err)
}
// Wasm module instances are not Goroutine safe, so we take look here since Analyze might be called concurrently.
// TODO: This is temporary solution and we could improve the Analyze performance by having module instance pool.
m.mux.Lock()
defer m.mux.Unlock()

// Pass memory fs to the analyze() function
ctx, closer := experimental.WithFS(ctx, memfs)
defer closer.Close(ctx)
if err := m.memFS.initialize(filePath, input.Content); err != nil {
return nil, err
}

inputPtr, inputSize, err := stringToPtrSize(ctx, filePath, m.mod, m.malloc)
if err != nil {
Expand Down