feat: Add Observe SDK support

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.22'
 
     - name: Build
       run: go build -v ./...

README.md

@@ -268,6 +268,42 @@ config := PluginConfig{
 _, err := NewPlugin(ctx, manifest, config, []HostFunction{})
 ```
 
+### Integrate with Dylibso Observe SDK
+Dylibso provides [observability SDKs](https://github.com/dylibso/observe-sdk) for WebAssembly (Wasm), enabling continuous monitoring of WebAssembly code as it executes within a runtime. It provides developers with the tools necessary to capture and emit telemetry data from Wasm code, including function execution and memory allocation traces, logs, and metrics.
+
+While Observe SDK has adapters for many popular observability platforms, it also ships with an stdout adapter:
+
+```
+ctx := context.Background()
+
+adapter := stdout.NewStdoutAdapter()
+adapter.Start(ctx)
+
+manifest := manifest("nested.c.instr.wasm")
+
+config := PluginConfig{
+    ModuleConfig:   wazero.NewModuleConfig().WithSysWalltime(),
+    EnableWasi:     true,
+    ObserveAdapter: adapter.AdapterBase,
+}
+
+plugin, err := NewPlugin(ctx, manifest, config, []HostFunction{})
+if err != nil {
+    panic(err)
+}
+
+meta := map[string]string{
+    "http.url":         "https://example.com/my-endpoint",
+    "http.status_code": "200",
+    "http.client_ip":   "192.168.1.0",
+}
+
+plugin.TraceCtx.Metadata(meta)
+
+_, _, _ = plugin.Call("_start", []byte("hello world"))
+plugin.Close()
+```
+
 ### Enable filesystem access
 
 WASM plugins can read/write files outside the runtime. To do this we add `AllowedPaths` mapping of "HOST:PLUGIN" to the `extism.Manifest` of our plugin.

extism.go

@@ -15,12 +15,18 @@ import (
 	"strings"
 	"time"
 
+	observe "github.com/dylibso/observe-sdk/go"
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/sys"
 )
 
+type module struct {
+	module api.Module
+	wasm   []byte
+}
+
 //go:embed extism-runtime.wasm
 var extismRuntimeWasm []byte
 
@@ -41,10 +47,12 @@ type Runtime struct {
 
 // PluginConfig contains configuration options for the Extism plugin.
 type PluginConfig struct {
-	ModuleConfig  wazero.ModuleConfig
-	RuntimeConfig wazero.RuntimeConfig
-	EnableWasi    bool
-	LogLevel      LogLevel
+	ModuleConfig   wazero.ModuleConfig
+	RuntimeConfig  wazero.RuntimeConfig
+	EnableWasi     bool
+	LogLevel       LogLevel
+	ObserveAdapter *observe.AdapterBase
+	ObserveOptions *observe.Options
 }
 
 // HttpRequest represents an HTTP request to be made by the plugin.
@@ -87,8 +95,8 @@ func (l LogLevel) String() string {
 // Plugin is used to call WASM functions
 type Plugin struct {
 	Runtime *Runtime
-	Modules map[string]api.Module
-	Main    api.Module
+	Modules map[string]module
+	Main    module
 	Timeout time.Duration
 	Config  map[string]string
 	// NOTE: maybe we can have some nice methods for getting/setting vars
@@ -101,6 +109,8 @@ type Plugin struct {
 	log                  func(LogLevel, string)
 	logLevel             LogLevel
 	guestRuntime         guestRuntime
+	Adapter              *observe.AdapterBase
+	TraceCtx             *observe.TraceCtx
 }
 
 func logStd(level LogLevel, message string) {
@@ -383,7 +393,7 @@ func NewPlugin(
 		return nil, fmt.Errorf("Manifest can't be empty.")
 	}
 
-	modules := map[string]api.Module{}
+	modules := map[string]module{}
 
 	// NOTE: this is only necessary for guest modules because
 	// host modules have the same access privileges as the host itself
@@ -419,6 +429,7 @@ func NewPlugin(
 	//  - If there is only one module in the manifest then that is the main module by default
 	//  - Otherwise the last module listed is the main module
 
+	var trace *observe.TraceCtx
 	for i, wasm := range manifest.Wasm {
 		data, err := wasm.ToWasmData(ctx)
 		if err != nil {
@@ -430,6 +441,15 @@ func NewPlugin(
 			data.Name = "main"
 		}
 
+		if data.Name == "main" && config.ObserveAdapter != nil {
+			trace, err = config.ObserveAdapter.NewTraceCtx(ctx, c.Wazero, data.Data, config.ObserveOptions)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to initialize Observe Adapter: %v", err)
+			}
+
+			trace.Finish()
+		}
+
 		_, okh := hostModules[data.Name]
 		_, okm := modules[data.Name]
 
@@ -449,7 +469,7 @@ func NewPlugin(
 			return nil, err
 		}
 
-		modules[data.Name] = m
+		modules[data.Name] = module{module: m, wasm: data.Data}
 	}
 
 	logLevel := LogLevelWarn
@@ -468,7 +488,7 @@ func NewPlugin(
 		varMax = int64(manifest.Memory.MaxVarBytes)
 	}
 	for _, m := range modules {
-		if m.Name() == "main" {
+		if m.module.Name() == "main" {
 			p := &Plugin{
 				Runtime:              &c,
 				Modules:              modules,
@@ -483,6 +503,8 @@ func NewPlugin(
 				MaxVarBytes:          varMax,
 				log:                  logStd,
 				logLevel:             logLevel,
+				Adapter:              config.ObserveAdapter,
+				TraceCtx:             trace,
 			}
 
 			p.guestRuntime = detectGuestRuntime(ctx, p)
@@ -574,7 +596,7 @@ func (plugin *Plugin) GetErrorWithContext(ctx context.Context) string {
 
 // FunctionExists returns true when the named function is present in the plugin's main module
 func (plugin *Plugin) FunctionExists(name string) bool {
-	return plugin.Main.ExportedFunction(name) != nil
+	return plugin.Main.module.ExportedFunction(name) != nil
 }
 
 // Call a function by name with the given input, returning the output
@@ -599,7 +621,7 @@ func (plugin *Plugin) CallWithContext(ctx context.Context, name string, data []b
 
 	ctx = context.WithValue(ctx, "inputOffset", intputOffset)
 
-	var f = plugin.Main.ExportedFunction(name)
+	var f = plugin.Main.module.ExportedFunction(name)
 
 	if f == nil {
 		return 1, []byte{}, errors.New(fmt.Sprintf("Unknown function: %s", name))
@@ -620,6 +642,10 @@ func (plugin *Plugin) CallWithContext(ctx context.Context, name string, data []b
 
 	res, err := f.Call(ctx)
 
+	if plugin.TraceCtx != nil {
+		defer plugin.TraceCtx.Finish()
+	}
+
 	// Try to extact WASI exit code
 	if exitErr, ok := err.(*sys.ExitError); ok {
 		exitCode := exitErr.ExitCode()

extism_test.go

@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	observe "github.com/dylibso/observe-sdk/go"
+	"github.com/dylibso/observe-sdk/go/adapter/stdout"
 	"github.com/stretchr/testify/assert"
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/experimental"
@@ -763,6 +765,79 @@ func TestInputOffset(t *testing.T) {
 	}
 }
 
+func TestObserve(t *testing.T) {
+	ctx := context.Background()
+
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+
+	adapter := stdout.NewStdoutAdapter()
+	adapter.Start(ctx)
+
+	manifest := manifest("nested.c.instr.wasm")
+
+	config := PluginConfig{
+		ModuleConfig:   wazero.NewModuleConfig().WithSysWalltime(),
+		EnableWasi:     true,
+		ObserveAdapter: adapter.AdapterBase,
+		ObserveOptions: &observe.Options{
+			SpanFilter:        &observe.SpanFilter{MinDuration: 1 * time.Nanosecond},
+			ChannelBufferSize: 1024,
+		},
+	}
+
+	// Plugin 1
+	plugin, err := NewPlugin(ctx, manifest, config, []HostFunction{})
+	if err != nil {
+		panic(err)
+	}
+
+	meta := map[string]string{
+		"http.url":         "https://example.com/my-endpoint",
+		"http.status_code": "200",
+		"http.client_ip":   "192.168.1.0",
+	}
+
+	plugin.TraceCtx.Metadata(meta)
+
+	_, _, _ = plugin.Call("_start", []byte("hello world"))
+	plugin.Close()
+
+	// HACK: make sure we give enough time for the events to get flushed
+	time.Sleep(100 * time.Millisecond)
+
+	actual := buf.String()
+	assert.Contains(t, actual, "  Call to _start took")
+	assert.Contains(t, actual, "      Call to main took")
+	assert.Contains(t, actual, "        Call to one took")
+	assert.Contains(t, actual, "          Call to two took")
+	assert.Contains(t, actual, "            Call to three took")
+	assert.Contains(t, actual, "              Call to printf took")
+
+	// Reset underlying buffer
+	buf.Reset()
+
+	// Plugin 2
+	plugin2, err := NewPlugin(ctx, manifest, config, []HostFunction{})
+	if err != nil {
+		panic(err)
+	}
+
+	_, _, _ = plugin2.Call("_start", []byte("hello world"))
+	plugin2.Close()
+
+	// HACK: make sure we give enough time for the events to get flushed
+	time.Sleep(100 * time.Millisecond)
+
+	actual2 := buf.String()
+	assert.Contains(t, actual2, "  Call to _start took")
+	assert.Contains(t, actual2, "      Call to main took")
+	assert.Contains(t, actual2, "        Call to one took")
+	assert.Contains(t, actual2, "          Call to two took")
+	assert.Contains(t, actual2, "            Call to three took")
+	assert.Contains(t, actual2, "              Call to printf took")
+}
+
 // make sure cancelling the context given to NewPlugin doesn't affect plugin calls
 func TestContextCancel(t *testing.T) {
 	manifest := manifest("sleep.wasm")

go.mod

@@ -1,15 +1,20 @@
 module github.com/extism/go-sdk
 
-go 1.20
+go 1.22
 
 require (
+	github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a
 	github.com/gobwas/glob v0.2.3
-	github.com/stretchr/testify v1.8.4
-	github.com/tetratelabs/wazero v1.7.3
+	github.com/stretchr/testify v1.9.0
+	github.com/tetratelabs/wazero v1.8.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,13 +1,29 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dylibso/observe-sdk/go v0.0.0-20240815143955-7e0389165219 h1:ISvdktS6sAspgbQ15M/eagCWo8TqOTRB5SlnH5BlPWQ=
+github.com/dylibso/observe-sdk/go v0.0.0-20240815143955-7e0389165219/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q=
+github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a h1:UwSIFv5g5lIvbGgtf3tVwC7Ky9rmMFBp0RMs+6f6YqE=
+github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca h1:T54Ema1DU8ngI+aef9ZhAhNGQhcRTrWxVeG07F+c/Rw=
+github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw=
-github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q=
+github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834/go.mod h1:m9ymHTgNSEjuxvw8E7WWe4Pl4hZQHXONY8wE6dMLaRk=
+github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
+github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
+go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
+go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

runtime.go

@@ -25,12 +25,12 @@ type guestRuntime struct {
 func detectGuestRuntime(ctx context.Context, p *Plugin) guestRuntime {
 	m := p.Main
 
-	runtime, ok := haskellRuntime(ctx, p, m)
+	runtime, ok := haskellRuntime(ctx, p, m.module)
 	if ok {
 		return runtime
 	}
 
-	runtime, ok = wasiRuntime(ctx, p, m)
+	runtime, ok = wasiRuntime(ctx, p, m.module)
 	if ok {
 		return runtime
 	}