From 3ffaa1d702f658f440b1d687aef0264c747ab282 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Thu, 22 Aug 2019 18:09:49 -0700 Subject: [PATCH 01/30] Upstream WebAssembly VM and Null VM from envoyproxy/envoy-wasm. Signed-off-by: John Plevyak --- CODEOWNERS | 2 + source/common/common/logger.h | 3 +- source/extensions/common/wasm/BUILD | 35 +++ source/extensions/common/wasm/null/BUILD | 47 +++++ source/extensions/common/wasm/null/null.cc | 27 +++ source/extensions/common/wasm/null/null.h | 20 ++ source/extensions/common/wasm/null/null_vm.cc | 85 ++++++++ source/extensions/common/wasm/null/null_vm.h | 72 +++++++ .../common/wasm/null/null_vm_plugin.h | 49 +++++ source/extensions/common/wasm/wasm_vm.cc | 31 +++ source/extensions/common/wasm/wasm_vm.h | 199 ++++++++++++++++++ .../extensions/common/wasm/well_known_names.h | 26 +++ test/extensions/common/wasm/BUILD | 19 ++ test/extensions/common/wasm/wasm_vm_test.cc | 96 +++++++++ 14 files changed, 710 insertions(+), 1 deletion(-) create mode 100644 source/extensions/common/wasm/BUILD create mode 100644 source/extensions/common/wasm/null/BUILD create mode 100644 source/extensions/common/wasm/null/null.cc create mode 100644 source/extensions/common/wasm/null/null.h create mode 100644 source/extensions/common/wasm/null/null_vm.cc create mode 100644 source/extensions/common/wasm/null/null_vm.h create mode 100644 source/extensions/common/wasm/null/null_vm_plugin.h create mode 100644 source/extensions/common/wasm/wasm_vm.cc create mode 100644 source/extensions/common/wasm/wasm_vm.h create mode 100644 source/extensions/common/wasm/well_known_names.h create mode 100644 test/extensions/common/wasm/BUILD create mode 100644 test/extensions/common/wasm/wasm_vm_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 7c3617ef6292..74b909209d91 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -43,6 +43,8 @@ extensions/filters/common/original_src @snowp @klarose # dynamic forward proxy /*/extensions/clusters/dynamic_forward_proxy @mattklein123 @alyssawilk /*/extensions/common/dynamic_forward_proxy @mattklein123 @alyssawilk +# wasm common extension +/*/extensions/common/wasm @jplevayk @PiotrSikora /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate /*/extensions/retry/host/omit_canary_hosts @sriduth @snowp diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 42360c572b39..a443ce11ab5e 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -60,7 +60,8 @@ namespace Logger { FUNCTION(thrift) \ FUNCTION(tracing) \ FUNCTION(upstream) \ - FUNCTION(udp) + FUNCTION(udp) \ + FUNCTION(wasm) enum class Id { ALL_LOGGER_IDS(GENERATE_ENUM) diff --git a/source/extensions/common/wasm/BUILD b/source/extensions/common/wasm/BUILD new file mode 100644 index 000000000000..ab972618e6ec --- /dev/null +++ b/source/extensions/common/wasm/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "well_known_names", + hdrs = ["well_known_names.h"], + deps = [ + "//source/common/singleton:const_singleton", + ], +) + +envoy_cc_library( + name = "wasm_vm_interface", + hdrs = ["wasm_vm.h"], + deps = [ + ":well_known_names", + ], +) + +envoy_cc_library( + name = "wasm_vm_lib", + srcs = ["wasm_vm.cc"], + deps = [ + ":wasm_vm_interface", + "//source/common/common:assert_lib", + "//source/extensions/common/wasm/null:null_lib", + ], +) diff --git a/source/extensions/common/wasm/null/BUILD b/source/extensions/common/wasm/null/BUILD new file mode 100644 index 000000000000..eed8e62d2e49 --- /dev/null +++ b/source/extensions/common/wasm/null/BUILD @@ -0,0 +1,47 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "null_vm_plugin_interface", + hdrs = ["null_vm_plugin.h"], + deps = [ + "//source/extensions/common/wasm:wasm_vm_interface", + "//source/extensions/common/wasm:well_known_names", + ], +) + +envoy_cc_library( + name = "null_vm_lib", + srcs = ["null_vm.cc"], + hdrs = ["null_vm.h"], + deps = [ + ":null_vm_plugin_interface", + "//external:abseil_node_hash_map", + "//include/envoy/registry", + "//source/common/common:assert_lib", + "//source/extensions/common/wasm:wasm_vm_interface", + "//source/extensions/common/wasm:well_known_names", + ], +) + +envoy_cc_library( + name = "null_lib", + srcs = ["null.cc"], + hdrs = ["null.h"], + deps = [ + ":null_vm_lib", + ":null_vm_plugin_interface", + "//external:abseil_node_hash_map", + "//include/envoy/registry", + "//source/common/common:assert_lib", + "//source/extensions/common/wasm:wasm_vm_interface", + "//source/extensions/common/wasm:well_known_names", + ], +) diff --git a/source/extensions/common/wasm/null/null.cc b/source/extensions/common/wasm/null/null.cc new file mode 100644 index 000000000000..06b439e85a42 --- /dev/null +++ b/source/extensions/common/wasm/null/null.cc @@ -0,0 +1,27 @@ +#include "extensions/common/wasm/null/null.h" + +#include +#include +#include + +#include "envoy/registry/registry.h" + +#include "common/common/assert.h" + +#include "extensions/common/wasm/null/null_vm.h" +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +std::unique_ptr createVm() { return std::make_unique(); } + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null.h b/source/extensions/common/wasm/null/null.h new file mode 100644 index 000000000000..1fe697890812 --- /dev/null +++ b/source/extensions/common/wasm/null/null.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/wasm_vm.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +std::unique_ptr createVm(); + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc new file mode 100644 index 000000000000..4c9faa117ac4 --- /dev/null +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -0,0 +1,85 @@ +#include "extensions/common/wasm/null/null_vm.h" + +#include +#include +#include + +#include "envoy/registry/registry.h" + +#include "common/common/assert.h" + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +std::unique_ptr NullVm::clone() { return std::make_unique(*this); } + +bool NullVm::load(const std::string& name, bool /* allow_precompiled */) { + auto factory = Registry::FactoryRegistry::getFactory(name); + if (!factory) { + return false; + } + plugin_name_ = name; + plugin_ = factory->create(); + return true; +} + +void NullVm::link(absl::string_view /* name */, bool /* needs_emscripten */) {} + +void NullVm::makeModule(absl::string_view /* name */) { + // NullVm does not advertize code as emscripten so this will not get called. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void NullVm::start(Common::Wasm::Context* context) { + SaveRestoreContext saved_context(context); + plugin_->start(); +} + +uint64_t NullVm::getMemorySize() { return std::numeric_limits::max(); } + +absl::optional NullVm::getMemory(uint64_t pointer, uint64_t size) { + if (pointer == 0 && size != 0) { + return absl::nullopt; + } + return absl::string_view(reinterpret_cast(pointer), static_cast(size)); +} + +bool NullVm::getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) { + *vm_pointer = reinterpret_cast(host_pointer); + return true; +} + +bool NullVm::setMemory(uint64_t pointer, uint64_t size, const void* data) { + if ((pointer == 0 || data == nullptr) && size != 0) { + return false; + } + auto p = reinterpret_cast(pointer); + memcpy(p, data, size); + return true; +} + +bool NullVm::setWord(uint64_t pointer, Word data) { + if (pointer == 0) { + return false; + } + auto p = reinterpret_cast(pointer); + memcpy(p, &data.u64, sizeof(data.u64)); + return true; +} + +absl::string_view NullVm::getUserSection(absl::string_view /* name */) { + // Return nothing: there is no WASM file. + return {}; +} + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h new file mode 100644 index 000000000000..1f02082674bb --- /dev/null +++ b/source/extensions/common/wasm/null/null_vm.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include + +#include "envoy/registry/registry.h" + +#include "common/common/assert.h" + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +struct NullVm : public WasmVm { + NullVm() = default; + NullVm(const NullVm& other) { load(other.plugin_name_, false /* unused */); } + ~NullVm() override{}; + + // WasmVm + absl::string_view vm() override { return WasmVmNames::get().Null; } + bool clonable() override { return true; }; + std::unique_ptr clone() override; + bool load(const std::string& code, bool allow_precompiled) override; + void link(absl::string_view debug_name, bool needs_emscripten) override; + void setMemoryLayout(uint64_t, uint64_t, uint64_t) override {} + void start(Common::Wasm::Context* context) override; + uint64_t getMemorySize() override; + absl::optional getMemory(uint64_t pointer, uint64_t size) override; + bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) override; + bool setMemory(uint64_t pointer, uint64_t size, const void* data) override; + bool setWord(uint64_t pointer, Word data) override; + void makeModule(absl::string_view name) override; + absl::string_view getUserSection(absl::string_view name) override; + +#define _FORWARD_GET_FUNCTION(_T) \ + void getFunction(absl::string_view functionName, _T* f) override { \ + plugin_->getFunction(functionName, f); \ + } + FOR_ALL_WASM_VM_EXPORTS(_FORWARD_GET_FUNCTION) +#undef _FORWARD_GET_FUNCTION + + // These are noops for NullVm. +#define _REGISTER_CALLBACK(_type) \ + void registerCallback(absl::string_view, absl::string_view, _type, \ + typename ConvertFunctionTypeWordToUint32<_type>::type) override{}; + FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) +#undef _REGISTER_CALLBACK + + // NullVm does not advertize code as emscripten so this will not get called. + std::unique_ptr> makeGlobal(absl::string_view, absl::string_view, + double) override { + NOT_REACHED_GCOVR_EXCL_LINE; + }; + std::unique_ptr> makeGlobal(absl::string_view, absl::string_view, Word) override { + NOT_REACHED_GCOVR_EXCL_LINE; + }; + + std::string plugin_name_; + std::unique_ptr plugin_; +}; + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null_vm_plugin.h b/source/extensions/common/wasm/null/null_vm_plugin.h new file mode 100644 index 000000000000..2a8767991ec1 --- /dev/null +++ b/source/extensions/common/wasm/null/null_vm_plugin.h @@ -0,0 +1,49 @@ +#pragma once + +#include "extensions/common/wasm/wasm_vm.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +class NullVmPlugin { +public: + NullVmPlugin() {} + virtual ~NullVmPlugin() {} + + // NB: These are defined rather than declared PURE because gmock uses __LINE__ internally for + // uniqueness, making it impossible to use FOR_ALL_WASM_VM_EXPORTS with MOCK_METHOD2. +#define _DEFINE_GET_FUNCTION(_t) \ + virtual void getFunction(absl::string_view, _t* f) { *f = nullptr; } + FOR_ALL_WASM_VM_EXPORTS(_DEFINE_GET_FUNCTION) +#undef _DEFIN_GET_FUNCTIONE + + virtual void start() PURE; +}; + +/** + * Pseudo-WASM plugins using the NullVM should implement this factory and register via + * Registry::registerFactory or the convenience class RegisterFactory. + */ +class NullVmPluginFactory { +public: + virtual ~NullVmPluginFactory() {} + + /** + * Name of the plugin. + */ + virtual const std::string name() const PURE; + + /** + * Create an instance of the plugin. + */ + virtual std::unique_ptr create() const PURE; +}; + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/wasm_vm.cc b/source/extensions/common/wasm/wasm_vm.cc new file mode 100644 index 000000000000..12966f6525df --- /dev/null +++ b/source/extensions/common/wasm/wasm_vm.cc @@ -0,0 +1,31 @@ +#include "extensions/common/wasm/wasm_vm.h" + +#include + +#include "extensions/common/wasm/null/null.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { + +thread_local Envoy::Extensions::Common::Wasm::Context* current_context_ = nullptr; +thread_local uint32_t effective_context_id_ = 0; + +std::unique_ptr createWasmVm(absl::string_view wasm_vm) { + if (wasm_vm.empty()) { + throw WasmException("Failed to create WASM VM with unspecified runtime."); + } else if (wasm_vm == WasmVmNames::get().Null) { + return Null::createVm(); + } else { + throw WasmException(fmt::format( + "Failed to create WASM VM using {} runtime. Envoy was compiled without support for it.", + wasm_vm)); + } +} + +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h new file mode 100644 index 000000000000..f6d93333536a --- /dev/null +++ b/source/extensions/common/wasm/wasm_vm.h @@ -0,0 +1,199 @@ +#pragma once + +#include + +#include "envoy/common/exception.h" + +#include "common/common/logger.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { + +class Context; + +// Represents a WASM-native word-sized datum. On 32-bit VMs, the high bits are always zero. +// The WASM/VM API treats all bits as significant. +struct Word { + Word(uint64_t w) : u64(w) {} // Implicit conversion into Word. + operator uint64_t() const { return u64; } // Implicit conversion into uint64_t. + // Note: no implicit conversion to uint32_t as it is lossy. + uint32_t u32() const { return static_cast(u64); } + uint64_t u64; +}; + +template struct ConvertWordTypeToUint32 { using type = T; }; +template <> struct ConvertWordTypeToUint32 { using type = uint32_t; }; + +template struct ConvertFunctionTypeWordToUint32 {}; +template struct ConvertFunctionTypeWordToUint32 { + using type = typename ConvertWordTypeToUint32::type (*)( + typename ConvertWordTypeToUint32::type...); +}; + +template struct Global { + virtual ~Global() {} + virtual T get() PURE; + virtual void set(const T& t) PURE; +}; + +// Calls into the WASM VM. +// 1st arg is always a pointer to Context (Context*). +using WasmCall0Void = std::function; +using WasmCall1Void = std::function; +using WasmCall2Void = std::function; +using WasmCall3Void = std::function; +using WasmCall4Void = std::function; +using WasmCall5Void = std::function; +using WasmCall6Void = std::function; +using WasmCall7Void = std::function; +using WasmCall8Void = std::function; +using WasmCall0Word = std::function; +using WasmCall1Word = std::function; +using WasmCall2Word = std::function; +using WasmCall3Word = std::function; +using WasmCall4Word = std::function; +using WasmCall5Word = std::function; +using WasmCall6Word = std::function; +using WasmCall7Word = std::function; +using WasmCall8Word = std::function; +#define FOR_ALL_WASM_VM_EXPORTS(_f) \ + _f(WasmCall0Void) _f(WasmCall1Void) _f(WasmCall2Void) _f(WasmCall3Void) _f(WasmCall4Void) \ + _f(WasmCall5Void) _f(WasmCall8Void) _f(WasmCall0Word) _f(WasmCall1Word) _f(WasmCall3Word) + +// Calls out of the WASM VM. +// 1st arg is always a pointer to raw_context (void*). +using WasmCallback0Void = void (*)(void*); +using WasmCallback1Void = void (*)(void*, Word); +using WasmCallback2Void = void (*)(void*, Word, Word); +using WasmCallback3Void = void (*)(void*, Word, Word, Word); +using WasmCallback4Void = void (*)(void*, Word, Word, Word, Word); +using WasmCallback5Void = void (*)(void*, Word, Word, Word, Word, Word); +using WasmCallback6Void = void (*)(void*, Word, Word, Word, Word, Word, Word); +using WasmCallback7Void = void (*)(void*, Word, Word, Word, Word, Word, Word, Word); +using WasmCallback8Void = void (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word); +using WasmCallback0Word = Word (*)(void*); +using WasmCallback1Word = Word (*)(void*, Word); +using WasmCallback2Word = Word (*)(void*, Word, Word); +using WasmCallback3Word = Word (*)(void*, Word, Word, Word); +using WasmCallback4Word = Word (*)(void*, Word, Word, Word, Word); +using WasmCallback5Word = Word (*)(void*, Word, Word, Word, Word, Word); +using WasmCallback6Word = Word (*)(void*, Word, Word, Word, Word, Word, Word); +using WasmCallback7Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word); +using WasmCallback8Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word, Word); +using WasmCallback9Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word, Word, + Word); +#define FOR_ALL_WASM_VM_IMPORTS(_f) \ + _f(WasmCallback0Void) _f(WasmCallback1Void) _f(WasmCallback2Void) _f(WasmCallback3Void) \ + _f(WasmCallback4Void) _f(WasmCallback0Word) _f(WasmCallback1Word) _f(WasmCallback2Word) \ + _f(WasmCallback3Word) _f(WasmCallback4Word) _f(WasmCallback5Word) _f(WasmCallback6Word) \ + _f(WasmCallback7Word) _f(WasmCallback8Word) _f(WasmCallback9Word) \ + _f(WasmCallback_WWl) _f(WasmCallback_WWm) + +// Using the standard g++/clang mangling algorithm: +// https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling-builtin +// Extended with W = Word +// Z = void, j = uint32_t, l = int64_t, m = uint64_t +using WasmCallback_WWl = Word (*)(void*, Word, int64_t); +using WasmCallback_WWm = Word (*)(void*, Word, uint64_t); + +// Wasm VM instance. Provides the low level WASM interface. +class WasmVm : public Logger::Loggable { +public: + virtual ~WasmVm() {} + virtual absl::string_view vm() PURE; + + // Whether or not the VM implementation supports cloning. + virtual bool clonable() PURE; + // Make a thread-specific copy. This may not be supported by the underlying VM system in which + // case it will return nullptr and the caller will need to create a new VM from scratch. + virtual std::unique_ptr clone() PURE; + + // Load the WASM code from a file. Return true on success. + virtual bool load(const std::string& code, bool allow_precompiled) PURE; + // Link to registered function. + virtual void link(absl::string_view debug_name, bool needs_emscripten) PURE; + + // Set memory layout (start of dynamic heap base, etc.) in the VM. + virtual void setMemoryLayout(uint64_t stack_base, uint64_t heap_base, + uint64_t heap_base_pointer) PURE; + + // Call the 'start' function and initialize globals. + virtual void start(Context*) PURE; + + // Get size of the currently allocated memory in the VM. + virtual uint64_t getMemorySize() PURE; + // Convert a block of memory in the VM to a string_view. Returns 'false' in second if the + // pointer/size is invalid. + virtual absl::optional getMemory(uint64_t pointer, uint64_t size) PURE; + // Convert a host pointer to memory in the VM into a VM "pointer" (an offset into the Memory). + virtual bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) PURE; + // Set a block of memory in the VM, returns true on success, false if the pointer/size is invalid. + virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; + // Set a Word in the VM, returns true on success, false if the pointer is invalid. + virtual bool setWord(uint64_t pointer, Word data) PURE; + // Make a new intrinsic module (e.g. for Emscripten support). + virtual void makeModule(absl::string_view name) PURE; + + // Get the contents of the user section with the given name or "" if it does not exist. + virtual absl::string_view getUserSection(absl::string_view name) PURE; + + // Get typed function exported by the WASM module. +#define _GET_FUNCTION(_T) virtual void getFunction(absl::string_view functionName, _T* f) PURE; + FOR_ALL_WASM_VM_EXPORTS(_GET_FUNCTION) +#undef _GET_FUNCTION + + // Register typed callbacks exported by the host environment. +#define _REGISTER_CALLBACK(_T) \ + virtual void registerCallback(absl::string_view moduleName, absl::string_view functionName, \ + _T f, typename ConvertFunctionTypeWordToUint32<_T>::type) PURE; + FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) +#undef _REGISTER_CALLBACK + + // Register typed value exported by the host environment. + virtual std::unique_ptr> makeGlobal(absl::string_view module_name, + absl::string_view name, Word initial_value) PURE; + virtual std::unique_ptr> + makeGlobal(absl::string_view module_name, absl::string_view name, double initial_value) PURE; +}; + +// Exceptions for issues with the WasmVm. +class WasmVmException : public EnvoyException { +public: + using EnvoyException::EnvoyException; +}; + +// Exceptions for issues with the WebAssembly code. +class WasmException : public EnvoyException { +public: + using EnvoyException::EnvoyException; +}; + +extern thread_local Envoy::Extensions::Common::Wasm::Context* current_context_; +extern thread_local uint32_t effective_context_id_; + +struct SaveRestoreContext { + explicit SaveRestoreContext(Context* context) { + saved_context = current_context_; + saved_effective_context_id_ = effective_context_id_; + current_context_ = context; + effective_context_id_ = 0; // No effective context id. + } + ~SaveRestoreContext() { + current_context_ = saved_context; + effective_context_id_ = saved_effective_context_id_; + } + Context* saved_context; + uint32_t saved_effective_context_id_; +}; + +// Create a new low-level WASM VM of the give type (e.g. "envoy.wasm.vm.wavm"). +std::unique_ptr createWasmVm(absl::string_view vm); + +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/well_known_names.h b/source/extensions/common/wasm/well_known_names.h new file mode 100644 index 000000000000..b5b0fdbbecae --- /dev/null +++ b/source/extensions/common/wasm/well_known_names.h @@ -0,0 +1,26 @@ +#pragma once + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { + +/** + * Well-known wasm VM names. + * NOTE: New wasm VMs should use the well known name: envoy.wasm.vm.name. + */ +class WasmVmValues { +public: + // Null sandbox: modules must be compiled into envoy and registered name is given in the + // DataSource.inline_string. + const std::string Null = "envoy.wasm.vm.null"; +}; + +typedef ConstSingleton WasmVmNames; + +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/wasm/BUILD b/test/extensions/common/wasm/BUILD new file mode 100644 index 000000000000..490402dc1784 --- /dev/null +++ b/test/extensions/common/wasm/BUILD @@ -0,0 +1,19 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_cc_test_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "wasm_vm_test", + srcs = ["wasm_vm_test.cc"], + deps = [ + "//source/extensions/common/wasm:wasm_vm_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/common/wasm/wasm_vm_test.cc b/test/extensions/common/wasm/wasm_vm_test.cc new file mode 100644 index 000000000000..4d209ce42229 --- /dev/null +++ b/test/extensions/common/wasm/wasm_vm_test.cc @@ -0,0 +1,96 @@ +#include "envoy/registry/registry.h" + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/wasm_vm.h" + +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace { + +class TestNullVmPlugin : public Null::NullVmPlugin { +public: + TestNullVmPlugin() {} + ~TestNullVmPlugin() {} + + MOCK_METHOD0(start, void()); +}; + +class PluginFactory : public Null::NullVmPluginFactory { +public: + PluginFactory() {} + + const std::string name() const override { return "test_null_vm_plugin"; } + std::unique_ptr create() const override; +}; + +TestNullVmPlugin* test_null_vm_plugin_ = nullptr; +Envoy::Registry::RegisterFactory register_; + +std::unique_ptr PluginFactory::create() const { + auto result = std::make_unique(); + test_null_vm_plugin_ = result.get(); + return result; +} + +TEST(WasmVmTest, BadVmType) { EXPECT_THROW(createWasmVm("bad.vm"), WasmException); } + +TEST(WasmVmTest, NullVmStartup) { + auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); + EXPECT_TRUE(wasm_vm != nullptr); + EXPECT_TRUE(wasm_vm->clonable()); + auto wasm_vm_clone = wasm_vm->clone(); + EXPECT_TRUE(wasm_vm_clone != nullptr); + EXPECT_TRUE(wasm_vm->getUserSection("user").empty()); +} + +TEST(WasmVmTest, NullVmMemory) { + auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); + EXPECT_EQ(wasm_vm->getMemorySize(), std::numeric_limits::max()); + std::string d = "data"; + auto m = wasm_vm->getMemory(reinterpret_cast(d.data()), d.size()).value(); + EXPECT_EQ(m.data(), d.data()); + EXPECT_EQ(m.size(), d.size()); + uint64_t offset; + char l; + wasm_vm->getMemoryOffset(&l, &offset); + EXPECT_EQ(offset, reinterpret_cast(&l)); + char c; + char z = 'z'; + wasm_vm->setMemory(reinterpret_cast(&c), 1, &z); + EXPECT_EQ(c, z); + Word w(0); + wasm_vm->setWord(reinterpret_cast(&w), std::numeric_limits::max()); + EXPECT_EQ(w.u64, std::numeric_limits::max()); +} + +TEST(WasmVmTest, NullVmStart) { + auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); + EXPECT_TRUE(wasm_vm->load("test_null_vm_plugin", true)); + wasm_vm->link("test", false); + // Test that context argument to start is pushed and that the effective_context_id_ is reset. + // Test that the original values are restored. + Context* context1 = reinterpret_cast(1); + Context* context2 = reinterpret_cast(2); + current_context_ = context1; + effective_context_id_ = 1; + EXPECT_CALL(*test_null_vm_plugin_, start()).WillOnce(Invoke([context2]() { + EXPECT_EQ(current_context_, context2); + EXPECT_EQ(effective_context_id_, 0); + })); + wasm_vm->start(context2); + EXPECT_EQ(current_context_, context1); + EXPECT_EQ(effective_context_id_, 1); +} + +} // namespace +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy From 2efd035c4773ba3e473050518134b0aadb79ff84 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Fri, 23 Aug 2019 12:35:41 -0700 Subject: [PATCH 02/30] Add WASM words to spelling dictionary. Signed-off-by: John Plevyak --- tools/spelling_dictionary.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index bb101fa95946..3cd439c17fc1 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -294,6 +294,8 @@ VC VH VHDS VLOG +VM +WASM WKT WRR WS @@ -451,6 +453,7 @@ dynamodb eg emplace emplaced +emscripten enablement encodings endian From c8125e4ad3262ab4c498e3c35750f562167f8743 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Fri, 23 Aug 2019 12:38:12 -0700 Subject: [PATCH 03/30] Address comments. Signed-off-by: John Plevyak --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 74b909209d91..0ff61156d002 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -43,7 +43,7 @@ extensions/filters/common/original_src @snowp @klarose # dynamic forward proxy /*/extensions/clusters/dynamic_forward_proxy @mattklein123 @alyssawilk /*/extensions/common/dynamic_forward_proxy @mattklein123 @alyssawilk -# wasm common extension +# web assembly common extension /*/extensions/common/wasm @jplevayk @PiotrSikora /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate From c4387e63ba76081109516e2d39e4e679294de302 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Fri, 23 Aug 2019 12:44:44 -0700 Subject: [PATCH 04/30] Define and use WasmVmPtr Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null.cc | 2 +- source/extensions/common/wasm/null/null.h | 2 +- source/extensions/common/wasm/null/null_vm.cc | 2 +- source/extensions/common/wasm/null/null_vm.h | 2 +- source/extensions/common/wasm/wasm_vm.cc | 2 +- source/extensions/common/wasm/wasm_vm.h | 7 +++++-- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/extensions/common/wasm/null/null.cc b/source/extensions/common/wasm/null/null.cc index 06b439e85a42..185dde60780a 100644 --- a/source/extensions/common/wasm/null/null.cc +++ b/source/extensions/common/wasm/null/null.cc @@ -18,7 +18,7 @@ namespace Common { namespace Wasm { namespace Null { -std::unique_ptr createVm() { return std::make_unique(); } +WasmVmPtr createVm() { return std::make_unique(); } } // namespace Null } // namespace Wasm diff --git a/source/extensions/common/wasm/null/null.h b/source/extensions/common/wasm/null/null.h index 1fe697890812..7d88fb356923 100644 --- a/source/extensions/common/wasm/null/null.h +++ b/source/extensions/common/wasm/null/null.h @@ -11,7 +11,7 @@ namespace Common { namespace Wasm { namespace Null { -std::unique_ptr createVm(); +WasmVmPtr createVm(); } // namespace Null } // namespace Wasm diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index 4c9faa117ac4..fdf6c99aad89 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -17,7 +17,7 @@ namespace Common { namespace Wasm { namespace Null { -std::unique_ptr NullVm::clone() { return std::make_unique(*this); } +WasmVmPtr NullVm::clone() { return std::make_unique(*this); } bool NullVm::load(const std::string& name, bool /* allow_precompiled */) { auto factory = Registry::FactoryRegistry::getFactory(name); diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index 1f02082674bb..e8fd48bbef91 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -25,7 +25,7 @@ struct NullVm : public WasmVm { // WasmVm absl::string_view vm() override { return WasmVmNames::get().Null; } bool clonable() override { return true; }; - std::unique_ptr clone() override; + WasmVmPtr clone() override; bool load(const std::string& code, bool allow_precompiled) override; void link(absl::string_view debug_name, bool needs_emscripten) override; void setMemoryLayout(uint64_t, uint64_t, uint64_t) override {} diff --git a/source/extensions/common/wasm/wasm_vm.cc b/source/extensions/common/wasm/wasm_vm.cc index 12966f6525df..9a8dc2f98778 100644 --- a/source/extensions/common/wasm/wasm_vm.cc +++ b/source/extensions/common/wasm/wasm_vm.cc @@ -13,7 +13,7 @@ namespace Wasm { thread_local Envoy::Extensions::Common::Wasm::Context* current_context_ = nullptr; thread_local uint32_t effective_context_id_ = 0; -std::unique_ptr createWasmVm(absl::string_view wasm_vm) { +WasmVmPtr createWasmVm(absl::string_view wasm_vm) { if (wasm_vm.empty()) { throw WasmException("Failed to create WASM VM with unspecified runtime."); } else if (wasm_vm == WasmVmNames::get().Null) { diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index f6d93333536a..6e97cf864122 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -103,6 +103,8 @@ using WasmCallback_WWm = Word (*)(void*, Word, uint64_t); // Wasm VM instance. Provides the low level WASM interface. class WasmVm : public Logger::Loggable { public: + using WasmVmPtr = std::unique_ptr; + virtual ~WasmVm() {} virtual absl::string_view vm() PURE; @@ -110,7 +112,7 @@ class WasmVm : public Logger::Loggable { virtual bool clonable() PURE; // Make a thread-specific copy. This may not be supported by the underlying VM system in which // case it will return nullptr and the caller will need to create a new VM from scratch. - virtual std::unique_ptr clone() PURE; + virtual WasmVmPtr clone() PURE; // Load the WASM code from a file. Return true on success. virtual bool load(const std::string& code, bool allow_precompiled) PURE; @@ -159,6 +161,7 @@ class WasmVm : public Logger::Loggable { virtual std::unique_ptr> makeGlobal(absl::string_view module_name, absl::string_view name, double initial_value) PURE; }; +using WasmVmPtr = std::unique_ptr; // Exceptions for issues with the WasmVm. class WasmVmException : public EnvoyException { @@ -191,7 +194,7 @@ struct SaveRestoreContext { }; // Create a new low-level WASM VM of the give type (e.g. "envoy.wasm.vm.wavm"). -std::unique_ptr createWasmVm(absl::string_view vm); +WasmVmPtr createWasmVm(absl::string_view vm); } // namespace Wasm } // namespace Common From 375de9f34acf45e8097eac64ba371b283a67b4ab Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Fri, 23 Aug 2019 14:30:24 -0700 Subject: [PATCH 05/30] Cleanup some comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.cc | 2 +- source/extensions/common/wasm/null/null_vm.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index fdf6c99aad89..f631006d090f 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -32,7 +32,7 @@ bool NullVm::load(const std::string& name, bool /* allow_precompiled */) { void NullVm::link(absl::string_view /* name */, bool /* needs_emscripten */) {} void NullVm::makeModule(absl::string_view /* name */) { - // NullVm does not advertize code as emscripten so this will not get called. + // NullVm does not advertise code as emscripten so this will not get called. NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index e8fd48bbef91..24ef28f58598 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -45,14 +45,14 @@ struct NullVm : public WasmVm { FOR_ALL_WASM_VM_EXPORTS(_FORWARD_GET_FUNCTION) #undef _FORWARD_GET_FUNCTION - // These are noops for NullVm. + // These are not needed for NullVm which invokes the handlers directly. #define _REGISTER_CALLBACK(_type) \ void registerCallback(absl::string_view, absl::string_view, _type, \ typename ConvertFunctionTypeWordToUint32<_type>::type) override{}; FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) #undef _REGISTER_CALLBACK - // NullVm does not advertize code as emscripten so this will not get called. + // NullVm does not advertise code as emscripten so this will not get called. std::unique_ptr> makeGlobal(absl::string_view, absl::string_view, double) override { NOT_REACHED_GCOVR_EXCL_LINE; From 87a4e5d93be1ca4f4da4ef72c6febc05ec1e4491 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Fri, 23 Aug 2019 14:54:12 -0700 Subject: [PATCH 06/30] Remove unnecessary destructor override. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.h | 1 - 1 file changed, 1 deletion(-) diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index 24ef28f58598..856894d73103 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -20,7 +20,6 @@ namespace Null { struct NullVm : public WasmVm { NullVm() = default; NullVm(const NullVm& other) { load(other.plugin_name_, false /* unused */); } - ~NullVm() override{}; // WasmVm absl::string_view vm() override { return WasmVmNames::get().Null; } From b04b285e6223e601e00281cbd16563c122e91b7b Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Fri, 23 Aug 2019 17:44:55 -0700 Subject: [PATCH 07/30] Address some tidy issues. Signed-off-by: John Plevyak --- source/extensions/common/wasm/BUILD | 1 + source/extensions/common/wasm/null/null_vm.cc | 8 ++++++-- source/extensions/common/wasm/null/null_vm_plugin.h | 8 ++++---- source/extensions/common/wasm/wasm_vm.h | 4 ++-- source/extensions/common/wasm/well_known_names.h | 4 +++- test/extensions/common/wasm/wasm_vm_test.cc | 6 +++--- 6 files changed, 19 insertions(+), 12 deletions(-) diff --git a/source/extensions/common/wasm/BUILD b/source/extensions/common/wasm/BUILD index ab972618e6ec..7eda2486185d 100644 --- a/source/extensions/common/wasm/BUILD +++ b/source/extensions/common/wasm/BUILD @@ -21,6 +21,7 @@ envoy_cc_library( hdrs = ["wasm_vm.h"], deps = [ ":well_known_names", + "//source/common/common:minimal_logger_lib", ], ) diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index f631006d090f..1555f171a933 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -56,8 +56,12 @@ bool NullVm::getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) { } bool NullVm::setMemory(uint64_t pointer, uint64_t size, const void* data) { - if ((pointer == 0 || data == nullptr) && size != 0) { - return false; + if ((pointer == 0 || data == nullptr)) { + if (size != 0) { + return false; + } else { + return true; + } } auto p = reinterpret_cast(pointer); memcpy(p, data, size); diff --git a/source/extensions/common/wasm/null/null_vm_plugin.h b/source/extensions/common/wasm/null/null_vm_plugin.h index 2a8767991ec1..8e900932f9dc 100644 --- a/source/extensions/common/wasm/null/null_vm_plugin.h +++ b/source/extensions/common/wasm/null/null_vm_plugin.h @@ -10,13 +10,13 @@ namespace Null { class NullVmPlugin { public: - NullVmPlugin() {} - virtual ~NullVmPlugin() {} + NullVmPlugin() = default; + virtual ~NullVmPlugin() = default; // NB: These are defined rather than declared PURE because gmock uses __LINE__ internally for // uniqueness, making it impossible to use FOR_ALL_WASM_VM_EXPORTS with MOCK_METHOD2. -#define _DEFINE_GET_FUNCTION(_t) \ - virtual void getFunction(absl::string_view, _t* f) { *f = nullptr; } +#define _DEFINE_GET_FUNCTION(_T) \ + virtual void getFunction(absl::string_view, _T* f) { *f = nullptr; } FOR_ALL_WASM_VM_EXPORTS(_DEFINE_GET_FUNCTION) #undef _DEFIN_GET_FUNCTIONE diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 6e97cf864122..b9a5179b8daf 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -35,7 +35,7 @@ template struct ConvertFunctionTypeWordToUint32 struct Global { - virtual ~Global() {} + virtual ~Global() = default; virtual T get() PURE; virtual void set(const T& t) PURE; }; @@ -105,7 +105,7 @@ class WasmVm : public Logger::Loggable { public: using WasmVmPtr = std::unique_ptr; - virtual ~WasmVm() {} + virtual ~WasmVm() = default; virtual absl::string_view vm() PURE; // Whether or not the VM implementation supports cloning. diff --git a/source/extensions/common/wasm/well_known_names.h b/source/extensions/common/wasm/well_known_names.h index b5b0fdbbecae..3674ed3ee8b5 100644 --- a/source/extensions/common/wasm/well_known_names.h +++ b/source/extensions/common/wasm/well_known_names.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "common/singleton/const_singleton.h" namespace Envoy { @@ -18,7 +20,7 @@ class WasmVmValues { const std::string Null = "envoy.wasm.vm.null"; }; -typedef ConstSingleton WasmVmNames; +using WasmVmNames = ConstSingleton; } // namespace Wasm } // namespace Common diff --git a/test/extensions/common/wasm/wasm_vm_test.cc b/test/extensions/common/wasm/wasm_vm_test.cc index 4d209ce42229..6479d219337e 100644 --- a/test/extensions/common/wasm/wasm_vm_test.cc +++ b/test/extensions/common/wasm/wasm_vm_test.cc @@ -16,15 +16,15 @@ namespace { class TestNullVmPlugin : public Null::NullVmPlugin { public: - TestNullVmPlugin() {} - ~TestNullVmPlugin() {} + TestNullVmPlugin() = default; + ~TestNullVmPlugin() override = default; MOCK_METHOD0(start, void()); }; class PluginFactory : public Null::NullVmPluginFactory { public: - PluginFactory() {} + PluginFactory() = default; const std::string name() const override { return "test_null_vm_plugin"; } std::unique_ptr create() const override; From 630803a85e0c7a32764c76be682ddaed37d02595 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Mon, 26 Aug 2019 15:53:54 -0700 Subject: [PATCH 08/30] Address comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.cc | 2 ++ source/extensions/common/wasm/null/null_vm.h | 3 +++ .../common/wasm/null/null_vm_plugin.h | 1 + source/extensions/common/wasm/wasm_vm.h | 17 ++++++++++++++--- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index 1555f171a933..0168b82e07b7 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -19,6 +19,7 @@ namespace Null { WasmVmPtr NullVm::clone() { return std::make_unique(*this); } +// "Load" the plugin by obtaining a pointer to it from the factory. bool NullVm::load(const std::string& name, bool /* allow_precompiled */) { auto factory = Registry::FactoryRegistry::getFactory(name); if (!factory) { @@ -43,6 +44,7 @@ void NullVm::start(Common::Wasm::Context* context) { uint64_t NullVm::getMemorySize() { return std::numeric_limits::max(); } +// NulVm pointers are just native pointers. absl::optional NullVm::getMemory(uint64_t pointer, uint64_t size) { if (pointer == 0 && size != 0) { return absl::nullopt; diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index 856894d73103..d3c9b9f1b00f 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -17,6 +17,9 @@ namespace Common { namespace Wasm { namespace Null { +// The NullVm wraps a C++ WASM plugin which has been compiled with the WASM API +// and linked directly into the Envoy process. This is useful for development +// in that it permits the debugger to set breakpoints in both Envoy and the plugin. struct NullVm : public WasmVm { NullVm() = default; NullVm(const NullVm& other) { load(other.plugin_name_, false /* unused */); } diff --git a/source/extensions/common/wasm/null/null_vm_plugin.h b/source/extensions/common/wasm/null/null_vm_plugin.h index 8e900932f9dc..940cb1bec017 100644 --- a/source/extensions/common/wasm/null/null_vm_plugin.h +++ b/source/extensions/common/wasm/null/null_vm_plugin.h @@ -8,6 +8,7 @@ namespace Common { namespace Wasm { namespace Null { +// A wrapper for the natively compiled NullVm plugin which implements the WASM ABI. class NullVmPlugin { public: NullVmPlugin() = default; diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index b9a5179b8daf..1141396b0679 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -18,22 +18,23 @@ class Context; // Represents a WASM-native word-sized datum. On 32-bit VMs, the high bits are always zero. // The WASM/VM API treats all bits as significant. struct Word { - Word(uint64_t w) : u64(w) {} // Implicit conversion into Word. - operator uint64_t() const { return u64; } // Implicit conversion into uint64_t. - // Note: no implicit conversion to uint32_t as it is lossy. + Word(uint64_t w) : u64(w) {} // Implicit conversion into Word. uint32_t u32() const { return static_cast(u64); } uint64_t u64; }; +// Convert Word type for use by 32-bit VMs. template struct ConvertWordTypeToUint32 { using type = T; }; template <> struct ConvertWordTypeToUint32 { using type = uint32_t; }; +// Convert Word-based function types for 32-bit VMs. template struct ConvertFunctionTypeWordToUint32 {}; template struct ConvertFunctionTypeWordToUint32 { using type = typename ConvertWordTypeToUint32::type (*)( typename ConvertWordTypeToUint32::type...); }; +// A wrapper for a global variable within the VM. template struct Global { virtual ~Global() = default; virtual T get() PURE; @@ -175,9 +176,19 @@ class WasmException : public EnvoyException { using EnvoyException::EnvoyException; }; +// Thread local state set during a call into a WASM VM so that calls coming out of the +// VM can be attributed correctly to calling Filter. We use thread_local instead of ThreadLocal +// because this state is live only during the calls and does not need to be initialized consistently +// over all workers as with ThreadLocal data. extern thread_local Envoy::Extensions::Common::Wasm::Context* current_context_; +// Requested effective context set by code within the VM to request that the calls coming out of the +// VM be attributed to another filter, for example if a constrol plane gRPC comes comes back to the +// RootContext which effects some set of waiting filters. extern thread_local uint32_t effective_context_id_; +// Helper to save and restore thread local VM call context information to support reentrant calls. +// NB: this happens for example when a call from the VM invokes a handler which needs to _malloc +// memory in the VM. struct SaveRestoreContext { explicit SaveRestoreContext(Context* context) { saved_context = current_context_; From 077335a069f258f323ee5b629eec1c93916a1d54 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 10:01:19 -0700 Subject: [PATCH 09/30] Add some doc comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 95 +++++++++++++++++++------ 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 1141396b0679..20b13da0ab88 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -107,56 +107,111 @@ class WasmVm : public Logger::Loggable { using WasmVmPtr = std::unique_ptr; virtual ~WasmVm() = default; + /** + * Return the VM identifier. + * @return one of WasmVmValues from well_known_names.h e.g. "envoy.wasm.vm.null". + */ virtual absl::string_view vm() PURE; - // Whether or not the VM implementation supports cloning. + /** + * Whether or not the VM implementation supports cloning. + * @return true if the VM is clonable. + */ virtual bool clonable() PURE; - // Make a thread-specific copy. This may not be supported by the underlying VM system in which - // case it will return nullptr and the caller will need to create a new VM from scratch. - virtual WasmVmPtr clone() PURE; - // Load the WASM code from a file. Return true on success. + /** + * Make a thread-specific copy. This may not be supported by the underlying VM system in which + * case it will return nullptr and the caller will need to create a new VM from scratch. + * @return a clone of 'this' (e.g. for a different Worker/thread). + */ + virtual WasmVmPtr clone() PURE; + /** + * Load the WASM code from a file. Return true on success. + * @param code the WASM binary code (or registered NullVm plugin name). + * @param allow_precompiled if true, allows suporting VMs (e.g. WAVM) to load the binary + * machine code from a user-defined section of the WASM file. + * @return whether or not the load was successful. + */ virtual bool load(const std::string& code, bool allow_precompiled) PURE; - // Link to registered function. + /** + * Link to registered function. + * @param debug_name user-provided name for use in error messages. + * @param needs_emscripten whether emscripten support should be provided (e.g. _emscripten_memcpy_bigHandler). + */ virtual void link(absl::string_view debug_name, bool needs_emscripten) PURE; - // Set memory layout (start of dynamic heap base, etc.) in the VM. + /** + * Set memory layout (start of dynamic heap base, etc.) in the VM. + * @param stack_base the location in VM memory of the stack. + * @param heap_base the location in VM memory of the heap. + * @param heap_base_ptr the location in VM memory of a location to store the heap pointer. + */ virtual void setMemoryLayout(uint64_t stack_base, uint64_t heap_base, uint64_t heap_base_pointer) PURE; - // Call the 'start' function and initialize globals. - virtual void start(Context*) PURE; + /** + * Call the 'start' function and initialize globals. + * @param vm_context a context which represents the caller: in this case Envoy itself. + */ + virtual void start(Context* vm_context) PURE; - // Get size of the currently allocated memory in the VM. + /** + * Get size of the currently allocated memory in the VM. + * @result the size of memory in bytes. + */ virtual uint64_t getMemorySize() PURE; - // Convert a block of memory in the VM to a string_view. Returns 'false' in second if the - // pointer/size is invalid. + /** + * Convert a block of memory in the VM to a string_view. + * @param pointer the offset into VM memory of the requested VM memory block. + * @param size the size of the requested VM memory block. + * @result if std::nullopt then the pointer/size pair were invalid, otherwise returns + * a host string_view pointing to the pointer/size pair in VM memory. + */ virtual absl::optional getMemory(uint64_t pointer, uint64_t size) PURE; - // Convert a host pointer to memory in the VM into a VM "pointer" (an offset into the Memory). + /** + * Convert a host pointer to memory in the VM into a VM "pointer" (an offset into the Memory). + * @param host_pointer a pointer to host memory to be converted into a VM offset (pointer). + * @param vm_pointer a pointer to an uint64_t to be filled with the offset in VM memory corresponding to 'host_pointer'. + * @result whether or not the host_pointer was a valid VM memory offset. + */ virtual bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) PURE; - // Set a block of memory in the VM, returns true on success, false if the pointer/size is invalid. + /** + * Set a block of memory in the VM, returns true on success, false if the pointer/size is invalid. + */ virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; - // Set a Word in the VM, returns true on success, false if the pointer is invalid. + /** + * Set a Word in the VM, returns true on success, false if the pointer is invalid. + */ virtual bool setWord(uint64_t pointer, Word data) PURE; - // Make a new intrinsic module (e.g. for Emscripten support). + /** + * Make a new intrinsic module (e.g. for Emscripten support). + */ virtual void makeModule(absl::string_view name) PURE; - // Get the contents of the user section with the given name or "" if it does not exist. + /** + * Get the contents of the user section with the given name or "" if it does not exist. + */ virtual absl::string_view getUserSection(absl::string_view name) PURE; - // Get typed function exported by the WASM module. + /** + * Get typed function exported by the WASM module. + */ #define _GET_FUNCTION(_T) virtual void getFunction(absl::string_view functionName, _T* f) PURE; FOR_ALL_WASM_VM_EXPORTS(_GET_FUNCTION) #undef _GET_FUNCTION - // Register typed callbacks exported by the host environment. + /** + * Register typed callbacks exported by the host environment. + */ #define _REGISTER_CALLBACK(_T) \ virtual void registerCallback(absl::string_view moduleName, absl::string_view functionName, \ _T f, typename ConvertFunctionTypeWordToUint32<_T>::type) PURE; FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) #undef _REGISTER_CALLBACK - // Register typed value exported by the host environment. + /** + * Register typed value exported by the host environment. + */ virtual std::unique_ptr> makeGlobal(absl::string_view module_name, absl::string_view name, Word initial_value) PURE; virtual std::unique_ptr> From d4fb6b03a42e475a3b75f89d607b2fc3f7a8c2c7 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 12:25:30 -0700 Subject: [PATCH 10/30] Add _ to u64 in Word. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.cc | 2 +- source/extensions/common/wasm/wasm_vm.h | 6 +++--- test/extensions/common/wasm/wasm_vm_test.cc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index 0168b82e07b7..6842ef239174 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -75,7 +75,7 @@ bool NullVm::setWord(uint64_t pointer, Word data) { return false; } auto p = reinterpret_cast(pointer); - memcpy(p, &data.u64, sizeof(data.u64)); + memcpy(p, &data.u64_, sizeof(data.u64_)); return true; } diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 20b13da0ab88..ee1950094955 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -18,9 +18,9 @@ class Context; // Represents a WASM-native word-sized datum. On 32-bit VMs, the high bits are always zero. // The WASM/VM API treats all bits as significant. struct Word { - Word(uint64_t w) : u64(w) {} // Implicit conversion into Word. - uint32_t u32() const { return static_cast(u64); } - uint64_t u64; + Word(uint64_t w) : u64_(w) {} // Implicit conversion into Word. + uint32_t u32() const { return static_cast(u64_); } + uint64_t u64_; }; // Convert Word type for use by 32-bit VMs. diff --git a/test/extensions/common/wasm/wasm_vm_test.cc b/test/extensions/common/wasm/wasm_vm_test.cc index 6479d219337e..e78dcdee4cdc 100644 --- a/test/extensions/common/wasm/wasm_vm_test.cc +++ b/test/extensions/common/wasm/wasm_vm_test.cc @@ -67,7 +67,7 @@ TEST(WasmVmTest, NullVmMemory) { EXPECT_EQ(c, z); Word w(0); wasm_vm->setWord(reinterpret_cast(&w), std::numeric_limits::max()); - EXPECT_EQ(w.u64, std::numeric_limits::max()); + EXPECT_EQ(w.u64_, std::numeric_limits::max()); } TEST(WasmVmTest, NullVmStart) { From 2a36aae99999810ebd609b07fb8eb3b964ae2130 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 12:34:23 -0700 Subject: [PATCH 11/30] Fix formatting and spelling. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index ee1950094955..2cb8c3546182 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -136,7 +136,8 @@ class WasmVm : public Logger::Loggable { /** * Link to registered function. * @param debug_name user-provided name for use in error messages. - * @param needs_emscripten whether emscripten support should be provided (e.g. _emscripten_memcpy_bigHandler). + * @param needs_emscripten whether emscripten support should be provided (e.g. + * _emscripten_memcpy_bigHandler). */ virtual void link(absl::string_view debug_name, bool needs_emscripten) PURE; @@ -171,7 +172,8 @@ class WasmVm : public Logger::Loggable { /** * Convert a host pointer to memory in the VM into a VM "pointer" (an offset into the Memory). * @param host_pointer a pointer to host memory to be converted into a VM offset (pointer). - * @param vm_pointer a pointer to an uint64_t to be filled with the offset in VM memory corresponding to 'host_pointer'. + * @param vm_pointer a pointer to an uint64_t to be filled with the offset in VM memory + * corresponding to 'host_pointer'. * @result whether or not the host_pointer was a valid VM memory offset. */ virtual bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) PURE; @@ -237,7 +239,7 @@ class WasmException : public EnvoyException { // over all workers as with ThreadLocal data. extern thread_local Envoy::Extensions::Common::Wasm::Context* current_context_; // Requested effective context set by code within the VM to request that the calls coming out of the -// VM be attributed to another filter, for example if a constrol plane gRPC comes comes back to the +// VM be attributed to another filter, for example if a control plane gRPC comes back to the // RootContext which effects some set of waiting filters. extern thread_local uint32_t effective_context_id_; From 9076aac13acc4273c301ff40f3a748178aaff59a Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 12:49:02 -0700 Subject: [PATCH 12/30] Fixup spelling. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 4 ++-- tools/spelling_dictionary.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 2cb8c3546182..51b70e8af30e 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -115,7 +115,7 @@ class WasmVm : public Logger::Loggable { /** * Whether or not the VM implementation supports cloning. - * @return true if the VM is clonable. + * @return true if the VM is clone-able. */ virtual bool clonable() PURE; @@ -128,7 +128,7 @@ class WasmVm : public Logger::Loggable { /** * Load the WASM code from a file. Return true on success. * @param code the WASM binary code (or registered NullVm plugin name). - * @param allow_precompiled if true, allows suporting VMs (e.g. WAVM) to load the binary + * @param allow_precompiled if true, allows supporting VMs (e.g. WAVM) to load the binary * machine code from a user-defined section of the WASM file. * @return whether or not the load was successful. */ diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 3cd439c17fc1..ce6cff5a08a6 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -9,6 +9,7 @@ ALPN ALS AMZ API +ABI ASAN ASCII ASSERTs @@ -296,6 +297,7 @@ VHDS VLOG VM WASM +WAVM WKT WRR WS From f31cdf49f1ffb1e2f412da375ee8da52ed1f4930 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 16:28:49 -0700 Subject: [PATCH 13/30] Address comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 27 ++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 51b70e8af30e..5824cfce4a95 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -158,14 +158,14 @@ class WasmVm : public Logger::Loggable { /** * Get size of the currently allocated memory in the VM. - * @result the size of memory in bytes. + * @return the size of memory in bytes. */ virtual uint64_t getMemorySize() PURE; /** * Convert a block of memory in the VM to a string_view. * @param pointer the offset into VM memory of the requested VM memory block. * @param size the size of the requested VM memory block. - * @result if std::nullopt then the pointer/size pair were invalid, otherwise returns + * @return if std::nullopt then the pointer/size pair were invalid, otherwise returns * a host string_view pointing to the pointer/size pair in VM memory. */ virtual absl::optional getMemory(uint64_t pointer, uint64_t size) PURE; @@ -174,24 +174,34 @@ class WasmVm : public Logger::Loggable { * @param host_pointer a pointer to host memory to be converted into a VM offset (pointer). * @param vm_pointer a pointer to an uint64_t to be filled with the offset in VM memory * corresponding to 'host_pointer'. - * @result whether or not the host_pointer was a valid VM memory offset. + * @return whether or not the host_pointer was a valid VM memory offset. */ virtual bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) PURE; /** * Set a block of memory in the VM, returns true on success, false if the pointer/size is invalid. + * @param pointer the offset into VM memory describing the start of a region of VM memory. + * @param size the size of the region of VM memory. + * @return whether or not the pointer/size pair was a valid VM memory block. */ virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; /** * Set a Word in the VM, returns true on success, false if the pointer is invalid. + * @param pointer the offset into VM memory describing the start of VM native word size block. + * @param data a Word whose contents will be written in VM native word size at 'pointer. + * @return whether or not the pointer was to a valid VM memory block of VM native word size. */ virtual bool setWord(uint64_t pointer, Word data) PURE; /** * Make a new intrinsic module (e.g. for Emscripten support). + * @param name the name of the module to make. */ virtual void makeModule(absl::string_view name) PURE; /** * Get the contents of the user section with the given name or "" if it does not exist. + * @param name the name of the user section to get. + * @return the contents of the user section (if any). The result will be empty() if there + * is no such section. */ virtual absl::string_view getUserSection(absl::string_view name) PURE; @@ -213,9 +223,20 @@ class WasmVm : public Logger::Loggable { /** * Register typed value exported by the host environment. + * @param module_name the name of the module to which to export the global. + * @param name the name of the global variable to export. + * @param initial_value the initial value of the global. + * @return a Global object which can be used to access the exported global. */ virtual std::unique_ptr> makeGlobal(absl::string_view module_name, absl::string_view name, Word initial_value) PURE; + /** + * Register typed value exported by the host environment. + * @param module_name the name of the module to which to export the global. + * @param name the name of the global variable to export. + * @param initial_value the initial value of the global. + * @return a Global object which can be used to access the exported global. + */ virtual std::unique_ptr> makeGlobal(absl::string_view module_name, absl::string_view name, double initial_value) PURE; }; From 56c091abe7b7fca115311b7df5ff9d7c20d3d191 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 17:01:53 -0700 Subject: [PATCH 14/30] Fix formatting. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 5824cfce4a95..0e0467a4cb26 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -200,7 +200,7 @@ class WasmVm : public Logger::Loggable { /** * Get the contents of the user section with the given name or "" if it does not exist. * @param name the name of the user section to get. - * @return the contents of the user section (if any). The result will be empty() if there + * @return the contents of the user section (if any). The result will be empty() if there * is no such section. */ virtual absl::string_view getUserSection(absl::string_view name) PURE; From 1a6a933670159b2b786a749ecd7beea948a4cdab Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Tue, 27 Aug 2019 19:07:54 -0700 Subject: [PATCH 15/30] Use templates to define the WASM call and callout signatures. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.cc | 6 +- source/extensions/common/wasm/null/null_vm.h | 2 +- .../common/wasm/null/null_vm_plugin.h | 2 +- source/extensions/common/wasm/wasm_vm.h | 81 ++++++++----------- 4 files changed, 42 insertions(+), 49 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index 6842ef239174..d97f5ca002b4 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -17,7 +17,11 @@ namespace Common { namespace Wasm { namespace Null { -WasmVmPtr NullVm::clone() { return std::make_unique(*this); } +WasmVmPtr NullVm::clone() { + auto cloned_null_vm = std::make_unique(*this); + cloned_null_vm->load(plugin_name_, false /* unused */); + return cloned_null_vm; +} // "Load" the plugin by obtaining a pointer to it from the factory. bool NullVm::load(const std::string& name, bool /* allow_precompiled */) { diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index d3c9b9f1b00f..b0eff4e5d3ae 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -22,7 +22,7 @@ namespace Null { // in that it permits the debugger to set breakpoints in both Envoy and the plugin. struct NullVm : public WasmVm { NullVm() = default; - NullVm(const NullVm& other) { load(other.plugin_name_, false /* unused */); } + NullVm(const NullVm& other) : plugin_name_(other.plugin_name_) {} // WasmVm absl::string_view vm() override { return WasmVmNames::get().Null; } diff --git a/source/extensions/common/wasm/null/null_vm_plugin.h b/source/extensions/common/wasm/null/null_vm_plugin.h index 940cb1bec017..4dce2c617238 100644 --- a/source/extensions/common/wasm/null/null_vm_plugin.h +++ b/source/extensions/common/wasm/null/null_vm_plugin.h @@ -30,7 +30,7 @@ class NullVmPlugin { */ class NullVmPluginFactory { public: - virtual ~NullVmPluginFactory() {} + virtual ~NullVmPluginFactory() = default; /** * Name of the plugin. diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 0e0467a4cb26..7a7ff7aa0186 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -41,58 +41,39 @@ template struct Global { virtual void set(const T& t) PURE; }; +// Templates for constructing signatures of functions calling into and out of WASM VMs. +template +struct WasmFuncTypeHelper {}; + +template +struct WasmFuncTypeHelper { + using type = typename WasmFuncTypeHelper::type; +}; + +template +struct WasmFuncTypeHelper<0, ReturnType, ContextType, ParamType, ReturnType(ContextType, Args...)> { + using type = ReturnType(ContextType, Args...); +}; + +template +using WasmFuncType = typename WasmFuncTypeHelper::type; + // Calls into the WASM VM. // 1st arg is always a pointer to Context (Context*). -using WasmCall0Void = std::function; -using WasmCall1Void = std::function; -using WasmCall2Void = std::function; -using WasmCall3Void = std::function; -using WasmCall4Void = std::function; -using WasmCall5Void = std::function; -using WasmCall6Void = std::function; -using WasmCall7Void = std::function; -using WasmCall8Void = std::function; -using WasmCall0Word = std::function; -using WasmCall1Word = std::function; -using WasmCall2Word = std::function; -using WasmCall3Word = std::function; -using WasmCall4Word = std::function; -using WasmCall5Word = std::function; -using WasmCall6Word = std::function; -using WasmCall7Word = std::function; -using WasmCall8Word = std::function; +template using WasmCallVoid = std::function>; +template using WasmCallWord = std::function>; + #define FOR_ALL_WASM_VM_EXPORTS(_f) \ - _f(WasmCall0Void) _f(WasmCall1Void) _f(WasmCall2Void) _f(WasmCall3Void) _f(WasmCall4Void) \ - _f(WasmCall5Void) _f(WasmCall8Void) _f(WasmCall0Word) _f(WasmCall1Word) _f(WasmCall3Word) + _f(WasmCallVoid<0>) _f(WasmCallVoid<1>) _f(WasmCallVoid<2>) _f(WasmCallVoid<3>) \ + _f(WasmCallVoid<4>) _f(WasmCallVoid<5>) _f(WasmCallVoid<8>) _f(WasmCallWord<0>) \ + _f(WasmCallWord<1>) _f(WasmCallWord<3>) // Calls out of the WASM VM. // 1st arg is always a pointer to raw_context (void*). -using WasmCallback0Void = void (*)(void*); -using WasmCallback1Void = void (*)(void*, Word); -using WasmCallback2Void = void (*)(void*, Word, Word); -using WasmCallback3Void = void (*)(void*, Word, Word, Word); -using WasmCallback4Void = void (*)(void*, Word, Word, Word, Word); -using WasmCallback5Void = void (*)(void*, Word, Word, Word, Word, Word); -using WasmCallback6Void = void (*)(void*, Word, Word, Word, Word, Word, Word); -using WasmCallback7Void = void (*)(void*, Word, Word, Word, Word, Word, Word, Word); -using WasmCallback8Void = void (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word); -using WasmCallback0Word = Word (*)(void*); -using WasmCallback1Word = Word (*)(void*, Word); -using WasmCallback2Word = Word (*)(void*, Word, Word); -using WasmCallback3Word = Word (*)(void*, Word, Word, Word); -using WasmCallback4Word = Word (*)(void*, Word, Word, Word, Word); -using WasmCallback5Word = Word (*)(void*, Word, Word, Word, Word, Word); -using WasmCallback6Word = Word (*)(void*, Word, Word, Word, Word, Word, Word); -using WasmCallback7Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word); -using WasmCallback8Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word, Word); -using WasmCallback9Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Word, Word, Word, - Word); -#define FOR_ALL_WASM_VM_IMPORTS(_f) \ - _f(WasmCallback0Void) _f(WasmCallback1Void) _f(WasmCallback2Void) _f(WasmCallback3Void) \ - _f(WasmCallback4Void) _f(WasmCallback0Word) _f(WasmCallback1Word) _f(WasmCallback2Word) \ - _f(WasmCallback3Word) _f(WasmCallback4Word) _f(WasmCallback5Word) _f(WasmCallback6Word) \ - _f(WasmCallback7Word) _f(WasmCallback8Word) _f(WasmCallback9Word) \ - _f(WasmCallback_WWl) _f(WasmCallback_WWm) +template using WasmCallbackVoid = WasmFuncType*; +template using WasmCallbackWord = WasmFuncType*; // Using the standard g++/clang mangling algorithm: // https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling-builtin @@ -101,6 +82,14 @@ using WasmCallback9Word = Word (*)(void*, Word, Word, Word, Word, Word, Word, Wo using WasmCallback_WWl = Word (*)(void*, Word, int64_t); using WasmCallback_WWm = Word (*)(void*, Word, uint64_t); +#define FOR_ALL_WASM_VM_IMPORTS(_f) \ + _f(WasmCallbackVoid<0>) _f(WasmCallbackVoid<1>) _f(WasmCallbackVoid<2>) _f(WasmCallbackVoid<3>) \ + _f(WasmCallbackVoid<4>) _f(WasmCallbackWord<0>) _f(WasmCallbackWord<1>) \ + _f(WasmCallbackWord<2>) _f(WasmCallbackWord<3>) _f(WasmCallbackWord<4>) \ + _f(WasmCallbackWord<5>) _f(WasmCallbackWord<6>) _f(WasmCallbackWord<7>) \ + _f(WasmCallbackWord<8>) _f(WasmCallbackWord<9>) _f(WasmCallback_WWl) \ + _f(WasmCallback_WWm) + // Wasm VM instance. Provides the low level WASM interface. class WasmVm : public Logger::Loggable { public: From 364f892bba25035dc1d36ef563d7e02bdda19d88 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 12:30:27 -0700 Subject: [PATCH 16/30] Address comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 7a7ff7aa0186..e204c9512098 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -114,6 +114,7 @@ class WasmVm : public Logger::Loggable { * @return a clone of 'this' (e.g. for a different Worker/thread). */ virtual WasmVmPtr clone() PURE; + /** * Load the WASM code from a file. Return true on success. * @param code the WASM binary code (or registered NullVm plugin name). @@ -122,6 +123,7 @@ class WasmVm : public Logger::Loggable { * @return whether or not the load was successful. */ virtual bool load(const std::string& code, bool allow_precompiled) PURE; + /** * Link to registered function. * @param debug_name user-provided name for use in error messages. @@ -150,6 +152,7 @@ class WasmVm : public Logger::Loggable { * @return the size of memory in bytes. */ virtual uint64_t getMemorySize() PURE; + /** * Convert a block of memory in the VM to a string_view. * @param pointer the offset into VM memory of the requested VM memory block. @@ -158,6 +161,7 @@ class WasmVm : public Logger::Loggable { * a host string_view pointing to the pointer/size pair in VM memory. */ virtual absl::optional getMemory(uint64_t pointer, uint64_t size) PURE; + /** * Convert a host pointer to memory in the VM into a VM "pointer" (an offset into the Memory). * @param host_pointer a pointer to host memory to be converted into a VM offset (pointer). @@ -166,6 +170,7 @@ class WasmVm : public Logger::Loggable { * @return whether or not the host_pointer was a valid VM memory offset. */ virtual bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) PURE; + /** * Set a block of memory in the VM, returns true on success, false if the pointer/size is invalid. * @param pointer the offset into VM memory describing the start of a region of VM memory. @@ -173,6 +178,7 @@ class WasmVm : public Logger::Loggable { * @return whether or not the pointer/size pair was a valid VM memory block. */ virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; + /** * Set a Word in the VM, returns true on success, false if the pointer is invalid. * @param pointer the offset into VM memory describing the start of VM native word size block. @@ -180,6 +186,7 @@ class WasmVm : public Logger::Loggable { * @return whether or not the pointer was to a valid VM memory block of VM native word size. */ virtual bool setWord(uint64_t pointer, Word data) PURE; + /** * Make a new intrinsic module (e.g. for Emscripten support). * @param name the name of the module to make. @@ -219,6 +226,7 @@ class WasmVm : public Logger::Loggable { */ virtual std::unique_ptr> makeGlobal(absl::string_view module_name, absl::string_view name, Word initial_value) PURE; + /** * Register typed value exported by the host environment. * @param module_name the name of the module to which to export the global. @@ -248,6 +256,7 @@ class WasmException : public EnvoyException { // because this state is live only during the calls and does not need to be initialized consistently // over all workers as with ThreadLocal data. extern thread_local Envoy::Extensions::Common::Wasm::Context* current_context_; + // Requested effective context set by code within the VM to request that the calls coming out of the // VM be attributed to another filter, for example if a control plane gRPC comes back to the // RootContext which effects some set of waiting filters. From 2571fe304c95e503b1fb5ded85b902ce95aacd8b Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 13:30:49 -0700 Subject: [PATCH 17/30] Add more comments for emscripten. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index e204c9512098..e7c51ae24760 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -125,10 +125,11 @@ class WasmVm : public Logger::Loggable { virtual bool load(const std::string& code, bool allow_precompiled) PURE; /** - * Link to registered function. + * Link the WASM code to the host-provided functions and globals, e.g. the ABI. * @param debug_name user-provided name for use in error messages. * @param needs_emscripten whether emscripten support should be provided (e.g. - * _emscripten_memcpy_bigHandler). + * _emscripten_memcpy_bigHandler). Emscripten (http://https://emscripten.org/) is + * a C++ WebAssembly tool chain. */ virtual void link(absl::string_view debug_name, bool needs_emscripten) PURE; From 86db36f144e008a09cfa1f3ce7ce26028a00b68f Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 14:39:41 -0700 Subject: [PATCH 18/30] Update comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index e7c51ae24760..9d71fb7b9fb9 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -109,9 +109,10 @@ class WasmVm : public Logger::Loggable { virtual bool clonable() PURE; /** - * Make a thread-specific copy. This may not be supported by the underlying VM system in which - * case it will return nullptr and the caller will need to create a new VM from scratch. - * @return a clone of 'this' (e.g. for a different Worker/thread). + * Make a worker/thread-specific copy if supported by the underlying VM system (see clonable() + * above). If not supported, the caller will need to create a new VM from scratch. If supported, + * the clone may share compiled code and other read-only data with the source VM. + * @return a clone of 'this' (e.g. for a different worker/thread). */ virtual WasmVmPtr clone() PURE; From e9b546f180a3dcf6caeec25af030e3e66ac9a576 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 15:33:42 -0700 Subject: [PATCH 19/30] Fix case of paramaters. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.h | 10 +++++----- source/extensions/common/wasm/wasm_vm.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index b0eff4e5d3ae..8e9f9e470a47 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -41,16 +41,16 @@ struct NullVm : public WasmVm { absl::string_view getUserSection(absl::string_view name) override; #define _FORWARD_GET_FUNCTION(_T) \ - void getFunction(absl::string_view functionName, _T* f) override { \ - plugin_->getFunction(functionName, f); \ + void getFunction(absl::string_view function_name, _T* f) override { \ + plugin_->getFunction(function_name, f); \ } FOR_ALL_WASM_VM_EXPORTS(_FORWARD_GET_FUNCTION) #undef _FORWARD_GET_FUNCTION // These are not needed for NullVm which invokes the handlers directly. -#define _REGISTER_CALLBACK(_type) \ - void registerCallback(absl::string_view, absl::string_view, _type, \ - typename ConvertFunctionTypeWordToUint32<_type>::type) override{}; +#define _REGISTER_CALLBACK(_T) \ + void registerCallback(absl::string_view, absl::string_view, _T, \ + typename ConvertFunctionTypeWordToUint32<_T>::type) override{}; FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) #undef _REGISTER_CALLBACK diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 9d71fb7b9fb9..6a0c1829cc80 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -206,7 +206,7 @@ class WasmVm : public Logger::Loggable { /** * Get typed function exported by the WASM module. */ -#define _GET_FUNCTION(_T) virtual void getFunction(absl::string_view functionName, _T* f) PURE; +#define _GET_FUNCTION(_T) virtual void getFunction(absl::string_view function_name, _T* f) PURE; FOR_ALL_WASM_VM_EXPORTS(_GET_FUNCTION) #undef _GET_FUNCTION @@ -214,7 +214,7 @@ class WasmVm : public Logger::Loggable { * Register typed callbacks exported by the host environment. */ #define _REGISTER_CALLBACK(_T) \ - virtual void registerCallback(absl::string_view moduleName, absl::string_view functionName, \ + virtual void registerCallback(absl::string_view moduleName, absl::string_view function_name, \ _T f, typename ConvertFunctionTypeWordToUint32<_T>::type) PURE; FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) #undef _REGISTER_CALLBACK From 3726db90287b7571661214b780ec4e3c255873ce Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 15:43:34 -0700 Subject: [PATCH 20/30] Fix formatting. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.h | 8 ++++---- source/extensions/common/wasm/wasm_vm.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index 8e9f9e470a47..b8c982bc33d4 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -41,15 +41,15 @@ struct NullVm : public WasmVm { absl::string_view getUserSection(absl::string_view name) override; #define _FORWARD_GET_FUNCTION(_T) \ - void getFunction(absl::string_view function_name, _T* f) override { \ - plugin_->getFunction(function_name, f); \ + void getFunction(absl::string_view function_name, _T* f) override { \ + plugin_->getFunction(function_name, f); \ } FOR_ALL_WASM_VM_EXPORTS(_FORWARD_GET_FUNCTION) #undef _FORWARD_GET_FUNCTION // These are not needed for NullVm which invokes the handlers directly. -#define _REGISTER_CALLBACK(_T) \ - void registerCallback(absl::string_view, absl::string_view, _T, \ +#define _REGISTER_CALLBACK(_T) \ + void registerCallback(absl::string_view, absl::string_view, _T, \ typename ConvertFunctionTypeWordToUint32<_T>::type) override{}; FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) #undef _REGISTER_CALLBACK diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 6a0c1829cc80..f553aee9339f 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -214,7 +214,7 @@ class WasmVm : public Logger::Loggable { * Register typed callbacks exported by the host environment. */ #define _REGISTER_CALLBACK(_T) \ - virtual void registerCallback(absl::string_view moduleName, absl::string_view function_name, \ + virtual void registerCallback(absl::string_view moduleName, absl::string_view function_name, \ _T f, typename ConvertFunctionTypeWordToUint32<_T>::type) PURE; FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) #undef _REGISTER_CALLBACK From 34c798b4a4376445516e276e511971305063121f Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 16:37:45 -0700 Subject: [PATCH 21/30] Incorporate template comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index f553aee9339f..5d5f4b6f3e3c 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -41,7 +41,14 @@ template struct Global { virtual void set(const T& t) PURE; }; -// Templates for constructing signatures of functions calling into and out of WASM VMs. +// These are templates and its helper for constructing signatures of functions calling into and out +// of WASM VMs. +// - WasmFuncTypeHelper is a helper for WasmFuncType and shouldn't be used anywhere else than +// WasmFuncType definition. +// - WasmFuncType takes 4 template parameter which are number of argument, return type, context type +// and param type respectively, resolve to a function type. +// For example `WasmFuncType<3, void, Context*, Word>` resolves to `void(Context*, Word, Word, +// Word)` template struct WasmFuncTypeHelper {}; From 2b86403bdd3a658881f854e93a05ae53a2fe8484 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 16:42:46 -0700 Subject: [PATCH 22/30] Update debug_name doc comment. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 5d5f4b6f3e3c..5c5eded55e82 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -134,7 +134,7 @@ class WasmVm : public Logger::Loggable { /** * Link the WASM code to the host-provided functions and globals, e.g. the ABI. - * @param debug_name user-provided name for use in error messages. + * @param debug_name user-provided name for use in log and error messages. * @param needs_emscripten whether emscripten support should be provided (e.g. * _emscripten_memcpy_bigHandler). Emscripten (http://https://emscripten.org/) is * a C++ WebAssembly tool chain. From 69283763c78c4adf7d3b7750f99cd66c9ebe71a8 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 16:48:25 -0700 Subject: [PATCH 23/30] Update doc comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 5c5eded55e82..48988b6639fa 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -127,7 +127,9 @@ class WasmVm : public Logger::Loggable { * Load the WASM code from a file. Return true on success. * @param code the WASM binary code (or registered NullVm plugin name). * @param allow_precompiled if true, allows supporting VMs (e.g. WAVM) to load the binary - * machine code from a user-defined section of the WASM file. + * machine code from a user-defined section of the WASM file. Because that code is not verified by + * the envoy process it is up to the user to ensure that the code is both safe and is built for + * the linked in version of WAVM. * @return whether or not the load was successful. */ virtual bool load(const std::string& code, bool allow_precompiled) PURE; From 58e2b8e93b5315d056581e82d93c04fe8d86aa5a Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 16:55:25 -0700 Subject: [PATCH 24/30] Update spelling. Signed-off-by: John Plevyak --- CODEOWNERS | 4 ++-- source/extensions/common/wasm/null/null_vm.h | 2 +- source/extensions/common/wasm/wasm_vm.h | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 0ff61156d002..59b85b44b615 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -43,8 +43,6 @@ extensions/filters/common/original_src @snowp @klarose # dynamic forward proxy /*/extensions/clusters/dynamic_forward_proxy @mattklein123 @alyssawilk /*/extensions/common/dynamic_forward_proxy @mattklein123 @alyssawilk -# web assembly common extension -/*/extensions/common/wasm @jplevayk @PiotrSikora /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate /*/extensions/retry/host/omit_canary_hosts @sriduth @snowp @@ -57,3 +55,5 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/listener/http_inspector @crazyxy @PiotrSikora @lizan # attribute context /*/extensions/filters/common/expr @kyessenov @yangminzhu +# webassembly common extension +/*/extensions/common/wasm @jplevyak @PiotrSikora diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index b8c982bc33d4..9c8d981799dd 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -26,7 +26,7 @@ struct NullVm : public WasmVm { // WasmVm absl::string_view vm() override { return WasmVmNames::get().Null; } - bool clonable() override { return true; }; + bool cloneable() override { return true; }; WasmVmPtr clone() override; bool load(const std::string& code, bool allow_precompiled) override; void link(absl::string_view debug_name, bool needs_emscripten) override; diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 48988b6639fa..bd3614e353c5 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -111,12 +111,12 @@ class WasmVm : public Logger::Loggable { /** * Whether or not the VM implementation supports cloning. - * @return true if the VM is clone-able. + * @return true if the VM is cloneable. */ - virtual bool clonable() PURE; + virtual bool cloneable() PURE; /** - * Make a worker/thread-specific copy if supported by the underlying VM system (see clonable() + * Make a worker/thread-specific copy if supported by the underlying VM system (see cloneable() * above). If not supported, the caller will need to create a new VM from scratch. If supported, * the clone may share compiled code and other read-only data with the source VM. * @return a clone of 'this' (e.g. for a different worker/thread). From 29cedee5124c1d447094c958632a8649a4591ce4 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 28 Aug 2019 17:43:03 -0700 Subject: [PATCH 25/30] Fix spelling error. Signed-off-by: John Plevyak --- test/extensions/common/wasm/wasm_vm_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/common/wasm/wasm_vm_test.cc b/test/extensions/common/wasm/wasm_vm_test.cc index e78dcdee4cdc..ff35e65b06f7 100644 --- a/test/extensions/common/wasm/wasm_vm_test.cc +++ b/test/extensions/common/wasm/wasm_vm_test.cc @@ -44,7 +44,7 @@ TEST(WasmVmTest, BadVmType) { EXPECT_THROW(createWasmVm("bad.vm"), WasmException TEST(WasmVmTest, NullVmStartup) { auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); EXPECT_TRUE(wasm_vm != nullptr); - EXPECT_TRUE(wasm_vm->clonable()); + EXPECT_TRUE(wasm_vm->cloneable()); auto wasm_vm_clone = wasm_vm->clone(); EXPECT_TRUE(wasm_vm_clone != nullptr); EXPECT_TRUE(wasm_vm->getUserSection("user").empty()); From 81e029c28f63c73658869bc01d61cf774be7dcf5 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Thu, 29 Aug 2019 10:40:56 -0700 Subject: [PATCH 26/30] Add getWord and tests. Signed-off-by: John Plevyak --- source/extensions/common/wasm/null/null_vm.cc | 9 +++++++++ source/extensions/common/wasm/null/null_vm.h | 1 + source/extensions/common/wasm/wasm_vm.h | 11 ++++++++++- test/extensions/common/wasm/wasm_vm_test.cc | 15 +++++++++++---- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc index d97f5ca002b4..b9957c6a67f2 100644 --- a/source/extensions/common/wasm/null/null_vm.cc +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -83,6 +83,15 @@ bool NullVm::setWord(uint64_t pointer, Word data) { return true; } +bool NullVm::getWord(uint64_t pointer, Word* data) { + if (pointer == 0) { + return false; + } + auto p = reinterpret_cast(pointer); + memcpy(&data->u64_, p, sizeof(data->u64_)); + return true; +} + absl::string_view NullVm::getUserSection(absl::string_view /* name */) { // Return nothing: there is no WASM file. return {}; diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h index 9c8d981799dd..f3b90fabf1c8 100644 --- a/source/extensions/common/wasm/null/null_vm.h +++ b/source/extensions/common/wasm/null/null_vm.h @@ -37,6 +37,7 @@ struct NullVm : public WasmVm { bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) override; bool setMemory(uint64_t pointer, uint64_t size, const void* data) override; bool setWord(uint64_t pointer, Word data) override; + bool getWord(uint64_t pointer, Word* data) override; void makeModule(absl::string_view name) override; absl::string_view getUserSection(absl::string_view name) override; diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index bd3614e353c5..2559842e2d61 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -190,10 +190,19 @@ class WasmVm : public Logger::Loggable { */ virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; + /** + * Get a Word in the VM, returns true on success, false if the pointer is invalid. + * @param pointer the offset into VM memory describing the start of VM native word size block. + * @param data a pointer to a Word whose contents will be filled from the VM native word at + * 'pointer'. + * @return whether or not the pointer was to a valid VM memory block of VM native word size. + */ + virtual bool getWord(uint64_t pointer, Word* data) PURE; + /** * Set a Word in the VM, returns true on success, false if the pointer is invalid. * @param pointer the offset into VM memory describing the start of VM native word size block. - * @param data a Word whose contents will be written in VM native word size at 'pointer. + * @param data a Word whose contents will be written in VM native word size at 'pointer'. * @return whether or not the pointer was to a valid VM memory block of VM native word size. */ virtual bool setWord(uint64_t pointer, Word data) PURE; diff --git a/test/extensions/common/wasm/wasm_vm_test.cc b/test/extensions/common/wasm/wasm_vm_test.cc index ff35e65b06f7..e1ecd2590f6f 100644 --- a/test/extensions/common/wasm/wasm_vm_test.cc +++ b/test/extensions/common/wasm/wasm_vm_test.cc @@ -59,15 +59,22 @@ TEST(WasmVmTest, NullVmMemory) { EXPECT_EQ(m.size(), d.size()); uint64_t offset; char l; - wasm_vm->getMemoryOffset(&l, &offset); + EXPECT_TRUE(wasm_vm->getMemoryOffset(&l, &offset)); EXPECT_EQ(offset, reinterpret_cast(&l)); char c; char z = 'z'; - wasm_vm->setMemory(reinterpret_cast(&c), 1, &z); + EXPECT_TRUE(wasm_vm->setMemory(reinterpret_cast(&c), 1, &z)); EXPECT_EQ(c, z); - Word w(0); - wasm_vm->setWord(reinterpret_cast(&w), std::numeric_limits::max()); + + Word w(13); + EXPECT_TRUE( + wasm_vm->setWord(reinterpret_cast(&w), std::numeric_limits::max())); EXPECT_EQ(w.u64_, std::numeric_limits::max()); + + Word w2(0); + w.u64_ = 7; + EXPECT_TRUE(wasm_vm->getWord(reinterpret_cast(&w), &w2)); + EXPECT_EQ(w2.u64_, 7); } TEST(WasmVmTest, NullVmStart) { From ac6f651065b7668378bf4aa1bee3bd3356501265 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Thu, 5 Sep 2019 12:18:25 -0700 Subject: [PATCH 27/30] Add more comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 2559842e2d61..309ae8756ff1 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -110,7 +110,12 @@ class WasmVm : public Logger::Loggable { virtual absl::string_view vm() PURE; /** - * Whether or not the VM implementation supports cloning. + * Whether or not the VM implementation supports cloning. Cloning is VM system dependent. + * When a VM is configured a single VM is intantiated to check that the .wasm file is valid and to + * do VM system specific initialization. In the case of WAVM this is potentially ahead-of-time + * compilation. Then, if cloning is supported, we clone that VM for each worker, potentially + * copying and sharing the initialized data structures for efficiency. Otherwise we create an new + * VM from scratch for each worker. * @return true if the VM is cloneable. */ virtual bool cloneable() PURE; From 5bae934543fc403fdddb8904b99e7be8658619a5 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Thu, 5 Sep 2019 12:39:45 -0700 Subject: [PATCH 28/30] Add comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 309ae8756ff1..6a7e621d28a7 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -111,8 +111,8 @@ class WasmVm : public Logger::Loggable { /** * Whether or not the VM implementation supports cloning. Cloning is VM system dependent. - * When a VM is configured a single VM is intantiated to check that the .wasm file is valid and to - * do VM system specific initialization. In the case of WAVM this is potentially ahead-of-time + * When a VM is configured a single VM is instantiated to check that the .wasm file is valid and + * to do VM system specific initialization. In the case of WAVM this is potentially ahead-of-time * compilation. Then, if cloning is supported, we clone that VM for each worker, potentially * copying and sharing the initialized data structures for efficiency. Otherwise we create an new * VM from scratch for each worker. @@ -129,7 +129,9 @@ class WasmVm : public Logger::Loggable { virtual WasmVmPtr clone() PURE; /** - * Load the WASM code from a file. Return true on success. + * Load the WASM code from a file. Return true on success. Once the module is loaded it can be + * queried, e.g. to see which version of emscriptent support is rquired. After loading, the + * appropriate ABI callbacks can be registered and then the module can be link()ed (see below). * @param code the WASM binary code (or registered NullVm plugin name). * @param allow_precompiled if true, allows supporting VMs (e.g. WAVM) to load the binary * machine code from a user-defined section of the WASM file. Because that code is not verified by @@ -140,7 +142,9 @@ class WasmVm : public Logger::Loggable { virtual bool load(const std::string& code, bool allow_precompiled) PURE; /** - * Link the WASM code to the host-provided functions and globals, e.g. the ABI. + * Link the WASM code to the host-provided functions and globals, e.g. the ABI. Prior to linking, + * the module should be loaded and the ABI callbacks registered (see above). Linking should be + * done once between load() and start(). * @param debug_name user-provided name for use in log and error messages. * @param needs_emscripten whether emscripten support should be provided (e.g. * _emscripten_memcpy_bigHandler). Emscripten (http://https://emscripten.org/) is @@ -158,7 +162,10 @@ class WasmVm : public Logger::Loggable { uint64_t heap_base_pointer) PURE; /** - * Call the 'start' function and initialize globals. + * Iniitalie globals (including calling global constructors) and call the 'start' function. Prior + * to calling start() the module should be load()ed, ABI callbacks should be registered + * (registerCallback), the module link()ed, and any exported functions should be gotten + * (getFunction). * @param vm_context a context which represents the caller: in this case Envoy itself. */ virtual void start(Context* vm_context) PURE; From b1d35938136310dda28d01d828118dd752184e2c Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Thu, 5 Sep 2019 13:10:08 -0700 Subject: [PATCH 29/30] Fix spelling. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 6a7e621d28a7..791a4f8371ea 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -130,7 +130,7 @@ class WasmVm : public Logger::Loggable { /** * Load the WASM code from a file. Return true on success. Once the module is loaded it can be - * queried, e.g. to see which version of emscriptent support is rquired. After loading, the + * queried, e.g. to see which version of emscripten support is required. After loading, the * appropriate ABI callbacks can be registered and then the module can be link()ed (see below). * @param code the WASM binary code (or registered NullVm plugin name). * @param allow_precompiled if true, allows supporting VMs (e.g. WAVM) to load the binary @@ -162,8 +162,8 @@ class WasmVm : public Logger::Loggable { uint64_t heap_base_pointer) PURE; /** - * Iniitalie globals (including calling global constructors) and call the 'start' function. Prior - * to calling start() the module should be load()ed, ABI callbacks should be registered + * Initialize globals (including calling global constructors) and call the 'start' function. + * Prior to calling start() the module should be load()ed, ABI callbacks should be registered * (registerCallback), the module link()ed, and any exported functions should be gotten * (getFunction). * @param vm_context a context which represents the caller: in this case Envoy itself. From de85b21d1db9e37c1e2f654c902dd127be08c1c5 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Thu, 5 Sep 2019 14:32:09 -0700 Subject: [PATCH 30/30] Add comments. Signed-off-by: John Plevyak --- source/extensions/common/wasm/wasm_vm.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 791a4f8371ea..42e5a08b0d33 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -203,7 +203,10 @@ class WasmVm : public Logger::Loggable { virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; /** - * Get a Word in the VM, returns true on success, false if the pointer is invalid. + * Get a VM native Word (e.g. sizeof(void*) or sizeof(size_t)) from VM memory, returns true on + * success, false if the pointer is invalid. WASM-32 VMs have 32-bit native words and WASM-64 VMs + * (not yet supported) will have 64-bit words as does the Null VM (compiled into 64-bit Envoy). + * This function can be used to chase pointers in VM memory. * @param pointer the offset into VM memory describing the start of VM native word size block. * @param data a pointer to a Word whose contents will be filled from the VM native word at * 'pointer'. @@ -213,6 +216,8 @@ class WasmVm : public Logger::Loggable { /** * Set a Word in the VM, returns true on success, false if the pointer is invalid. + * See getWord above for details. This function can be used (for example) to set indirect pointer + * return values (e.g. proxy_getHeaderHapValue(... const char** value_ptr, size_t* value_size). * @param pointer the offset into VM memory describing the start of VM native word size block. * @param data a Word whose contents will be written in VM native word size at 'pointer'. * @return whether or not the pointer was to a valid VM memory block of VM native word size.