feat(sqlite): SQLite Module for Hermes

hermes/Earthfile

@@ -84,6 +84,7 @@ test-wasm-integration:
     COPY ../wasm/integration-test/hashing+build/hashing.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/localtime+build/localtime.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/logger+build/logger.wasm ../wasm/test-components/
+    COPY ../wasm/integration-test/sqlite+build/sqlite.wasm ../wasm/test-components/
     COPY ../wasm/integration-test/smoke-test+build/smoke-test.wasm ../wasm/test-components/
 
     # List all WASM integration tests/benches and also run them.

hermes/bin/src/runtime_extensions/hermes/sqlite/connection/host.rs

@@ -2,9 +2,7 @@
 
 //! `SQLite` connection object host implementation for WASM runtime.
 
-use libsqlite3_sys::sqlite3;
-
-use super::core;
+use super::{super::state, core};
 use crate::{
     runtime_context::HermesRuntimeContext,
     runtime_extensions::bindings::hermes::sqlite::api::{
@@ -26,9 +24,12 @@ impl HostSqlite for HermesRuntimeContext {
     fn close(
         &mut self, resource: wasmtime::component::Resource<Sqlite>,
     ) -> wasmtime::Result<Result<(), Errno>> {
-        let db_ptr: *mut sqlite3 = resource.rep() as *mut _;
+        let db_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_db_state()
+            .delete_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `close`"))?;
 
-        Ok(core::close(db_ptr))
+        Ok(core::close(db_ptr as *mut _))
     }
 
     /// Retrieves the numeric result code for the most recent failed `SQLite` operation on
@@ -40,9 +41,12 @@ impl HostSqlite for HermesRuntimeContext {
     fn errcode(
         &mut self, resource: wasmtime::component::Resource<Sqlite>,
     ) -> wasmtime::Result<Option<ErrorInfo>> {
-        let db_ptr: *mut sqlite3 = resource.rep() as *mut _;
+        let db_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_db_state()
+            .get_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `errcode`"))?;
 
-        Ok(core::errcode(db_ptr))
+        Ok(core::errcode(db_ptr as *mut _))
     }
 
     /// Compiles SQL text into byte-code that will do the work of querying or updating the
@@ -61,16 +65,27 @@ impl HostSqlite for HermesRuntimeContext {
     fn prepare(
         &mut self, resource: wasmtime::component::Resource<Sqlite>, sql: String,
     ) -> wasmtime::Result<Result<wasmtime::component::Resource<Statement>, Errno>> {
-        let db_ptr: *mut sqlite3 = resource.rep() as *mut _;
+        let db_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_db_state()
+            .get_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `prepare`"))?;
 
-        let result = core::prepare(db_ptr, sql.as_str());
+        let result = core::prepare(db_ptr as *mut _, sql.as_str());
 
         match result {
             Ok(stmt_ptr) => {
                 if stmt_ptr.is_null() {
                     Ok(Err(Errno::ReturnedNullPointer))
                 } else {
-                    Ok(Ok(wasmtime::component::Resource::new_own(stmt_ptr as u32)))
+                    let stmt_id =
+                        state::InternalState::get_or_create_resource(self.app_name().clone())
+                            .get_stmt_state()
+                            .allocate_object(stmt_ptr as _)
+                            .ok_or_else(|| {
+                                wasmtime::Error::msg("Internal state error while calling `prepare`")
+                            })?;
+
+                    Ok(Ok(wasmtime::component::Resource::new_own(stmt_id)))
                 }
             },
             Err(errno) => Ok(Err(errno)),
@@ -86,15 +101,22 @@ impl HostSqlite for HermesRuntimeContext {
     fn execute(
         &mut self, resource: wasmtime::component::Resource<Sqlite>, sql: String,
     ) -> wasmtime::Result<Result<(), Errno>> {
-        let db_ptr: *mut sqlite3 = resource.rep() as *mut _;
+        let db_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_db_state()
+            .get_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `execute`"))?;
 
-        Ok(core::execute(db_ptr, sql.as_str()))
+        Ok(core::execute(db_ptr as *mut _, sql.as_str()))
     }
 
     fn drop(&mut self, rep: wasmtime::component::Resource<Sqlite>) -> wasmtime::Result<()> {
-        let db_ptr: *mut sqlite3 = rep.rep() as *mut _;
+        let db_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_db_state()
+            .delete_object_by_id(rep.rep());
 
-        let _ = core::close(db_ptr);
+        if let Some(db_ptr) = db_ptr {
+            let _ = core::close(db_ptr as *mut _);
+        }
 
         Ok(())
     }

hermes/bin/src/runtime_extensions/hermes/sqlite/host.rs

@@ -1,10 +1,7 @@
 //! `SQLite` host implementation for WASM runtime.
 
-use anyhow::Ok;
-
-use super::core;
+use super::{core, state};
 use crate::{
-    app::HermesAppName,
     runtime_context::HermesRuntimeContext,
     runtime_extensions::bindings::hermes::sqlite::api::{Errno, Host, Sqlite},
 };
@@ -25,10 +22,18 @@ impl Host for HermesRuntimeContext {
     fn open(
         &mut self, readonly: bool, memory: bool,
     ) -> wasmtime::Result<Result<wasmtime::component::Resource<Sqlite>, Errno>> {
-        // TODO: use actual app name for this
-        let app_name = HermesAppName(String::from("tmp"));
+        match core::open(readonly, memory, self.app_name().clone()) {
+            Ok(db_ptr) => {
+                let db_id = state::InternalState::get_or_create_resource(self.app_name().clone())
+                    .get_db_state()
+                    .allocate_object(db_ptr as _)
+                    .ok_or_else(|| {
+                        wasmtime::Error::msg("Internal state error while calling `open`")
+                    })?;
 
-        Ok(core::open(readonly, memory, app_name)
-            .map(|db_ptr| wasmtime::component::Resource::new_own(db_ptr as u32)))
+                Ok(Ok(wasmtime::component::Resource::new_own(db_id)))
+            },
+            Err(err) => Ok(Err(err)),
+        }
     }
 }

hermes/bin/src/runtime_extensions/hermes/sqlite/mod.rs

@@ -3,6 +3,7 @@
 mod connection;
 mod core;
 mod host;
+mod state;
 mod statement;
 
 /// Advise Runtime Extensions of a new context

hermes/bin/src/runtime_extensions/hermes/sqlite/state.rs

@@ -0,0 +1,108 @@
+// cspell: words mapref
+
+//! Internal state implementation for the `SQLite` module.
+
+use std::collections::HashMap;
+
+use dashmap::{mapref::one::RefMut, DashMap};
+use once_cell::sync::Lazy;
+
+use crate::app::HermesAppName;
+
+/// Represents an individual state for a particular object.
+#[derive(Debug)]
+pub(crate) struct ResourceObjectState {
+    /// A map holding key-value pairs of an object ID and a value.
+    id_map: HashMap<u32, usize>,
+    /// The current incremental state of ID.
+    current_id: Option<u32>,
+}
+
+/// Represents the state of resources.
+pub(crate) struct ResourceState {
+    /// The state of database object.
+    db_state: ResourceObjectState,
+    /// The state of database statement object.
+    stmt_state: ResourceObjectState,
+}
+
+impl ResourceObjectState {
+    /// Create a new `ResourceObjectState` with initial state.
+    fn new() -> Self {
+        Self {
+            id_map: HashMap::new(),
+            current_id: None,
+        }
+    }
+
+    /// Adds a value into the resource. If it does not exist, allocate one and returns the
+    /// new created key ID. In case of the key ID is running out of numbers, returns
+    /// `None`.
+    pub(super) fn allocate_object(&mut self, object_ptr: usize) -> Option<u32> {
+        if let Some((existing_id, _)) = self.id_map.iter().find(|(_, val)| val == &&object_ptr) {
+            Some(*existing_id)
+        } else {
+            let (new_id, is_overflow) = self
+                .current_id
+                .map_or_else(|| (0, false), |id| id.overflowing_add(1));
+
+            if is_overflow {
+                None
+            } else {
+                self.id_map.insert(new_id, object_ptr);
+                self.current_id = Some(new_id);
+                Some(new_id)
+            }
+        }
+    }
+
+    /// Retrieves a value according to its key ID.
+    pub(super) fn get_object_by_id(&self, id: u32) -> Option<usize> {
+        self.id_map.get(&id).map(ToOwned::to_owned)
+    }
+
+    /// Deletes a value according to its key ID, and returns the removed value if exists.
+    pub(super) fn delete_object_by_id(&mut self, id: u32) -> Option<usize> {
+        self.id_map.remove(&id)
+    }
+}
+
+impl ResourceState {
+    /// Create a new `ResourceState` with initial state.
+    pub(super) fn new() -> Self {
+        Self {
+            db_state: ResourceObjectState::new(),
+            stmt_state: ResourceObjectState::new(),
+        }
+    }
+
+    /// Gets the state for managing database objects.
+    pub(super) fn get_db_state(&mut self) -> &mut ResourceObjectState {
+        &mut self.db_state
+    }
+
+    /// Gets the state for managing statement objects.
+    pub(super) fn get_stmt_state(&mut self) -> &mut ResourceObjectState {
+        &mut self.stmt_state
+    }
+}
+
+/// Map of app name to resource holder
+type State = DashMap<HermesAppName, ResourceState>;
+
+/// Global state to hold `SQLite` resources.
+static SQLITE_INTERNAL_STATE: Lazy<State> = Lazy::new(DashMap::new);
+
+/// Represents the internal state object for `SQLite` module.
+pub(crate) struct InternalState;
+
+impl InternalState {
+    /// Set the state according to the app context.
+    pub(crate) fn get_or_create_resource<'a>(
+        app_name: HermesAppName,
+    ) -> RefMut<'a, HermesAppName, ResourceState> {
+        SQLITE_INTERNAL_STATE
+            .entry(app_name)
+            .or_insert_with(ResourceState::new)
+    }
+}

hermes/bin/src/runtime_extensions/hermes/sqlite/statement/host.rs

@@ -1,8 +1,6 @@
 //! `SQLite` statement host implementation for WASM runtime.
 
-use libsqlite3_sys::sqlite3_stmt;
-
-use super::core;
+use super::{super::state, core};
 use crate::{
     runtime_context::HermesRuntimeContext,
     runtime_extensions::bindings::hermes::sqlite::api::{Errno, HostStatement, Statement, Value},
@@ -18,11 +16,14 @@ impl HostStatement for HermesRuntimeContext {
     fn bind(
         &mut self, resource: wasmtime::component::Resource<Statement>, index: u32, value: Value,
     ) -> wasmtime::Result<Result<(), Errno>> {
-        let stmt_ptr: *mut sqlite3_stmt = resource.rep() as *mut _;
+        let stmt_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_stmt_state()
+            .get_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `bind`"))?;
 
         let index = i32::try_from(index).map_err(|_| Errno::ConvertingNumeric)?;
 
-        Ok(core::bind(stmt_ptr, index, value))
+        Ok(core::bind(stmt_ptr as *mut _, index, value))
     }
 
     /// Advances a statement to the next result row or to completion.
@@ -32,9 +33,12 @@ impl HostStatement for HermesRuntimeContext {
     fn step(
         &mut self, resource: wasmtime::component::Resource<Statement>,
     ) -> wasmtime::Result<Result<(), Errno>> {
-        let stmt_ptr: *mut sqlite3_stmt = resource.rep() as *mut _;
+        let stmt_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_stmt_state()
+            .get_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `step`"))?;
 
-        Ok(core::step(stmt_ptr))
+        Ok(core::step(stmt_ptr as *mut _))
     }
 
     /// Returns information about a single column of the current result row of a query.
@@ -53,11 +57,14 @@ impl HostStatement for HermesRuntimeContext {
     fn column(
         &mut self, resource: wasmtime::component::Resource<Statement>, index: u32,
     ) -> wasmtime::Result<Result<Value, Errno>> {
-        let stmt_ptr: *mut sqlite3_stmt = resource.rep() as *mut _;
+        let stmt_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_stmt_state()
+            .get_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `column`"))?;
 
         let index = i32::try_from(index).map_err(|_| Errno::ConvertingNumeric)?;
 
-        Ok(core::column(stmt_ptr, index))
+        Ok(core::column(stmt_ptr as *mut _, index))
     }
 
     /// Destroys a prepared statement object. If the most recent evaluation of the
@@ -73,15 +80,22 @@ impl HostStatement for HermesRuntimeContext {
     fn finalize(
         &mut self, resource: wasmtime::component::Resource<Statement>,
     ) -> wasmtime::Result<Result<(), Errno>> {
-        let stmt_ptr: *mut sqlite3_stmt = resource.rep() as *mut _;
+        let stmt_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_stmt_state()
+            .delete_object_by_id(resource.rep())
+            .ok_or_else(|| wasmtime::Error::msg("Internal state error while calling `finalize`"))?;
 
-        Ok(core::finalize(stmt_ptr))
+        Ok(core::finalize(stmt_ptr as *mut _))
     }
 
     fn drop(&mut self, resource: wasmtime::component::Resource<Statement>) -> wasmtime::Result<()> {
-        let stmt_ptr: *mut sqlite3_stmt = resource.rep() as *mut _;
+        let stmt_ptr = state::InternalState::get_or_create_resource(self.app_name().clone())
+            .get_stmt_state()
+            .delete_object_by_id(resource.rep());
 
-        let _ = core::finalize(stmt_ptr);
+        if let Some(stmt_ptr) = stmt_ptr {
+            let _ = core::finalize(stmt_ptr as *mut _);
+        }
 
         Ok(())
     }

wasm/integration-test/README.md

@@ -0,0 +1,29 @@
+# WASM Integration Test
+
+This directory comprises of integration test modules implemented for Hermes including both APIs of Hermes and WASI.
+
+## Flow
+
+Each module has its own `Earthfile`. The file has the `+build` target,
+to compile each module into `.wasm` component and execute with Hermes test engine.
+The output of each module resides at `wasm/test-components` from the project root.
+Some modules might be implemented in C or Rust or any languages that support WASM target compilation.
+
+## Adding a new module
+
+You can create a new directory inside `wasm/integration-test` if the test module hasn't been created yet, or modify the existing module.
+The new test module you created must have an `Earthfile` inside the directory with the `+build` target.
+The `+build` target must output a WASM component you want to test with.
+Make sure to setup the language of your choice properly.
+When you are working with the test module, firstly, you need to generate Hermes bindings.
+You can visit `wasm/wasi/Earthfile` for supported languages needed to generate Hermes bindings.
+
+## Notes
+
+### SQLite benchmark result
+
+- Test: simple sequential insertion between persistent and in-memory database (100 iterations)
+  - Persistent: 8,493,667 ns/iter (+/- 0)
+  - In-memory: 37,492,916 ns/iter (+/- 0)
+
+Tested on MacBook Pro with M3 Pro chip 12-core CPU, 18-core GPU, and 18GB unified memory.

wasm/integration-test/sqlite/.gitignore

@@ -0,0 +1,2 @@
+target/
+src/hermes.rs

wasm/integration-test/sqlite/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "sqlite-test-component"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+wit-bindgen = "0.24.0"