diff --git a/Cargo.lock b/Cargo.lock index abd7f0031..f8480a4c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2341,7 +2341,7 @@ dependencies = [ [[package]] name = "host" version = "0.0.0" -source = "git+https://github.com/fermyon/spin-componentize?rev=51c3fade751c4e364142719e42130943fd8b0a76#51c3fade751c4e364142719e42130943fd8b0a76" +source = "git+https://github.com/fermyon/spin-componentize?rev=b6d42fe41e5690844a661deb631d996a2b49debc#b6d42fe41e5690844a661deb631d996a2b49debc" dependencies = [ "anyhow", "async-trait", @@ -5155,7 +5155,7 @@ dependencies = [ [[package]] name = "spin-componentize" version = "0.1.0" -source = "git+https://github.com/fermyon/spin-componentize?rev=51c3fade751c4e364142719e42130943fd8b0a76#51c3fade751c4e364142719e42130943fd8b0a76" +source = "git+https://github.com/fermyon/spin-componentize?rev=b6d42fe41e5690844a661deb631d996a2b49debc#b6d42fe41e5690844a661deb631d996a2b49debc" dependencies = [ "anyhow", "wasm-encoder 0.26.0", @@ -5421,6 +5421,22 @@ dependencies = [ "wit-bindgen-rust", ] +[[package]] +name = "spin-sqlite" +version = "0.1.0" +dependencies = [ + "anyhow", + "once_cell", + "rand 0.8.5", + "rusqlite", + "spin-app", + "spin-core", + "spin-key-value", + "spin-world", + "tokio", + "wit-bindgen-wasmtime", +] + [[package]] name = "spin-templates" version = "1.3.0-pre0" @@ -5505,6 +5521,7 @@ dependencies = [ "spin-key-value-sqlite", "spin-loader", "spin-manifest", + "spin-sqlite", "tempfile", "tokio", "toml 0.5.11", @@ -6159,7 +6176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand 0.7.3", "static_assertions", ] @@ -6354,7 +6371,7 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-cap-std-sync" version = "0.0.0" -source = "git+https://github.com/fermyon/spin-componentize?rev=51c3fade751c4e364142719e42130943fd8b0a76#51c3fade751c4e364142719e42130943fd8b0a76" +source = "git+https://github.com/fermyon/spin-componentize?rev=b6d42fe41e5690844a661deb631d996a2b49debc#b6d42fe41e5690844a661deb631d996a2b49debc" dependencies = [ "anyhow", "async-trait", @@ -6402,7 +6419,7 @@ dependencies = [ [[package]] name = "wasi-common" version = "0.0.0" -source = "git+https://github.com/fermyon/spin-componentize?rev=51c3fade751c4e364142719e42130943fd8b0a76#51c3fade751c4e364142719e42130943fd8b0a76" +source = "git+https://github.com/fermyon/spin-componentize?rev=b6d42fe41e5690844a661deb631d996a2b49debc#b6d42fe41e5690844a661deb631d996a2b49debc" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 47782f23c..cf584d7f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,10 +115,10 @@ tracing = { version = "0.1", features = ["log"] } wasmtime-wasi = { version = "8.0.1", features = ["tokio"] } wasi-common-preview1 = { package = "wasi-common", version = "8.0.1" } wasmtime = { version = "8.0.1", features = ["component-model"] } -spin-componentize = { git = "https://github.com/fermyon/spin-componentize", rev = "51c3fade751c4e364142719e42130943fd8b0a76" } -wasi-host = { package = "host", git = "https://github.com/fermyon/spin-componentize", rev = "51c3fade751c4e364142719e42130943fd8b0a76" } -wasi-common = { git = "https://github.com/fermyon/spin-componentize", rev = "51c3fade751c4e364142719e42130943fd8b0a76" } -wasi-cap-std-sync = { git = "https://github.com/fermyon/spin-componentize", rev = "51c3fade751c4e364142719e42130943fd8b0a76" } +spin-componentize = { git = "https://github.com/fermyon/spin-componentize", rev = "b6d42fe41e5690844a661deb631d996a2b49debc" } +wasi-host = { package = "host", git = "https://github.com/fermyon/spin-componentize", rev = "b6d42fe41e5690844a661deb631d996a2b49debc" } +wasi-common = { git = "https://github.com/fermyon/spin-componentize", rev = "b6d42fe41e5690844a661deb631d996a2b49debc" } +wasi-cap-std-sync = { git = "https://github.com/fermyon/spin-componentize", rev = "b6d42fe41e5690844a661deb631d996a2b49debc" } [workspace.dependencies.bindle] git = "https://github.com/fermyon/bindle" diff --git a/crates/bindle/src/expander.rs b/crates/bindle/src/expander.rs index 4ba24d394..00c2c488f 100644 --- a/crates/bindle/src/expander.rs +++ b/crates/bindle/src/expander.rs @@ -123,6 +123,7 @@ async fn bindle_component_manifest( files: asset_group, allowed_http_hosts: local.wasm.allowed_http_hosts.clone(), key_value_stores: local.wasm.key_value_stores.clone(), + sqlite_databases: local.wasm.sqlite_databases.clone(), }, trigger: local.trigger.clone(), config: local.config.clone(), diff --git a/crates/key-value/Cargo.toml b/crates/key-value/Cargo.toml index b04977664..696851bfd 100644 --- a/crates/key-value/Cargo.toml +++ b/crates/key-value/Cargo.toml @@ -9,7 +9,7 @@ doctest = false [dependencies] anyhow = "1.0" -tokio = { version = "1", features = [ "macros" ] } +tokio = { version = "1", features = [ "macros", "sync" ] } spin-app = { path = "../app" } spin-core = { path = "../core" } spin-world = { path = "../world" } diff --git a/crates/key-value/src/lib.rs b/crates/key-value/src/lib.rs index 880bb6cc7..0d307ef3e 100644 --- a/crates/key-value/src/lib.rs +++ b/crates/key-value/src/lib.rs @@ -6,7 +6,7 @@ use std::{collections::HashSet, sync::Arc}; use table::Table; mod host_component; -mod table; +pub mod table; mod util; pub use host_component::{manager, KeyValueComponent}; diff --git a/crates/key-value/src/table.rs b/crates/key-value/src/table.rs index 9023482dc..3ff4b4b93 100644 --- a/crates/key-value/src/table.rs +++ b/crates/key-value/src/table.rs @@ -34,6 +34,7 @@ impl Table { /// /// This function will attempt to avoid reusing recently closed identifiers, but after 2^32 calls to this /// function they will start repeating. + #[allow(clippy::result_unit_err)] pub fn push(&mut self, value: V) -> Result { if self.tuples.len() == self.capacity as usize { Err(()) diff --git a/crates/loader/src/bindle/config.rs b/crates/loader/src/bindle/config.rs index a0a7cbade..eb73d0d5c 100644 --- a/crates/loader/src/bindle/config.rs +++ b/crates/loader/src/bindle/config.rs @@ -49,6 +49,8 @@ pub struct RawWasmConfig { pub allowed_http_hosts: Option>, /// Optional list of key-value stores the component is allowed to use. pub key_value_stores: Option>, + /// Optional list of SQLite databases the component is allowed to use. + pub sqlite_databases: Option>, /// Environment variables to be mapped inside the Wasm module at runtime. pub environment: Option>, } diff --git a/crates/loader/src/bindle/mod.rs b/crates/loader/src/bindle/mod.rs index 1eacc2941..74220567a 100644 --- a/crates/loader/src/bindle/mod.rs +++ b/crates/loader/src/bindle/mod.rs @@ -131,11 +131,13 @@ async fn core( let environment = raw.wasm.environment.unwrap_or_default(); let allowed_http_hosts = raw.wasm.allowed_http_hosts.unwrap_or_default(); let key_value_stores = raw.wasm.key_value_stores.unwrap_or_default(); + let sqlite_databases = raw.wasm.sqlite_databases.unwrap_or_default(); let wasm = WasmConfig { environment, mounts, allowed_http_hosts, key_value_stores, + sqlite_databases, }; let config = raw.config.unwrap_or_default(); Ok(CoreComponent { diff --git a/crates/loader/src/local/config.rs b/crates/loader/src/local/config.rs index 02ffc8ef2..7d16064c2 100644 --- a/crates/loader/src/local/config.rs +++ b/crates/loader/src/local/config.rs @@ -142,6 +142,8 @@ pub struct RawWasmConfig { pub allowed_http_hosts: Option>, /// Optional list of key-value stores the component is allowed to use. pub key_value_stores: Option>, + /// Optional list of sqlite databases the component is allowed to use. + pub sqlite_databases: Option>, /// Environment variables to be mapped inside the Wasm module at runtime. pub environment: Option>, } diff --git a/crates/loader/src/local/mod.rs b/crates/loader/src/local/mod.rs index 63cbaeb91..a257357b6 100644 --- a/crates/loader/src/local/mod.rs +++ b/crates/loader/src/local/mod.rs @@ -204,11 +204,13 @@ async fn core( let environment = raw.wasm.environment.unwrap_or_default(); let allowed_http_hosts = raw.wasm.allowed_http_hosts.unwrap_or_default(); let key_value_stores = raw.wasm.key_value_stores.unwrap_or_default(); + let sqlite_databases = raw.wasm.sqlite_databases.unwrap_or_default(); let wasm = WasmConfig { environment, mounts, allowed_http_hosts, key_value_stores, + sqlite_databases, }; let config = raw.config.unwrap_or_default(); Ok(CoreComponent { diff --git a/crates/manifest/src/lib.rs b/crates/manifest/src/lib.rs index a099051e7..ff2e7a657 100644 --- a/crates/manifest/src/lib.rs +++ b/crates/manifest/src/lib.rs @@ -270,6 +270,8 @@ pub struct WasmConfig { pub allowed_http_hosts: Vec, /// Optional list of key-value stores the component is allowed to use. pub key_value_stores: Vec, + /// Optional list of sqlite databases the component is allowed to use. + pub sqlite_databases: Vec, } /// Directory mount for the assets of a component. diff --git a/crates/sqlite/Cargo.toml b/crates/sqlite/Cargo.toml new file mode 100644 index 000000000..4631e9f26 --- /dev/null +++ b/crates/sqlite/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "spin-sqlite" +version = "0.1.0" +edition = "2021" + +[dependencies] +spin-core = { path = "../core" } +spin-app = { path = "../app" } +spin-key-value = { path = "../key-value" } +spin-world = { path = "../world" } +anyhow = "1.0" +wit-bindgen-wasmtime = { workspace = true } +rusqlite = { version = "0.29.0", features = [ "bundled" ] } +rand = "0.8" +once_cell = "1" +tokio = "1" \ No newline at end of file diff --git a/crates/sqlite/src/host_component.rs b/crates/sqlite/src/host_component.rs new file mode 100644 index 000000000..2fde3f123 --- /dev/null +++ b/crates/sqlite/src/host_component.rs @@ -0,0 +1,93 @@ +use std::{ + collections::HashMap, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use once_cell::sync::OnceCell; +use rusqlite::Connection; +use spin_app::{AppComponent, DynamicHostComponent}; +use spin_core::HostComponent; +use spin_world::sqlite; + +use crate::SqliteImpl; + +#[derive(Debug, Clone)] +pub enum DatabaseLocation { + InMemory, + Path(PathBuf), +} + +/// A connection to a sqlite database +pub struct SqliteConnection { + location: DatabaseLocation, + connection: OnceCell>>, +} + +impl SqliteConnection { + pub fn new(location: DatabaseLocation) -> Self { + Self { + location, + connection: OnceCell::new(), + } + } +} + +impl ConnectionManager for SqliteConnection { + fn get_connection(&self) -> Result>, sqlite::Error> { + let connection = self + .connection + .get_or_try_init(|| -> Result<_, sqlite::Error> { + let c = match &self.location { + DatabaseLocation::InMemory => Connection::open_in_memory(), + DatabaseLocation::Path(path) => Connection::open(path), + } + .map_err(|e| sqlite::Error::Io(e.to_string()))?; + Ok(Arc::new(Mutex::new(c))) + })? + .clone(); + Ok(connection) + } +} + +pub trait ConnectionManager: Send + Sync { + fn get_connection(&self) -> Result>, sqlite::Error>; +} + +pub struct SqliteComponent { + connection_managers: HashMap>, +} + +impl SqliteComponent { + pub fn new(connection_managers: HashMap>) -> Self { + Self { + connection_managers, + } + } +} + +impl HostComponent for SqliteComponent { + type Data = super::SqliteImpl; + + fn add_to_linker( + linker: &mut spin_core::Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> anyhow::Result<()> { + sqlite::add_to_linker(linker, get) + } + + fn build_data(&self) -> Self::Data { + SqliteImpl::new(self.connection_managers.clone()) + } +} + +impl DynamicHostComponent for SqliteComponent { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { + let allowed_databases = component + .get_metadata(crate::DATABASES_KEY)? + .unwrap_or_default(); + data.component_init(allowed_databases); + // TODO: allow dynamically updating connection manager + Ok(()) + } +} diff --git a/crates/sqlite/src/lib.rs b/crates/sqlite/src/lib.rs new file mode 100644 index 000000000..2a67d8ecb --- /dev/null +++ b/crates/sqlite/src/lib.rs @@ -0,0 +1,146 @@ +mod host_component; + +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, Mutex, MutexGuard}, +}; + +use rusqlite::Connection; +use spin_app::MetadataKey; +use spin_core::async_trait; +use spin_world::sqlite::{self, Host}; + +pub use host_component::{ConnectionManager, DatabaseLocation, SqliteComponent, SqliteConnection}; +use spin_key_value::table; + +pub const DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); + +pub struct SqliteImpl { + allowed_databases: HashSet, + connections: table::Table>>, + client_manager: HashMap>, +} + +impl SqliteImpl { + pub fn new(client_manager: HashMap>) -> Self { + Self { + connections: table::Table::new(256), + allowed_databases: HashSet::new(), + client_manager, + } + } + + pub fn component_init(&mut self, allowed_databases: HashSet) { + self.allowed_databases = allowed_databases + } + + fn get_connection( + &self, + connection: sqlite::Connection, + ) -> Result, sqlite::Error> { + Ok(self + .connections + .get(connection) + .ok_or(sqlite::Error::InvalidConnection)? + .lock() + .unwrap()) + } +} + +#[async_trait] +impl Host for SqliteImpl { + async fn open( + &mut self, + database: String, + ) -> anyhow::Result> { + Ok(tokio::task::block_in_place(|| { + if !self.allowed_databases.contains(&database) { + return Err(sqlite::Error::AccessDenied); + } + self.connections + .push( + self.client_manager + .get(&database) + .ok_or(sqlite::Error::NoSuchDatabase)? + .get_connection()?, + ) + .map_err(|()| sqlite::Error::DatabaseFull) + })) + } + + async fn execute( + &mut self, + connection: sqlite::Connection, + query: String, + parameters: Vec, + ) -> anyhow::Result> { + Ok(tokio::task::block_in_place(|| { + let conn = self.get_connection(connection)?; + let mut statement = conn + .prepare_cached(&query) + .map_err(|e| sqlite::Error::Io(e.to_string()))?; + let columns = statement + .column_names() + .into_iter() + .map(ToOwned::to_owned) + .collect(); + let rows = statement + .query_map( + rusqlite::params_from_iter(convert_data(parameters.into_iter())), + |row| { + let mut values = vec![]; + for column in 0.. { + let value = row.get::(column); + if let Err(rusqlite::Error::InvalidColumnIndex(_)) = value { + break; + } + let value = value?.0; + values.push(value); + } + Ok(sqlite::RowResult { values }) + }, + ) + .map_err(|e| sqlite::Error::Io(e.to_string()))?; + let rows = rows + .into_iter() + .map(|r| r.map_err(|e| sqlite::Error::Io(e.to_string()))) + .collect::>()?; + Ok(sqlite::QueryResult { columns, rows }) + })) + } + + async fn close(&mut self, connection: sqlite::Connection) -> anyhow::Result<()> { + let _ = self.connections.remove(connection); + Ok(()) + } +} + +// A wrapper around sqlite::Value so that we can convert from rusqlite ValueRef +struct ValueWrapper(sqlite::Value); + +impl rusqlite::types::FromSql for ValueWrapper { + fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult { + let value = match value { + rusqlite::types::ValueRef::Null => sqlite::Value::Null, + rusqlite::types::ValueRef::Integer(i) => sqlite::Value::Integer(i), + rusqlite::types::ValueRef::Real(f) => sqlite::Value::Real(f), + rusqlite::types::ValueRef::Text(t) => { + sqlite::Value::Text(String::from_utf8(t.to_vec()).unwrap()) + } + rusqlite::types::ValueRef::Blob(b) => sqlite::Value::Blob(b.to_vec()), + }; + Ok(ValueWrapper(value)) + } +} + +fn convert_data( + arguments: impl Iterator, +) -> impl Iterator { + arguments.map(|a| match a { + sqlite::Value::Null => rusqlite::types::Value::Null, + sqlite::Value::Integer(i) => rusqlite::types::Value::Integer(i), + sqlite::Value::Real(r) => rusqlite::types::Value::Real(r), + sqlite::Value::Text(t) => rusqlite::types::Value::Text(t), + sqlite::Value::Blob(b) => rusqlite::types::Value::Blob(b), + }) +} diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml index 8dc471360..ee730df36 100644 --- a/crates/trigger/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -21,6 +21,7 @@ spin-key-value = { path = "../key-value" } spin-key-value-azure = { path = "../key-value-azure" } spin-key-value-redis = { path = "../key-value-redis" } spin-key-value-sqlite = { path = "../key-value-sqlite" } +spin-sqlite = { path = "../sqlite" } sanitize-filename = "0.4" serde = "1.0" serde_json = "1.0" diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 6b2241aff..f57f59907 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -106,6 +106,10 @@ where #[clap(long = "key-value", parse(try_from_str = parse_kv))] key_values: Vec<(String, String)>, + /// Run a sqlite migration against the default database + #[clap(long = "sqlite")] + sqlite_statements: Vec, + #[clap(long = "help-args-only", hide = true)] pub help_args_only: bool, } @@ -136,6 +140,7 @@ where let init_data = crate::HostComponentInitData { kv: self.key_values.clone(), + sqlite: self.sqlite_statements.clone(), }; let loader = TriggerLoader::new(working_dir, self.allow_transient_write); diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index d742a8955..0b76eeae2 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -123,6 +123,10 @@ impl TriggerExecutorBuilder { ) .await?, )?; + self.loader.add_dynamic_host_component( + &mut builder, + runtime_config::sqlite::build_component(&runtime_config, &init_data.sqlite)?, + )?; self.loader.add_dynamic_host_component( &mut builder, outbound_http::OutboundHttpComponent, @@ -154,6 +158,7 @@ impl TriggerExecutorBuilder { #[derive(Default)] // TODO: the implementation of Default is only for tests - would like to get rid of pub struct HostComponentInitData { kv: Vec<(String, String)>, + sqlite: Vec, } /// Execution context for a TriggerExecutor executing a particular App. @@ -335,8 +340,15 @@ pub fn decode_preinstantiation_error(e: anyhow::Error) -> anyhow::Error { if err_text.contains("unknown import") && err_text.contains("has not been defined") { // TODO: how to maintain this list? - let sdk_imported_interfaces = - &["config", "http", "key-value", "mysql", "postgres", "redis"]; + let sdk_imported_interfaces = &[ + "config", + "http", + "key-value", + "mysql", + "postgres", + "redis", + "sqlite", + ]; if sdk_imported_interfaces .iter() diff --git a/crates/trigger/src/locked.rs b/crates/trigger/src/locked.rs index c77d00d59..88f595bb1 100644 --- a/crates/trigger/src/locked.rs +++ b/crates/trigger/src/locked.rs @@ -17,6 +17,7 @@ use spin_manifest::{ Application, ApplicationInformation, ApplicationOrigin, ApplicationTrigger, CoreComponent, HttpConfig, HttpTriggerConfiguration, RedisConfig, TriggerConfig, }; +use spin_sqlite::DATABASES_KEY; pub const NAME_KEY: MetadataKey = MetadataKey::new("name"); pub const VERSION_KEY: MetadataKey = MetadataKey::new("version"); @@ -146,6 +147,7 @@ impl LockedAppBuilder { .string_option(DESCRIPTION_KEY, component.description) .string_array(ALLOWED_HTTP_HOSTS_KEY, component.wasm.allowed_http_hosts) .string_array(KEY_VALUE_STORES_KEY, component.wasm.key_value_stores) + .string_array(DATABASES_KEY, component.wasm.sqlite_databases) .take(); let source = { diff --git a/crates/trigger/src/runtime_config.rs b/crates/trigger/src/runtime_config.rs index 34ccc8178..64d2a7313 100644 --- a/crates/trigger/src/runtime_config.rs +++ b/crates/trigger/src/runtime_config.rs @@ -1,5 +1,6 @@ pub mod config_provider; pub mod key_value; +pub mod sqlite; use std::{ collections::HashMap, @@ -13,11 +14,14 @@ use serde::Deserialize; use self::{ config_provider::{ConfigProvider, ConfigProviderOpts}, key_value::{KeyValueStore, KeyValueStoreOpts}, + sqlite::SqliteDatabaseOpts, }; pub const DEFAULT_STATE_DIR: &str = ".spin"; const DEFAULT_LOGS_DIR: &str = "logs"; +const DEFAULT_SQLITE_DB_FILENAME: &str = "sqlite.db"; + /// RuntimeConfig allows multiple sources of runtime configuration to be /// queried uniformly. #[derive(Debug, Default)] @@ -92,6 +96,29 @@ impl RuntimeConfig { .unwrap_or_else(|| KeyValueStoreOpts::default_store_opts(self)) } + /// Return an iterator of named configured [`SqliteDatabase`]s. + pub fn sqlite_databases( + &self, + ) -> Result> { + let mut databases = HashMap::new(); + // Insert explicitly-configured databases + for opts in self.opts_layers() { + for (name, database) in &opts.sqlite_databases { + if !databases.contains_key(name) { + let store = database.build(name, opts)?; + databases.insert(name.to_owned(), store); + } + } + } + // Upsert default store + if !databases.contains_key("default") { + let store = SqliteDatabaseOpts::default(self) + .build("default", &RuntimeConfigOpts::default())?; + databases.insert("default".into(), store); + } + Ok(databases.into_iter()) + } + /// Set the state dir, overriding any other runtime config source. pub fn set_state_dir(&mut self, state_dir: impl Into) { self.overrides.state_dir = Some(state_dir.into()); @@ -131,6 +158,16 @@ impl RuntimeConfig { } } + /// Return a path to the sqlite DB used for key value storage if set. + pub fn sqlite_db_path(&self) -> Option { + if let Some(state_dir) = self.state_dir() { + // If the state dir is set, build the default path + Some(state_dir.join(DEFAULT_SQLITE_DB_FILENAME)) + } else { + None + } + } + /// Returns an iterator of RuntimeConfigOpts in order of decreasing precedence fn opts_layers(&self) -> impl Iterator { std::iter::once(&self.overrides).chain(self.files.iter().rev()) @@ -157,6 +194,9 @@ pub struct RuntimeConfigOpts { #[serde(rename = "key_value_store", default)] pub key_value_stores: HashMap, + #[serde(rename = "sqlite_database", default)] + pub sqlite_databases: HashMap, + #[serde(skip)] pub file_path: Option, } diff --git a/crates/trigger/src/runtime_config/sqlite.rs b/crates/trigger/src/runtime_config/sqlite.rs new file mode 100644 index 000000000..bcd519c98 --- /dev/null +++ b/crates/trigger/src/runtime_config/sqlite.rs @@ -0,0 +1,100 @@ +use std::{collections::HashMap, path::PathBuf, sync::Arc}; + +use crate::runtime_config::RuntimeConfig; +use anyhow::Context; +use spin_sqlite::{DatabaseLocation, SqliteComponent, SqliteConnection}; + +use super::RuntimeConfigOpts; + +pub type SqliteDatabase = Arc; + +pub(crate) fn build_component( + runtime_config: &RuntimeConfig, + sqlite_statements: &[String], +) -> anyhow::Result { + let databases: HashMap<_, _> = runtime_config + .sqlite_databases() + .context("Failed to build sqlite component")? + .into_iter() + .collect(); + execute_statements(sqlite_statements, &databases)?; + Ok(SqliteComponent::new(databases)) +} + +fn execute_statements( + statements: &[String], + databases: &HashMap>, +) -> anyhow::Result<()> { + if !statements.is_empty() { + if let Some(default) = databases.get("default") { + let c = default.get_connection().context( + "could not get connection to default database in order to execute statements", + )?; + let c = c.lock().unwrap(); + for m in statements { + if let Some(file) = m.strip_prefix('@') { + let sql = std::fs::read_to_string(file).with_context(|| { + format!("could not read file '{file}' containing sql statements") + })?; + c.execute_batch(&sql) + .with_context(|| format!("failed to execute sql from file '{file}'"))?; + } else { + c.execute(m, []) + .with_context(|| format!("failed to execute statement: '{m}'"))?; + } + } + } + } + Ok(()) +} + +// Holds deserialized options from a `[sqlite_database.]` runtime config section. +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum SqliteDatabaseOpts { + Spin(SpinSqliteDatabaseOpts), +} + +impl SqliteDatabaseOpts { + pub fn default(runtime_config: &RuntimeConfig) -> Self { + Self::Spin(SpinSqliteDatabaseOpts::default(runtime_config)) + } + + pub fn build( + &self, + name: &str, + config_opts: &RuntimeConfigOpts, + ) -> anyhow::Result { + match self { + Self::Spin(opts) => opts.build(name, config_opts), + } + } +} + +#[derive(Clone, Debug, Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SpinSqliteDatabaseOpts { + pub path: Option, +} + +impl SpinSqliteDatabaseOpts { + pub fn default(runtime_config: &RuntimeConfig) -> Self { + // If the state dir is set, build the default path + let path = runtime_config.state_dir(); + Self { path } + } + + fn build(&self, name: &str, config_opts: &RuntimeConfigOpts) -> anyhow::Result { + let location = match self.path.as_ref() { + Some(path) => { + let path = super::resolve_config_path(path, config_opts)?; + // Create the store's parent directory if necessary + std::fs::create_dir_all(path.parent().unwrap()) + .context("Failed to create sqlite database directory")?; + DatabaseLocation::Path(path.join(format!("{name}.db"))) + } + None => DatabaseLocation::InMemory, + }; + Ok(Arc::new(SqliteConnection::new(location))) + } +} diff --git a/docs/content/sips/013-sqlite.md b/docs/content/sips/013-sqlite.md new file mode 100644 index 000000000..adeb2014e --- /dev/null +++ b/docs/content/sips/013-sqlite.md @@ -0,0 +1,173 @@ +title = "SIP 013 - sqlite" +template = "main" +date = "2023-04-17:00:00Z" +--- + +Summary: Provide a generic interface for access to a sqlite databases + +Owner(s): ryan.levick@fermyon.com + +Created: Apr 17, 2023 + +## Background + +Spin currently supports two database types: mysql and [postgres](https://developer.fermyon.com/cloud/data-postgres) which both require the user to provide their own database that is exposed to users through the SDK. This is largely a stopgap until sockets are supported in wasi and there is no longer a need for bespoke clients for these databases (users can bring their favorite client libraries instead). + +In contrast to the these other interfaces, the sqlite implementation would easily allow local Spin deployment to use a local sqlite database file, and it provides those hosting Spin deployment envionments (e.g., Fermyon Cloud) to implement lightweight sqlite implementations. In short, a sqlite interface in Spin would allow for a "zero config" experience when users want to work with a SQL database. + +### What about `wasi-sql`? + +[`wasi-sql`](https://github.com/WebAssembly/wasi-sql) is a work-in-progress spec for a generic SQL interface that aims to support "the features commonly used by 80% of user application". It is likely that when `wasi-sql` is more mature users will be able to successfully use functionality based on the `wasi-sql` interface to interact with a sqlite databases. However, there are still reasons that a dedicated sqlite interface would still be useful: + +* For the 20% of use cases where `wasi-sql` is too generic, a dedicated `sqlite` interface can provide that functionality. +* The `wasi-sql` spec is under active investigation, and there are large questions about how to best support such a wide breadth of sql flavors. This implementation can help clarify those questions and push upstream work further along. + +## Proposal + +In order to support sqlite, the following need to be added to Spin: + +- A `WIT` file that defines the sqlite interface +- SDK implementations for various programming languages +- A default local sqlite store (note: Spin already uses sqlite for the KV implementation so this should be trivial) +- Potentially runtime configuration for configuring how sqlite is provisioned. +- Potentially a mechansim for handling database migrations + +### Interface (`.wit`) + +We will start with the `wasi-sql` interface but deliberately change that interface as to better match sqlite semantics. This will ensure that we're not simply implementing early versions of the `wasi-sql` interface while still having good answers for why the interface differs when it does. + +Like `wasi-sql` and the key-value store, we model resources such as database connections as pseudo-[resource handles](https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md#item-resource) which may be created using an `open` function and disposed using a `close` function. Each operation on a connection is a function which accepts a handle as its first parameter. + +Note that the syntax of the following `WIT` file matches the `wit-bindgen` version currently used by Spin, which is out-of-date with respect to the latest [`WIT` specification](https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md) and implementation. Once we're able to update `wit-bindgen`, we'll update the syntax of all the Spin `WIT` files, including this one. + +```fsharp +// A handle to an open sqlite instance +type connection = u32 + +// The set of errors which may be raised by functions in this interface +variant error { + // A database with the supplied name does not exist + no-such-database, + // The requesting component does not have access to the specified database (which may or may not exist). + access-denied, + // The provided connection is not valid + invalid-connection, + // The database has reached its capacity + database-full, + // Some implementation-specific error has occurred (e.g. I/O) + io(string) +} + +// Open a connection to a named database instance. +// +// If `database` is "default", the default instance is opened. +// +// `error::no-such-database` will be raised if the `name` is not recognized. +open: func(name: string) -> expected + +// Execute a statement +execute: func(conn: connection, statement: string, parameters: list) -> expected + +// Query data +query: func(conn: connection, query: string, parameters: list) -> expected + +// Close the specified `connection`. +close: func(conn: connection) + +// A result of a query +record query-result { + // The names of the columns retrieved in the query + columns: list, + // The row results each containing the values for all the columns for a given row + rows: list, +} + +// A set of values for each of the columns in a query-result +record row-result { + values: list +} + +// The values used in statements/queries and returned in query results +variant value { + integer(s64), + real(float64), + text(string), + blob(list), + null +} +``` + +*Note: the pseudo-resource design was inspired by the interface of similar functions in [WASI preview 2](https://github.com/bytecodealliance/preview2-prototyping/blob/d56b8977a2b700432d1f7f84656d542f1d8854b0/wit/wasi.wit#L772-L794).* + +#### Interface open questions + +**TODO**: answer these questions +* `row-result` can be very large. Should we provide some paging mechanism or a different API that allows for reading subsets of the returned data? + * Crossing the wit boundary could potentially be expensive if the results are large enough. Giving the user control of how they read that data could be helpful. +* Is there really a need for query *and* execute functions since at the end of the day, they are basically equivalent? + +#### Database migrations + +Database tables typically require some sort of configuration in the form of database migrations to get table schemas into the correct state. To begin with a command line option supplied to `spin up` will be available for running any arbitrary SQL statements on start up and thus will be a place for users to run their migrations (i.e., `--sqlite "CREATE TABLE users..."`). It will be up to the user to provide idempotent statements such that running them multiple times does not produce unexpected results. + +##### Future approaches + +This CLI approach (while useful) is likely to not be sufficient for more advanced use cases. There are several alternative ways to address the need for migrations: +* Some mechanism for running spin components before others where the component receives the current schema version and decides whether or not to perform migrations. +* The spin component could expose a current schema version as an exported value type so that an exported function would not need to called. If the exported schema version does not match the current schema version, an exported migrate function then gets called. +* A spin component that gets called just after pre-initialization finishes. Similarly, this component would expose a schema version and have an exported migration function called when the exported schema version does not match the current schema version. +* Configuration option in spin.toml manifest for running arbitrary SQL instructions on start up (e.g., `sqlite.execute = "CREATE TABLE users..."`) + +It should be noted that many of these options are not mutually exclusive and we could introduce more than one (perhaps starting with one option that will mostly be replaced later with a more generalized approach). + +For now, we punt on this question and only provide a mechanism for running SQL statements on start up through the CLI. + +##### Alternatives + +An alternative approach that was considered but ultimately reject was to require the user to ensure that the database is in the correct state each time their trigger handler function is run (i.e., provide no bespoke mechanism for migrations - the user only has access to the database when their component runs). There are a few issues with taking such an approach: +* Schema tracking schemes (e.g., a "migrations" table) themselves require some sort of bootstrap step. +* This goes against the design principle of keeping components handler functions simple and single purpose. + +#### Implementation requirements + +### Minimum capacity limits + +In addition to the above interface, we specify a few additional implementation requirements which guest components may rely on. At minimum, an conforming implementation must support: +* At least 10,000 rows +* Text and blob sizes of at least 1 megabyte +* Maximum number of columns of at least 32 + +**TODO**: Open questions: +* Assumed sqlite version? + * Semantics may change slightly depending on the sqlite version. It's unlikely that we'll be able to match the exact versions between whatever sqlite implementation spin users, Fermyon Cloud, and the user (if they decide to create their own databases manually). Having some guidance on which versions are expected to work might make it easier to guide the user down the right path. + +#### Built-in local database + +By default, each app will have its own default database which is independent of all other apps. For local apps, the database will be stored by default in a hidden `.spin` directory adjacent to the app's `spin.toml`. For remote apps, the user should be able to rely on a default database as well. It is up to the implementor how this remote database is exposed (i.e., by having a sqlite database on disk or by using a third party network enabled database like [Turso](https://turso.tech)). + +#### Granting access to components + +By default, a given component of an app will _not_ have access to any database. Access must be granted specifically to each component via the following `spin.toml` syntax: + +```toml +sqlite_databases = ["", ""] +``` + +For example, a component could be given access to the default database using `sqlite_databases = ["default"]`. + +### Runtime Config + +Sqlite databases may be configured with `[sqlite_database.]` sections in the runtime config file: + +```toml +# The `default` config can be overridden +[sqlite_database.default] +path = ".spin/some-other-database.db" + +[sqlite_database.other] +path = ".spin/yet-another-database.db" +``` + +## Future work + +In the future we may want to try to unify the three SQL flavors we currently have support for (sqlite, mysql, and postgres). This may not be desirable if it becomes clear that unifying these three (fairly different) SQL flavors actually causes more confusion than is worthwhile. diff --git a/sdk/rust/src/lib.rs b/sdk/rust/src/lib.rs index 248dbcba5..02927d60b 100644 --- a/sdk/rust/src/lib.rs +++ b/sdk/rust/src/lib.rs @@ -8,6 +8,9 @@ pub mod outbound_http; /// Key/Value storage. pub mod key_value; +/// Sqlite +pub mod sqlite; + /// Exports the procedural macros for writing handlers for Spin components. pub use spin_macro::*; diff --git a/sdk/rust/src/sqlite.rs b/sdk/rust/src/sqlite.rs new file mode 100644 index 000000000..bb3090d61 --- /dev/null +++ b/sdk/rust/src/sqlite.rs @@ -0,0 +1,107 @@ +wit_bindgen_rust::import!("../../wit/ephemeral/sqlite.wit"); + +use sqlite::Connection as RawConnection; + +/// Errors which may be raised by the methods of `Store` +pub type Error = sqlite::Error; + +/// A parameter used when executing a sqlite statement +pub type ValueParam<'a> = sqlite::ValueParam<'a>; +/// A single column's result from a database query +pub type ValueResult = sqlite::ValueResult; + +/// Represents a store in which key value tuples may be placed +#[derive(Debug)] +pub struct Connection(RawConnection); + +impl Connection { + /// Open a connection to the default database + pub fn open_default() -> Result { + Ok(Self(sqlite::open("default")?)) + } + + /// Open a connection + pub fn open(database: &str) -> Result { + Ok(Self(sqlite::open(database)?)) + } + + /// Execute a statement against the database + pub fn execute( + &self, + query: &str, + parameters: &[ValueParam<'_>], + ) -> Result { + sqlite::execute(self.0, query, parameters) + } +} + +impl sqlite::QueryResult { + /// Get all the rows for this query result + pub fn rows(&self) -> impl Iterator> { + self.rows.iter().map(|r| Row { + columns: self.columns.as_slice(), + result: r, + }) + } +} + +/// A database row result +pub struct Row<'a> { + columns: &'a [String], + result: &'a sqlite::RowResult, +} + +impl<'a> Row<'a> { + /// Get a value by its column name + pub fn get>(&self, column: &str) -> Option { + let i = self.columns.iter().position(|c| c == column)?; + self.result.get(i) + } +} + +impl sqlite::RowResult { + pub fn get<'a, T: TryFrom<&'a ValueResult>>(&'a self, index: usize) -> Option { + self.values.get(index).and_then(|c| c.try_into().ok()) + } +} + +impl<'a> TryFrom<&'a ValueResult> for bool { + type Error = (); + + fn try_from(value: &'a ValueResult) -> Result { + match value { + ValueResult::Integer(i) => Ok(*i != 0), + _ => Err(()), + } + } +} + +impl<'a> TryFrom<&'a ValueResult> for u32 { + type Error = (); + + fn try_from(value: &'a ValueResult) -> Result { + match value { + ValueResult::Integer(i) => Ok(*i as u32), + _ => Err(()), + } + } +} + +impl<'a> TryFrom<&'a ValueResult> for &'a str { + type Error = (); + + fn try_from(value: &'a ValueResult) -> Result { + match value { + ValueResult::Text(s) => Ok(s.as_str()), + _ => Err(()), + } + } +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for Error {} diff --git a/wit/ephemeral/sqlite.wit b/wit/ephemeral/sqlite.wit new file mode 100644 index 000000000..8bf9384b5 --- /dev/null +++ b/wit/ephemeral/sqlite.wit @@ -0,0 +1,50 @@ +// A handle to an open sqlite instance +type connection = u32 + +// The set of errors which may be raised by functions in this interface +variant error { + // The host does not recognize the database name requested. + no-such-database, + // The requesting component does not have access to the specified database (which may or may not exist). + access-denied, + // The provided connection is not valid + invalid-connection, + // The database has reached its capacity + database-full, + // Some implementation-specific error has occurred (e.g. I/O) + io(string) +} + +// Open a connection to a named database instance. +// +// If `database` is "default", the default instance is opened. +// +// `error::no-such-database` will be raised if the `name` is not recognized. +open: func(name: string) -> expected + +// Execute a statement +execute: func(conn: connection, statement: string, parameters: list) -> expected + +// Close the specified `connection`. +close: func(conn: connection) + +// A result of a query +record query-result { + // The names of the columns retrieved in the query + columns: list, + // the row results each containing the values for all the columns for a given row + rows: list, +} + +// A set of values for each of the columns in a query-result +record row-result { + values: list +} + +variant value { + integer(s64), + real(float64), + text(string), + blob(list), + null +} diff --git a/wit/preview2/reactor.wit b/wit/preview2/reactor.wit index 954aa38a9..042c1da4b 100644 --- a/wit/preview2/reactor.wit +++ b/wit/preview2/reactor.wit @@ -2,6 +2,7 @@ default world reactor { import config: pkg.config import postgres: pkg.postgres import mysql: pkg.mysql + import sqlite: pkg.sqlite import redis: pkg.redis import key-value: pkg.key-value import http: pkg.http diff --git a/wit/preview2/sqlite.wit b/wit/preview2/sqlite.wit new file mode 100644 index 000000000..a539b905e --- /dev/null +++ b/wit/preview2/sqlite.wit @@ -0,0 +1,52 @@ +default interface sqlite { + // A handle to an open sqlite instance + type connection = u32 + + // The set of errors which may be raised by functions in this interface + variant error { + // The host does not recognize the database name requested. + no-such-database, + // The requesting component does not have access to the specified database (which may or may not exist). + access-denied, + // The provided connection is not valid + invalid-connection, + // The database has reached its capacity + database-full, + // Some implementation-specific error has occurred (e.g. I/O) + io(string) + } + + // Open a connection to a named database instance. + // + // If `database` is "default", the default instance is opened. + // + // `error::no-such-database` will be raised if the `name` is not recognized. + open: func(database: string) -> result + + // Execute a statement returning back data if there is any + execute: func(conn: connection, statement: string, parameters: list) -> result + + // Close the specified `connection`. + close: func(conn: connection) + + // A result of a query + record query-result { + // The names of the columns retrieved in the query + columns: list, + // the row results each containing the values for all the columns for a given row + rows: list, + } + + // A set of values for each of the columns in a query-result + record row-result { + values: list + } + + variant value { + integer(s64), + real(float64), + text(string), + blob(list), + null + } +}