diff --git a/Cargo.lock b/Cargo.lock index e37b8f5..57d5bb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,9 +80,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -94,15 +94,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys", @@ -179,6 +179,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http", + "log", + "native-tls", + "serde", + "serde_json", + "url", +] + [[package]] name = "atty" version = "0.2.14" @@ -196,6 +210,32 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +dependencies = [ + "attohttpc", + "dirs", + "log", + "quick-xml", + "rust-ini", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba" +dependencies = [ + "thiserror", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -357,9 +397,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -367,9 +407,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle", @@ -544,6 +584,9 @@ name = "deranged" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +dependencies = [ + "serde", +] [[package]] name = "difflib" @@ -568,6 +611,16 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", ] [[package]] @@ -580,6 +633,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -591,6 +655,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "doc-comment" version = "0.3.3" @@ -620,9 +690,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", @@ -944,6 +1014,15 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "http" version = "0.2.9" @@ -1151,9 +1230,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" [[package]] name = "lock_api" @@ -1180,6 +1259,17 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "md-5" version = "0.9.1" @@ -1191,11 +1281,17 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -1212,6 +1308,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minidom" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278" +dependencies = [ + "rxml", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1372,6 +1477,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1551,6 +1666,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.33" @@ -1640,9 +1765,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" [[package]] name = "reqwest" @@ -1673,10 +1798,12 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg", ] @@ -1707,6 +1834,7 @@ dependencies = [ "reqwest", "rusoto_core", "rusoto_s3", + "rust-s3", "simple_logger", "snap", "sqlx", @@ -1801,7 +1929,7 @@ dependencies = [ "digest 0.9.0", "futures", "hex", - "hmac", + "hmac 0.11.0", "http", "hyper", "log", @@ -1815,6 +1943,49 @@ dependencies = [ "tokio", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac 0.12.1", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_derive", + "sha2 0.10.8", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1832,9 +2003,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.14" +version = "0.38.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +checksum = "d2f9da0cbd88f9f09e7814e388301c8414c51c62aa6ce1e4b5c551d49d96e531" dependencies = [ "bitflags 2.4.0", "errno", @@ -1856,6 +2027,23 @@ dependencies = [ "webpki", ] +[[package]] +name = "rxml" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" +dependencies = [ + "bytes", + "rxml_validation", + "smartstring", +] + +[[package]] +name = "rxml_validation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530" + [[package]] name = "ryu" version = "1.0.15" @@ -2026,6 +2214,17 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "snap" version = "1.1.0" @@ -2163,6 +2362,12 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stringprep" version = "0.1.4" @@ -2604,6 +2809,19 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.64" @@ -2743,7 +2961,7 @@ dependencies = [ [[package]] name = "workers" version = "0.1.0" -source = "git+https://github.com/threefoldtech/tokio-worker-pool.git#5dbb1fbc02a1f1bdfc21456d4d4f1b6b08577051" +source = "git+https://github.com/threefoldtech/tokio-worker-pool.git#70cbc96a547e6bdf9bd2e5341532438a48745e17" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index c529038..a56c57f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ simple_logger = {version = "1.0.1", optional = true} daemonize = { version = "0.5", optional = true } tempfile = { version = "3.3.0", optional = true } workers = { git="https://github.com/threefoldtech/tokio-worker-pool.git" } +rust-s3 = "0.33" [dependencies.polyfuse] branch = "master" diff --git a/src/lib.rs b/src/lib.rs index dc1dc28..56db8e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -344,7 +344,7 @@ where type Input = (Ino, PathBuf); type Output = (); - async fn run(&self, (ino, path): Self::Input) -> Self::Output { + async fn run(&mut self, (ino, path): Self::Input) -> Self::Output { if let Err(err) = self.upload(ino, &path).await { log::error!("failed to upload file ({:?}): {}", path, err); } diff --git a/src/store/mod.rs b/src/store/mod.rs index 3fbd5d5..3448f24 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -2,13 +2,14 @@ mod bs; pub mod dir; mod router; pub mod zdb; -pub mod s3; +pub mod s3store; use rand::seq::SliceRandom; use std::{collections::HashMap, pin::Pin}; pub use bs::BlockStore; use futures::Future; +use s3; lazy_static::lazy_static! { static ref STORES: HashMap = register_stores(); @@ -20,7 +21,7 @@ fn register_stores() -> HashMap { let mut m: HashMap = HashMap::default(); m.insert("dir".into(), dir::make); m.insert("zdb".into(), zdb::make); - m.insert("s3".into(), s3::make); + m.insert("s3".into(), s3store::make); m } @@ -57,6 +58,10 @@ pub enum Error { #[error("bucket creation error")] BucketCreationError, + #[error("invalid host")] + InvalidHost, + #[error("invalid url configuration")] + InvalidConfigs, // TODO: better display for the Box> #[error("multiple error: {0:?}")] Multiple(Box>), @@ -66,6 +71,10 @@ pub enum Error { #[error("url parse error: {0}")] Url(#[from] url::ParseError), + #[error("create bucket error")] + S3Error(#[from] s3::error::S3Error), + #[error("utf8 error")] + Utf8Error(#[from] std::str::Utf8Error), #[error("unknown store type '{0}'")] UnknownStore(String), #[error("invalid schema '{0}' expected '{1}'")] diff --git a/src/store/s3.rs b/src/store/s3.rs deleted file mode 100644 index 77c1bcb..0000000 --- a/src/store/s3.rs +++ /dev/null @@ -1,129 +0,0 @@ -use super::{Error, Result, Route, Store}; -use anyhow::Context; -use futures::Future; -use std::pin::Pin; - -use rusoto_core::{ByteStream, Region, RusotoError}; -use rusoto_s3::{ - CreateBucketError, CreateBucketRequest, GetObjectRequest, PutObjectRequest, S3Client, S3, -}; -use tokio::io::AsyncReadExt; - -fn get_config() -> Result<(String, String, Credentials)> { - // TODO: get these from .env? - Ok(( - String::from(""), - String::from(""), - Credentials { - access_key: String::from(""), - secret_key: String::from(""), - }, - )) -} - -async fn make_inner(url: String) -> Result> { - let (region, bucket, cred) = get_config()?; - // TODO: move creating the bucket here - Ok(Box::new(S3Store::new(&url, ®ion, &bucket, cred).await?)) -} - -pub fn make(url: &str) -> Pin>>>> { - Box::pin(make_inner(url.into())) -} - -#[derive(Clone)] -struct S3Store { - client: S3Client, - bucket: String, - endpoint: String, -} - -struct Credentials { - access_key: String, - secret_key: String, -} - -impl S3Store { - pub async fn new( - endpoint: &str, - region: &str, - bucket: &str, - cred: Credentials, - ) -> Result { - let region = Region::Custom { - name: region.to_owned(), - endpoint: endpoint.to_owned(), - }; - - let dispatcher = - rusoto_core::request::HttpClient::new().context("failed to create http client.")?; - - let provider = rusoto_core::credential::StaticProvider::new_minimal( - cred.access_key.clone(), - cred.secret_key.clone(), - ); - - let client = S3Client::new_with(dispatcher, provider, region); - - let create_bucket_request = CreateBucketRequest { - bucket: bucket.to_owned(), - ..Default::default() - }; - - match client.create_bucket(create_bucket_request).await { - Ok(_) | Err(RusotoError::Service(CreateBucketError::BucketAlreadyOwnedByYou(_))) => { - Ok(Self { - client, - bucket: bucket.to_owned(), - endpoint: endpoint.to_owned(), - }) - } - Err(_) => return Err(Error::BucketCreationError), - } - } -} - -#[async_trait::async_trait] -impl Store for S3Store { - async fn get(&self, key: &[u8]) -> super::Result> { - let get_object_request = GetObjectRequest { - bucket: self.bucket.clone(), - key: hex::encode(key), - ..Default::default() - }; - - let res = self - .client - .get_object(get_object_request) - .await - .context("failed to get blob")?; - - let body = res.body.ok_or(Error::KeyNotFound)?; - - let mut buffer = Vec::new(); - if let Err(_) = body.into_async_read().read_to_end(&mut buffer).await { - return Err(Error::InvalidBlob); - } - - Ok(buffer) - } - - async fn set(&self, key: &[u8], blob: &[u8]) -> Result<()> { - let put_object_request = PutObjectRequest { - bucket: self.bucket.clone(), - key: hex::encode(key), - body: Some(ByteStream::from(blob.to_owned())), - ..Default::default() - }; - self.client - .put_object(put_object_request) - .await - .context("failed to set blob")?; - - Ok(()) - } - - fn routes(&self) -> Vec { - vec![Route::url(self.endpoint.clone())] - } -} diff --git a/src/store/s3store.rs b/src/store/s3store.rs new file mode 100644 index 0000000..7cd6ed0 --- /dev/null +++ b/src/store/s3store.rs @@ -0,0 +1,109 @@ +use super::{Error, Result, Route, Store}; +use futures::Future; +use std::pin::Pin; + +use s3::creds::Credentials; +use s3::{Bucket, Region}; + +const REGION_NAME: &str = "minio"; + +fn get_config(url: &str) -> Result<(Credentials, Region, String)> { + let url = url::Url::parse(url.as_ref())?; + + let (access_key, access_secret, endpoint, bucket_name) = match url.host() { + Some(_) => { + let access_key = url.username().into(); + let access_secret = url.password().unwrap_or_default().into(); + + let host = url.host_str().unwrap_or_default(); + let port = url.port().unwrap_or(9000); + let endpoint = format!("{}:{}", host, port); + + let bucket_name = url.path().trim_start_matches('/').into(); + + // TODO: add region to the url? + + (access_key, access_secret, endpoint, bucket_name) + } + None => return Err(Error::InvalidConfigs), + }; + + let region_name = String::from(REGION_NAME); + + Ok(( + Credentials { + access_key: Some(access_key), + secret_key: Some(access_secret), + security_token: None, + session_token: None, + expiration: None, + }, + Region::Custom { + region: region_name, + endpoint: endpoint, + }, + bucket_name, + )) +} + +async fn make_inner(url: String) -> Result> { + let (cred, region, bucket_name) = get_config(&url)?; + Ok(Box::new( + S3Store::new(&url, &bucket_name, region, cred).await?, + )) +} + +pub fn make(url: &str) -> Pin>>>> { + Box::pin(make_inner(url.into())) +} + +#[derive(Clone)] +struct S3Store { + bucket: Bucket, + url: String, +} + +impl S3Store { + pub async fn new( + url: &str, + bucket_name: &str, + region: Region, + cred: Credentials, + ) -> Result { + let bucket = Bucket::new(bucket_name, region, cred)?.with_path_style(); + + Ok(Self { + bucket: bucket, + url: url.to_owned(), + }) + } +} + +#[async_trait::async_trait] +impl Store for S3Store { + async fn get(&self, key: &[u8]) -> super::Result> { + let response = self.bucket.get_object(std::str::from_utf8(key)?).await?; + Ok(response.to_vec()) + } + + async fn set(&self, key: &[u8], blob: &[u8]) -> Result<()> { + self.bucket + .put_object(std::str::from_utf8(key)?, blob) + .await?; + Ok(()) + } + + fn routes(&self) -> Vec { + vec![Route::url(self.url.clone())] + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_get_config() { + let (cred, region, bucket_name) = get_config("s3//minioadmin:minioadmin@127.0.0.1:9000/mybucket")?; + } +} \ No newline at end of file