From 6baee8b0f628c172d71e69775cd724c49e504d46 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 13 May 2022 11:53:10 +0200 Subject: [PATCH 01/61] feat: simple RDS support --- Cargo.lock | 455 +++++++++++++++++++++++++++++++++++++++++++- api/Cargo.toml | 3 + api/src/database.rs | 131 ++++++++++++- api/src/factory.rs | 13 +- 4 files changed, 584 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9860d0559..30438d561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,9 @@ dependencies = [ "anyhow", "async-mutex", "async-trait", + "aws-config", + "aws-sdk-rds", + "aws-smithy-types", "base64 0.13.0", "cargo", "cargo-util", @@ -417,6 +420,289 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91154572c69defdbe4ee7e7c86a0d1ad1f8c49655bedb7c6be61a7c1dc43105" +dependencies = [ + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex 0.4.3", + "http", + "hyper", + "ring", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "515bd0f038107827309daa28612941ff559e71a6e96335e336d4fdf4caffb34b" +dependencies = [ + "aws-smithy-http", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1378c2c2430a063076621ec8c6435cdbd97b3e053111aebb52c9333fc793f32c" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "lazy_static", + "percent-encoding", + "tracing", +] + +[[package]] +name = "aws-sdk-rds" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8660d4b8cc0675ed96b9816be78b183a3adc0545239cc110b8cbbb02dbf1dec7" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d74bff9a790eceb16a7825b11c4d9526fe6d3649349ba04beb5cb4770b7822b4" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e879f3619e0b444218ab76d28364d57b81820ad792fd58088ab675237e4d7b5f" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tower", +] + +[[package]] +name = "aws-sig-auth" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b44c7698294a89fabbadb538560ad34b6fdd26e926dee3c5e710928c3093fcf0" +dependencies = [ + "aws-sigv4", + "aws-smithy-http", + "aws-types", + "http", + "thiserror", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fe0aee7539298b8447e1a60d2d6de329fb1619f116c5f488e0b3aa136d49696" +dependencies = [ + "aws-smithy-http", + "form_urlencoded", + "hex 0.4.3", + "http", + "once_cell", + "percent-encoding", + "regex", + "ring", + "time 0.3.9", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dcbf7d119f514a627d236412626645c4378b126e30dc61db9de3e069fa1676" +dependencies = [ + "futures-util", + "pin-project-lite 0.2.8", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-client" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "511ac6c65f2a89cfcd74fe78aa6d07216095a53cbaeab493b17f6df82cd65b86" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project", + "pin-project-lite 0.2.8", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d800c8684fa567cdf1abd9654c7997b2a887e7b06022938756193472ec7ec251" +dependencies = [ + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project", + "tokio", + "tokio-util 0.6.9", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8017959786cce64e690214d303d062c97fcd38a68df7cb444255e534c9bbce49" +dependencies = [ + "aws-smithy-http", + "bytes", + "http", + "http-body", + "pin-project", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3796e2a4a3b7d15db2fd5aec2de9220919332648f0a56a77b5c53caf4a9653fa" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76694d1a2dacefa347b921c61bf64b5cc493a898971b3c18fa636ce1788ceabe" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7f957a2250cc0fa4ccf155e00aeac9a81f600df7cd4ecc910c75030e6534f5" +dependencies = [ + "itoa", + "num-integer", + "ryu", + "time 0.3.9", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b01d77433f248d9a2b08f519b403d6aa8435bf144b5d8585862f8dd599eb843" +dependencies = [ + "thiserror", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "394d5c945b747ab3292b94509b78c91191aacfd1deacbcd58371d6f61f8be78a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version 0.4.0", + "tracing", + "zeroize", +] + [[package]] name = "axum" version = "0.5.1" @@ -577,6 +863,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bytes-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "1.1.0" @@ -1011,6 +1307,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + [[package]] name = "ctor" version = "0.1.22" @@ -1819,6 +2124,23 @@ dependencies = [ "unicase", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "webpki", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2201,7 +2523,7 @@ dependencies = [ "log", "memchr", "mime", - "spin", + "spin 0.9.2", "tokio", "tokio-util 0.6.9", "version_check", @@ -2356,6 +2678,15 @@ dependencies = [ "syn", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" version = "1.10.0" @@ -2955,6 +3286,21 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rocket" version = "0.5.0-rc.1" @@ -3061,6 +3407,15 @@ dependencies = [ "semver 0.9.0", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.6", +] + [[package]] name = "rustfix" version = "0.6.0" @@ -3073,6 +3428,31 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -3116,6 +3496,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -3401,6 +3791,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.2" @@ -3540,7 +3936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -3803,6 +4199,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +dependencies = [ + "libc", + "num_threads", +] + [[package]] name = "time-macros" version = "0.1.1" @@ -3882,6 +4288,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-stream" version = "0.1.8" @@ -4129,6 +4546,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -4142,6 +4565,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" + [[package]] name = "utf8parse" version = "0.2.0" @@ -4363,6 +4792,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "wepoll-ffi" version = "0.1.2" @@ -4471,8 +4910,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "xmlparser" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" + [[package]] name = "yansi" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "zeroize" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" diff --git a/api/Cargo.toml b/api/Cargo.toml index 2dd3ad215..6eb1b92b7 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -5,6 +5,9 @@ edition = "2021" [dependencies] rocket = { version = "0.5.0-rc.1", features = ["uuid", "serde_json", "json"] } +aws-config = "0.11" +aws-sdk-rds = "0.11" +aws-smithy-types = "0.41" cargo = "0.59.0" cargo-util = "0.1" chrono = "0.4" diff --git a/api/src/database.rs b/api/src/database.rs index 66420d48e..1af8ed9c6 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -1,10 +1,14 @@ -use std::time::Duration; - +use anyhow::anyhow; +use aws_config::timeout; +use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, types::SdkError}; +use aws_smithy_types::tristate::TriState; use lazy_static::lazy_static; use rand::Rng; -use shuttle_common::project::ProjectName; -use shuttle_common::DatabaseReadyInfo; +use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; +use shuttle_service::error::CustomError; use sqlx::postgres::{PgPool, PgPoolOptions}; +use std::time::Duration; +use tokio::time::sleep; lazy_static! { static ref SUDO_POSTGRES_CONNECTION_STRING: String = format!( @@ -117,21 +121,128 @@ impl State { pub(crate) fn to_info(&self) -> Option { self.info.clone() } + + pub(crate) async fn aws_rds(&self) -> Result { + let client = &self.context.rds_client; + + let username = self.project.to_string().replace("-", "_"); + let password = generate_role_password(); + let engine = "postgres"; + let class = "db.t3.micro"; + let instance_name = format!("{}-{}", self.project, engine); + let db_name = "postgres"; + + let instances = client + .modify_db_instance() + .db_instance_identifier(&instance_name) + .master_user_password(&password) + .send() + .await; + debug!("got describe response"); + + let mut instance = match instances { + Ok(instances) => instances.db_instance.unwrap().clone(), + Err(SdkError::ServiceError { err, .. }) => { + if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { + debug!("creating new"); + + client + .create_db_instance() + .db_instance_identifier(&instance_name) + .master_username(username) + .master_user_password(&password) + .engine(engine) + .db_instance_class(class) + .allocated_storage(20) + .backup_retention_period(0) + .publicly_accessible(true) + .db_name(db_name) + .send() + .await + .map_err(shuttle_service::error::CustomError::new)? + .db_instance + .unwrap() + } else { + return Err(shuttle_service::Error::Custom(anyhow!( + "got unexpected error from AWS: {}", + err + ))); + } + } + Err(unexpected) => { + return Err(shuttle_service::Error::Custom(anyhow!( + "got unexpected error from AWS: {}", + unexpected + ))) + } + }; + + // Wait for up + debug!("waiting for password update"); + sleep(Duration::from_secs(30)).await; + loop { + instance = client + .describe_db_instances() + .db_instance_identifier(&instance_name) + .send() + .await + .map_err(CustomError::new)? + .db_instances + .unwrap() + .get(0) + .unwrap() + .clone(); + + let status = instance.db_instance_status.as_ref().unwrap().clone(); + + debug!("status: {status}"); + if status == "available" { + break; + } + sleep(Duration::from_secs(1)).await; + } + + println!("{instance:#?}"); + // let info = DatabaseReadyInfo::new(role_name, role_password, database_name); + let conn_string = format!( + "postgres://{}:{}@{}/{}", + instance.master_username.unwrap(), + password, + instance.endpoint.unwrap().address.unwrap(), + db_name + ); + + Ok(conn_string) + } } #[derive(Clone)] pub struct Context { sudo_pool: PgPool, + rds_client: aws_sdk_rds::Client, } impl Context { pub async fn new() -> sqlx::Result { - Ok(Context { - sudo_pool: PgPoolOptions::new() - .min_connections(4) - .max_connections(12) - .connect_timeout(Duration::from_secs(60)) - .connect_lazy(&SUDO_POSTGRES_CONNECTION_STRING)?, + let sudo_pool = PgPoolOptions::new() + .min_connections(4) + .max_connections(12) + .connect_timeout(Duration::from_secs(60)) + .connect_lazy(&SUDO_POSTGRES_CONNECTION_STRING)?; + + let api_timeout_config = + timeout::Api::new().with_call_timeout(TriState::Set(Duration::from_secs(5))); + let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); + let aws_config = aws_config::from_env() + .timeout_config(timeout_config) + .load() + .await; + + let rds_client = aws_sdk_rds::Client::new(&aws_config); + + Ok(Self { + sudo_pool, + rds_client, }) } } diff --git a/api/src/factory.rs b/api/src/factory.rs index 9f26f321e..c51c84b03 100644 --- a/api/src/factory.rs +++ b/api/src/factory.rs @@ -23,12 +23,13 @@ impl ShuttleFactory { #[async_trait] impl Factory for ShuttleFactory { async fn get_sql_connection_string(&mut self) -> Result { - let conn_str = self - .database - .request() - .await - .map_err(shuttle_service::error::CustomError::new)? - .connection_string("localhost"); + // let conn_str = self + // .database + // .request() + // .await + // .map_err(shuttle_service::error::CustomError::new)? + // .connection_string("localhost"); + let conn_str = self.database.aws_rds().await?; debug!("giving a sql connection string: {}", conn_str); Ok(conn_str) } From 008b2047b0c01ab0cd352a1da79c6d94702feae0 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 13 May 2022 12:51:10 +0200 Subject: [PATCH 02/61] refactor: cleanup and prep --- api/src/database.rs | 71 +++++++++++++++++++++++++++++++++------------ api/src/factory.rs | 13 ++++----- common/src/lib.rs | 36 ++++++++++++++--------- 3 files changed, 82 insertions(+), 38 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 1af8ed9c6..4542f55fb 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -10,6 +10,12 @@ use sqlx::postgres::{PgPool, PgPoolOptions}; use std::time::Duration; use tokio::time::sleep; +#[cfg(debug_assertions)] +const PUBLIC_PG_IP: &str = "localhost"; + +#[cfg(not(debug_assertions))] +const PUBLIC_PG_IP: &'static str = "pg.shuttle.rs"; + lazy_static! { static ref SUDO_POSTGRES_CONNECTION_STRING: String = format!( "postgres://postgres:{}@localhost", @@ -44,6 +50,7 @@ impl State { pub(crate) async fn request(&mut self) -> sqlx::Result { if self.info.is_some() { + // Safe to unwrap since we just confirmed it is `Some` return Ok(self.info.clone().unwrap()); } @@ -113,7 +120,13 @@ impl State { ); } - let info = DatabaseReadyInfo::new(role_name, role_password, database_name); + let info = DatabaseReadyInfo::new( + role_name, + role_password, + database_name, + "localhost".to_string(), + PUBLIC_PG_IP.to_string(), + ); self.info = Some(info.clone()); Ok(info) } @@ -122,7 +135,12 @@ impl State { self.info.clone() } - pub(crate) async fn aws_rds(&self) -> Result { + pub(crate) async fn aws_rds(&mut self) -> Result { + if self.info.is_some() { + // Safe to unwrap since we just confirmed it is `Some` + return Ok(self.info.clone().unwrap()); + } + let client = &self.context.rds_client; let username = self.project.to_string().replace("-", "_"); @@ -132,7 +150,7 @@ impl State { let instance_name = format!("{}-{}", self.project, engine); let db_name = "postgres"; - let instances = client + let instance = client .modify_db_instance() .db_instance_identifier(&instance_name) .master_user_password(&password) @@ -140,8 +158,11 @@ impl State { .await; debug!("got describe response"); - let mut instance = match instances { - Ok(instances) => instances.db_instance.unwrap().clone(), + let mut instance = match instance { + Ok(instance) => instance + .db_instance + .expect("aws response should contain an instance") + .clone(), Err(SdkError::ServiceError { err, .. }) => { if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { debug!("creating new"); @@ -161,7 +182,7 @@ impl State { .await .map_err(shuttle_service::error::CustomError::new)? .db_instance - .unwrap() + .expect("to be able to create instance") } else { return Err(shuttle_service::Error::Custom(anyhow!( "got unexpected error from AWS: {}", @@ -188,31 +209,45 @@ impl State { .await .map_err(CustomError::new)? .db_instances - .unwrap() + .expect("aws to return instances") .get(0) - .unwrap() + .expect("to find the instance just created or modified") .clone(); - let status = instance.db_instance_status.as_ref().unwrap().clone(); + let status = instance + .db_instance_status + .as_ref() + .expect("instance to have a status") + .clone(); - debug!("status: {status}"); if status == "available" { break; } + sleep(Duration::from_secs(1)).await; } - println!("{instance:#?}"); - // let info = DatabaseReadyInfo::new(role_name, role_password, database_name); - let conn_string = format!( - "postgres://{}:{}@{}/{}", - instance.master_username.unwrap(), + let address = instance + .endpoint + .expect("instance to have an endpoint") + .address + .expect("endpoint to have an address"); + + let info = DatabaseReadyInfo::new( + instance + .master_username + .expect("instance to have a username"), password, - instance.endpoint.unwrap().address.unwrap(), - db_name + instance + .db_name + .expect("instance to have a default database"), + address.clone(), + address, ); - Ok(conn_string) + self.info = Some(info.clone()); + + Ok(info) } } diff --git a/api/src/factory.rs b/api/src/factory.rs index c51c84b03..1b7ba746b 100644 --- a/api/src/factory.rs +++ b/api/src/factory.rs @@ -23,13 +23,12 @@ impl ShuttleFactory { #[async_trait] impl Factory for ShuttleFactory { async fn get_sql_connection_string(&mut self) -> Result { - // let conn_str = self - // .database - // .request() - // .await - // .map_err(shuttle_service::error::CustomError::new)? - // .connection_string("localhost"); - let conn_str = self.database.aws_rds().await?; + let conn_str = self + .database + .request() + .await + .map_err(shuttle_service::error::CustomError::new)? + .connection_string_private(); debug!("giving a sql connection string: {}", conn_str); Ok(conn_str) } diff --git a/common/src/lib.rs b/common/src/lib.rs index c907b1f26..ca5153219 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -70,19 +70,13 @@ impl DeploymentMeta { } } -#[cfg(debug_assertions)] -const PUBLIC_IP: &str = "localhost"; - -#[cfg(not(debug_assertions))] -const PUBLIC_IP: &'static str = "pg.shuttle.rs"; - impl Display for DeploymentMeta { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let db = { if let Some(info) = &self.database_deployment { format!( "\n Database URI: {}", - info.connection_string(PUBLIC_IP) + info.connection_string_public() ) } else { "".to_string() @@ -104,23 +98,39 @@ impl Display for DeploymentMeta { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DatabaseReadyInfo { - pub role_name: String, - pub role_password: String, - pub database_name: String, + role_name: String, + role_password: String, + database_name: String, + address_private: String, + address_public: String, } impl DatabaseReadyInfo { - pub fn new(role_name: String, role_password: String, database_name: String) -> Self { + pub fn new( + role_name: String, + role_password: String, + database_name: String, + address_private: String, + address_public: String, + ) -> Self { Self { role_name, role_password, database_name, + address_private, + address_public, } } - pub fn connection_string(&self, ip: &str) -> String { + pub fn connection_string_private(&self) -> String { + format!( + "postgres://{}:{}@{}/{}", + self.role_name, self.role_password, self.address_private, self.database_name + ) + } + pub fn connection_string_public(&self) -> String { format!( "postgres://{}:{}@{}/{}", - self.role_name, self.role_password, ip, self.database_name + self.role_name, self.role_password, self.address_public, self.database_name ) } } From 5fbff83a4ebec8eaf11977e522ae74838362f3a6 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 13 May 2022 13:47:38 +0200 Subject: [PATCH 03/61] feat: Simple AWS RDS Postgres tuple --- api/src/factory.rs | 24 ++++++++++++++++-------- codegen/src/lib.rs | 28 ++++++++++++++++++++++++++++ service/Cargo.toml | 1 + service/src/database.rs | 38 ++++++++++++++++++++++++++++++++++++++ service/src/lib.rs | 11 +++++++++-- service/tests/loader.rs | 4 ++-- 6 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 service/src/database.rs diff --git a/api/src/factory.rs b/api/src/factory.rs index 1b7ba746b..a174790ac 100644 --- a/api/src/factory.rs +++ b/api/src/factory.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use shuttle_common::DatabaseReadyInfo; -use shuttle_service::Factory; +use shuttle_service::{database::Type, Factory}; use crate::database; @@ -22,13 +22,21 @@ impl ShuttleFactory { #[async_trait] impl Factory for ShuttleFactory { - async fn get_sql_connection_string(&mut self) -> Result { - let conn_str = self - .database - .request() - .await - .map_err(shuttle_service::error::CustomError::new)? - .connection_string_private(); + async fn get_sql_connection_string( + &mut self, + db_type: Type, + ) -> Result { + let db_info = match db_type { + Type::Shared => self + .database + .request() + .await + .map_err(shuttle_service::error::CustomError::new)?, + Type::AwsRdsPostgres => self.database.aws_rds().await?, + }; + + let conn_str = db_info.connection_string_private(); + debug!("giving a sql connection string: {}", conn_str); Ok(conn_str) } diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index dcf0471f6..927feadc8 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -50,6 +50,18 @@ impl Wrapper { }) .filter_map(|typed| match typed.pat.as_ref() { Pat::Ident(ident) => Some(ident), + Pat::TupleStruct(ident) => { + match ident + .pat + .elems + .first() + .as_ref() + .expect("a single pattern item") + { + Pat::Ident(ident) => Some(ident), + _ => None, + } + } _ => None, }) .map(|pat_ident| pat_ident.ident.clone()) @@ -249,4 +261,20 @@ mod tests { assert_eq!(actual.to_string(), expected.to_string()); } + + #[test] + fn from_tuple_input() { + let input = parse_quote!( + async fn tuple(Postgres(pool): Postgres) -> Result<(), Box> {} + ); + + let actual = Wrapper::from_item_fn(&input); + let expected_ident: Ident = parse_quote!(tuple); + let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); + let expected_inputs: Vec = vec![parse_quote!(pool)]; + + assert_eq!(actual.fn_ident, expected_ident); + assert_eq!(actual.fn_output, expected_output); + assert_eq!(actual.fn_inputs, expected_inputs); + } } diff --git a/service/Cargo.toml b/service/Cargo.toml index 40628ad2a..a4a6abc6d 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -40,6 +40,7 @@ loader = ["libloading"] sqlx-integration = ["sqlx/runtime-tokio-native-tls"] sqlx-postgres = ["sqlx-integration", "sqlx/postgres"] +sqlx-aws = ["sqlx-integration", "sqlx/postgres"] secrets = ["sqlx-postgres"] diff --git a/service/src/database.rs b/service/src/database.rs new file mode 100644 index 000000000..bffd489cd --- /dev/null +++ b/service/src/database.rs @@ -0,0 +1,38 @@ +pub enum Type { + AwsRdsPostgres, + Shared, +} + +#[cfg(feature = "sqlx-aws")] +pub mod aws_rds { + use super::Type; + use crate::{error::CustomError, Factory, GetResource}; + use async_trait::async_trait; + use sqlx::PgPool; + use tokio::runtime::Runtime; + + pub struct Postgres(pub PgPool); + + /// Get an `sqlx::PgPool` connected to an AWS RDS Postgres instance + #[async_trait] + impl GetResource for &mut dyn Factory { + async fn get_resource(self, runtime: &Runtime) -> Result { + let connection_string = self.get_sql_connection_string(Type::AwsRdsPostgres).await?; + + // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end + let pool = runtime + .spawn(async move { + sqlx::postgres::PgPoolOptions::new() + .min_connections(1) + .max_connections(5) + .connect(&connection_string) + .await + }) + .await + .map_err(CustomError::new)? + .map_err(CustomError::new)?; + + Ok(Postgres(pool)) + } + } +} diff --git a/service/src/lib.rs b/service/src/lib.rs index feccd7c7d..f1a65c545 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -173,6 +173,8 @@ pub use log; pub use logger::Logger; pub use tokio::runtime::Runtime; +pub mod database; + pub mod error; pub use error::Error; @@ -251,7 +253,10 @@ pub trait Factory: Send + Sync { /// Declare that the [Service][Service] requires a Postgres database. /// /// Returns the connection string to the provisioned database. - async fn get_sql_connection_string(&mut self) -> Result; + async fn get_sql_connection_string( + &mut self, + db_type: database::Type, + ) -> Result; } /// Used to get resources of type `T` from factories. @@ -271,7 +276,9 @@ impl GetResource for &mut dyn Factory { async fn get_resource(self, runtime: &Runtime) -> Result { use error::CustomError; - let connection_string = self.get_sql_connection_string().await?; + let connection_string = self + .get_sql_connection_string(database::Type::Shared) + .await?; // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end let pool = runtime diff --git a/service/tests/loader.rs b/service/tests/loader.rs index 7043f2b06..e2f0ab8f2 100644 --- a/service/tests/loader.rs +++ b/service/tests/loader.rs @@ -8,7 +8,7 @@ mod helpers; use async_trait::async_trait; use helpers::PostgresInstance; use shuttle_service::loader::{Loader, LoaderError}; -use shuttle_service::{Error, Factory}; +use shuttle_service::{database, Error, Factory}; use uuid::Uuid; struct DummyFactory { @@ -25,7 +25,7 @@ impl DummyFactory { #[async_trait] impl Factory for DummyFactory { - async fn get_sql_connection_string(&mut self) -> Result { + async fn get_sql_connection_string(&mut self, _: database::Type) -> Result { let uri = if let Some(postgres_instance) = &self.postgres_instance { postgres_instance.get_uri() } else { From e743f3ddc5e4d18934453f8d214a03d75f498756 Mon Sep 17 00:00:00 2001 From: chesedo Date: Wed, 18 May 2022 12:49:41 +0200 Subject: [PATCH 04/61] feat: MySql and MariaDB support --- Cargo.lock | 940 ++++++++++++++++++++++++++-------------- api/Cargo.toml | 2 +- api/src/database.rs | 21 +- api/src/factory.rs | 2 +- api/src/main.rs | 2 +- common/Cargo.toml | 2 +- service/Cargo.toml | 5 +- service/src/database.rs | 103 +++-- service/src/secrets.rs | 14 +- 9 files changed, 723 insertions(+), 368 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30438d561..6aa0eccc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array", +] + [[package]] name = "aes" version = "0.6.0" @@ -25,7 +34,19 @@ checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" dependencies = [ "aes-soft", "aesni", - "cipher", + "cipher 0.2.5", +] + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher 0.3.0", + "cpufeatures", + "opaque-debug", ] [[package]] @@ -34,11 +55,25 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", + "aead 0.3.2", + "aes 0.6.0", + "cipher 0.2.5", + "ctr 0.6.0", + "ghash 0.3.1", + "subtle", +] + +[[package]] +name = "aes-gcm" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" +dependencies = [ + "aead 0.4.3", + "aes 0.7.5", + "cipher 0.3.0", + "ctr 0.8.0", + "ghash 0.4.4", "subtle", ] @@ -48,7 +83,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" dependencies = [ - "cipher", + "cipher 0.2.5", "opaque-debug", ] @@ -58,7 +93,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" dependencies = [ - "cipher", + "cipher 0.2.5", "opaque-debug", ] @@ -68,7 +103,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.5", + "getrandom 0.2.6", "once_cell", "version_check", ] @@ -93,9 +128,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "api" @@ -128,7 +163,7 @@ dependencies = [ "structopt", "tokio", "toml", - "uuid", + "uuid 1.0.0", ] [[package]] @@ -296,7 +331,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde_json", - "sha2", + "sha2 0.9.9", ] [[package]] @@ -335,7 +370,7 @@ dependencies = [ "memchr", "num_cpus", "once_cell", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "pin-utils", "slab", "wasm-bindgen-futures", @@ -370,9 +405,9 @@ checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ "proc-macro2", "quote", @@ -581,7 +616,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dcbf7d119f514a627d236412626645c4378b126e30dc61db9de3e069fa1676" dependencies = [ "futures-util", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", "tokio-stream", ] @@ -604,7 +639,7 @@ dependencies = [ "hyper-rustls", "lazy_static", "pin-project", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", "tower", "tracing", @@ -627,7 +662,7 @@ dependencies = [ "percent-encoding", "pin-project", "tokio", - "tokio-util 0.6.9", + "tokio-util 0.6.10", "tracing", ] @@ -705,9 +740,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.5.1" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25" +checksum = "ab2504b827a8bef941ba3dd64bdffe9cf56ca182908a147edd6189c95fbcae7d" dependencies = [ "async-trait", "axum-core", @@ -722,7 +757,7 @@ dependencies = [ "memchr", "mime", "percent-encoding", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "serde", "serde_json", "serde_urlencoded", @@ -736,9 +771,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244" +checksum = "da31c0ed7b4690e2c78fe4b880d21cd7db04a346ebc658b4270251b695437f17" dependencies = [ "async-trait", "bytes", @@ -750,9 +785,9 @@ dependencies = [ [[package]] name = "base-x" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +checksum = "dc19a4937b4fbd3fe3379793130e42060d10627a360f2127802b10b87e7baf74" [[package]] name = "base64" @@ -766,6 +801,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64ct" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea908e7347a8c64e378c17e30ef880ad73e3b4498346b055c2c00ea342f3179" + [[package]] name = "binascii" version = "0.1.4" @@ -808,7 +849,7 @@ dependencies = [ "cfg-if 0.1.10", "constant_time_eq", "crypto-mac 0.8.0", - "digest", + "digest 0.9.0", ] [[package]] @@ -820,6 +861,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + [[package]] name = "blocking" version = "1.2.0" @@ -887,9 +937,9 @@ checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" [[package]] name = "camino" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3132262930b0522068049f5870a856ab8affc80c70d08b6ecb785771a6fc23" +checksum = "07fd178c5af4d59e83498ef15cf3f154e1a6f9d091270cb86283c65ef44e9ef0" dependencies = [ "serde", ] @@ -936,7 +986,7 @@ dependencies = [ "percent-encoding", "rustc-workspace-hack", "rustfix", - "semver 1.0.6", + "semver 1.0.9", "serde", "serde_ignored", "serde_json", @@ -1018,7 +1068,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.6", + "semver 1.0.9", "serde", "serde_json", ] @@ -1060,7 +1110,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time 0.1.43", + "time 0.1.44", "winapi", ] @@ -1073,6 +1123,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", +] + [[package]] name = "clap" version = "2.34.0" @@ -1101,9 +1160,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.3" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b727aacc797f9fc28e355d21f34709ac4fc9adecfe470ad07b8f4464f53062" +checksum = "2a604e93b79d1808327a6fca85a6f2d69de66461e7620f5a4cbf5fb4d1d7c948" dependencies = [ "bytes", "memchr", @@ -1136,6 +1195,12 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "const-oid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" + [[package]] name = "const_fn" version = "0.4.9" @@ -1154,25 +1219,32 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ - "aes-gcm", + "aes-gcm 0.8.0", "base64 0.13.0", - "hkdf", + "hkdf 0.10.0", "hmac 0.10.1", "percent-encoding", "rand 0.8.5", - "sha2", + "sha2 0.9.9", "time 0.2.27", "version_check", ] [[package]] name = "cookie" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" +checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" dependencies = [ + "aes-gcm 0.9.4", + "base64 0.13.0", + "hkdf 0.12.3", + "hmac 0.12.1", "percent-encoding", - "time 0.2.27", + "rand 0.8.5", + "sha2 0.10.2", + "subtle", + "time 0.3.9", "version_check", ] @@ -1265,6 +1337,26 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-bigint" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-hash" version = "0.3.4" @@ -1297,16 +1389,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "crypto-mac" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "ct-logs" version = "0.8.0" @@ -1332,7 +1414,16 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" dependencies = [ - "cipher", + "cipher 0.2.5", +] + +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher 0.3.0", ] [[package]] @@ -1352,9 +1443,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.53+curl-7.82.0" +version = "0.4.55+curl-7.83.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8092905a5a9502c312f223b2775f57ec5c5b715f9a15ee9d2a8591d1364a0352" +checksum = "23734ec77368ec583c2e61dd3f0b0e5c98b93abe6d2a004ca06b91dd7e3e2762" dependencies = [ "cc", "libc", @@ -1368,9 +1459,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -1378,9 +1469,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", @@ -1392,15 +1483,26 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", "syn", ] +[[package]] +name = "der" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" +dependencies = [ + "const-oid", + "crypto-bigint", + "pem-rfc7468", +] + [[package]] name = "devise" version = "0.3.1" @@ -1455,6 +1557,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", + "subtle", +] + [[package]] name = "dirs" version = "4.0.0" @@ -1511,9 +1624,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ "cfg-if 1.0.0", ] @@ -1587,9 +1700,9 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1599,9 +1712,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -1726,7 +1839,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "waker-fn", ] @@ -1766,7 +1879,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "pin-utils", "slab", ] @@ -1817,13 +1930,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] @@ -1833,7 +1946,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" dependencies = [ "opaque-debug", - "polyval", + "polyval 0.4.5", +] + +[[package]] +name = "ghash" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99" +dependencies = [ + "opaque-debug", + "polyval 0.5.3", ] [[package]] @@ -1896,9 +2019,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -1909,7 +2032,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.6.9", + "tokio-util 0.7.2", "tracing", ] @@ -1940,6 +2063,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1967,10 +2099,19 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ - "digest", + "digest 0.9.0", "hmac 0.10.1", ] +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.8.1" @@ -1978,7 +2119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" dependencies = [ "crypto-mac 0.8.0", - "digest", + "digest 0.9.0", ] [[package]] @@ -1988,17 +2129,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ "crypto-mac 0.10.1", - "digest", + "digest 0.9.0", ] [[package]] name = "hmac" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "crypto-mac 0.11.1", - "digest", + "digest 0.10.3", ] [[package]] @@ -2012,9 +2152,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", @@ -2029,7 +2169,7 @@ checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", ] [[package]] @@ -2063,7 +2203,7 @@ dependencies = [ "cookie 0.14.4", "futures-lite", "infer", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "rand 0.7.3", "serde", "serde_json", @@ -2074,9 +2214,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -2106,7 +2246,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "socket2", "tokio", "tower-service", @@ -2191,12 +2331,12 @@ dependencies = [ [[package]] name = "im-rc" -version = "15.0.0" +version = "15.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ca8957e71f04a205cb162508f9326aea04676c8dfd0711220190d6b83664f3f" +checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" dependencies = [ "bitmaps", - "rand_core 0.5.1", + "rand_core 0.6.3", "rand_xoshiro", "sized-chunks", "typenum", @@ -2205,9 +2345,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown", @@ -2237,9 +2377,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itertools" @@ -2252,9 +2392,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "jni" @@ -2287,9 +2427,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] @@ -2308,6 +2448,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "lazycell" @@ -2317,9 +2460,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" @@ -2345,6 +2488,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" + [[package]] name = "libnghttp2-sys" version = "0.1.7+1.45.0" @@ -2371,9 +2520,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" dependencies = [ "cc", "libc", @@ -2383,18 +2532,19 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", "serde", @@ -2403,9 +2553,9 @@ dependencies = [ [[package]] name = "loom" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5c7d328e32cc4954e8e01193d7f0ef5ab257b5090b70a964e099a36034309" +checksum = "85eb735cf3c8ebac6cc3655c5da2f4a088b6a19133aa482471a21ba0eb5d83ab" dependencies = [ "cfg-if 1.0.0", "generator", @@ -2439,20 +2589,18 @@ checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" [[package]] name = "md-5" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" dependencies = [ - "block-buffer", - "digest", - "opaque-debug", + "digest 0.10.3", ] [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mime" @@ -2478,26 +2626,23 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" dependencies = [ "adler", - "autocfg", ] [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", + "windows-sys", ] [[package]] @@ -2523,17 +2668,17 @@ dependencies = [ "log", "memchr", "mime", - "spin 0.9.2", + "spin 0.9.3", "tokio", - "tokio-util 0.6.9", + "tokio-util 0.6.10", "version_check", ] [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -2562,15 +2707,15 @@ dependencies = [ [[package]] name = "ndk-context" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5cc68637e21fe8f077f6a1c9e0b9ca495bb74895226b476310f613325884" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" [[package]] name = "ndk-glue" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ffb7443daba48349d545028777ca98853b018b4c16624aa01223bc29e078da" +checksum = "0d0c4a7b83860226e6b4183edac21851f05d5a51756e97a1144b7f5a6b63e65f" dependencies = [ "lazy_static", "libc", @@ -2620,31 +2765,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" [[package]] -name = "ntapi" -version = "0.3.7" +name = "num-bigint" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" dependencies = [ - "winapi", + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566d173b2f9406afbc5510a90925d5a2cd80cae4605631f1212303df265de011" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", ] [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2711,18 +2887,30 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ "bitflags", "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -2731,9 +2919,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -2744,9 +2932,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023df84d545ef479cf67fd2f4459a613585c9db4852c2fad12ab70587859d340" +checksum = "04304d855bb5385d4b595edf0147b8e281871766b75dd4c87b2bdf3c9e5c2d19" dependencies = [ "log", "serde", @@ -2786,7 +2974,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.1", + "parking_lot_core 0.9.3", ] [[package]] @@ -2805,9 +2993,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if 1.0.0", "libc", @@ -2818,9 +3006,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" [[package]] name = "pear" @@ -2845,6 +3033,15 @@ dependencies = [ "syn", ] +[[package]] +name = "pem-rfc7468" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01de5d978f34aa4b2296576379fcc416034702fd94117c56ffd8a1a767cefb30" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -2879,9 +3076,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -2889,11 +3086,33 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78f66c04ccc83dd4486fd46c33896f4e17b24a7a3a6400dedc48ed0ddd72320" +dependencies = [ + "der", + "pkcs8", + "zeroize", +] + +[[package]] +name = "pkcs8" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" +dependencies = [ + "der", + "spki", + "zeroize", +] + [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "polling" @@ -2919,6 +3138,18 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portpicker" version = "0.1.1" @@ -2966,9 +3197,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c038cb5319b9c704bf9c227c261d275bfec0ad438118a2787ce47944fb228b" +checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" dependencies = [ "ansi_term", "ctor", @@ -3018,11 +3249,11 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -3040,9 +3271,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -3106,7 +3337,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.5", + "getrandom 0.2.6", ] [[package]] @@ -3120,47 +3351,47 @@ dependencies = [ [[package]] name = "rand_xoshiro" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" dependencies = [ - "rand_core 0.5.1", + "rand_core 0.6.3", ] [[package]] name = "redox_syscall" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.5", + "getrandom 0.2.6", "redox_syscall", "thiserror", ] [[package]] name = "ref-cast" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300f2a835d808734ee295d45007adacb9ebb29dd3ae2424acfa17930cae541da" +checksum = "685d58625b6c2b83e4cc88a27c4bf65adb7b6b16dbdc413e515c9405b47432ab" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c38e3aecd2b21cb3959637b883bb3714bc7e43f0268b9a29d3743ee3e55cdd2" +checksum = "a043824e29c94169374ac5183ac0ed43f5724dc4556b19568007486bd840fa1f" dependencies = [ "proc-macro2", "quote", @@ -3226,7 +3457,7 @@ dependencies = [ "mime_guess", "native-tls", "percent-encoding", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "serde", "serde_json", "serde_urlencoded", @@ -3241,9 +3472,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b58621b8223cfc85b63d38b8d335c69b96a666d9b7561aa30a3b070ce1df31c" +checksum = "69539cea4148dce683bec9dc95be3f0397a9bb2c248a49c8296a9d21659a8cdd" dependencies = [ "anyhow", "async-trait", @@ -3257,9 +3488,9 @@ dependencies = [ [[package]] name = "reqwest-retry" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5cb0170c4fa3f251c036ef482b35abc550afac6305cf875541c4b78dd76c6c" +checksum = "ce246a729eaa6aff5e215aee42845bf5fed9893cc6cd51aeeb712f34e04dd9f3" dependencies = [ "anyhow", "async-trait", @@ -3303,9 +3534,9 @@ dependencies = [ [[package]] name = "rocket" -version = "0.5.0-rc.1" +version = "0.5.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a71c18c42a0eb15bf3816831caf0dad11e7966f2a41aaf486a701979c4dd1f2" +checksum = "98ead083fce4a405feb349cf09abdf64471c6077f14e0ce59364aa90d4b99317" dependencies = [ "async-stream", "async-trait", @@ -3321,8 +3552,8 @@ dependencies = [ "memchr", "multer", "num_cpus", - "parking_lot 0.11.2", - "pin-project-lite 0.2.8", + "parking_lot 0.12.0", + "pin-project-lite 0.2.9", "rand 0.8.5", "ref-cast", "rocket_codegen", @@ -3331,21 +3562,21 @@ dependencies = [ "serde_json", "state", "tempfile", - "time 0.2.27", + "time 0.3.9", "tokio", "tokio-stream", - "tokio-util 0.6.9", + "tokio-util 0.7.2", "ubyte", - "uuid", + "uuid 1.0.0", "version_check", "yansi", ] [[package]] name = "rocket_codegen" -version = "0.5.0-rc.1" +version = "0.5.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f5fa462f7eb958bba8710c17c5d774bbbd59809fa76fb1957af7e545aea8bb" +checksum = "d6aeb6bb9c61e9cd2c00d70ea267bf36f76a4cc615e5908b349c2f9d93999b47" dependencies = [ "devise", "glob", @@ -3359,31 +3590,30 @@ dependencies = [ [[package]] name = "rocket_http" -version = "0.5.0-rc.1" +version = "0.5.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c8b7d512d2fcac2316ebe590cde67573844b99e6cc9ee0f53375fa16e25ebd" +checksum = "2ded65d127954de3c12471630bf4b81a2792f065984461e65b91d0fdaafc17a2" dependencies = [ - "cookie 0.15.1", + "cookie 0.16.0", "either", + "futures", "http", "hyper", "indexmap", "log", "memchr", - "mime", - "parking_lot 0.11.2", "pear", "percent-encoding", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "ref-cast", "serde", "smallvec", "stable-pattern", "state", - "time 0.2.27", + "time 0.3.9", "tokio", "uncased", - "uuid", + "uuid 1.0.0", ] [[package]] @@ -3392,6 +3622,26 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56770675ebc04927ded3e60633437841581c285dc6236109ea25fbf3beb7b59e" +[[package]] +name = "rsa" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf22754c49613d2b3b119f0e5d46e34a2c628a937e3024b8762de4e7d8c710b" +dependencies = [ + "byteorder", + "digest 0.10.3", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.3", + "smallvec", + "subtle", + "zeroize", +] + [[package]] name = "rustc-workspace-hack" version = "1.0.0" @@ -3413,7 +3663,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.6", + "semver 1.0.9", ] [[package]] @@ -3461,9 +3711,9 @@ checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "same-file" @@ -3476,12 +3726,12 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "winapi", + "windows-sys", ] [[package]] @@ -3540,9 +3790,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" dependencies = [ "serde", ] @@ -3555,18 +3805,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -3584,18 +3834,18 @@ dependencies = [ [[package]] name = "serde_ignored" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c2c7d39d14f2f2ea82239de71594782f186fd03501ac81f0ce08e674819ff2f" +checksum = "1940036ca2411651a40012009d062087dfe62817b2191a03750fb569e11fa633" dependencies = [ "serde", ] [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", @@ -3627,15 +3877,13 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ - "block-buffer", "cfg-if 1.0.0", "cpufeatures", - "digest", - "opaque-debug", + "digest 0.10.3", ] [[package]] @@ -3659,13 +3907,24 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -3702,7 +3961,7 @@ dependencies = [ "rocket", "serde", "serde_json", - "uuid", + "uuid 1.0.0", ] [[package]] @@ -3717,6 +3976,7 @@ dependencies = [ "lazy_static", "libloading", "log", + "paste", "portpicker", "regex", "rocket", @@ -3728,7 +3988,7 @@ dependencies = [ "tide", "tokio", "tower", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -3771,9 +4031,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" @@ -3799,9 +4059,19 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spin" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" + +[[package]] +name = "spki" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" +dependencies = [ + "base64ct", + "der", +] [[package]] name = "sqlformat" @@ -3816,9 +4086,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc15591eb44ffb5816a4a70a7efd5dd87bfd3aa84c4c200401c4396140525826" +checksum = "551873805652ba0d912fec5bbb0f8b4cdd96baf8e2ebf5970e5671092966019b" dependencies = [ "sqlx-core", "sqlx-macros", @@ -3826,9 +4096,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195183bf6ff8328bb82c0511a83faf60aacf75840103388851db61d7a9854ae3" +checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5" dependencies = [ "ahash", "atoi", @@ -3838,29 +4108,35 @@ dependencies = [ "bytes", "crc", "crossbeam-queue", + "digest 0.10.3", "dirs", "either", + "event-listener", "futures-channel", "futures-core", "futures-intrusive", "futures-util", + "generic-array", "hashlink", "hex 0.4.3", - "hmac 0.11.0", + "hkdf 0.12.3", + "hmac 0.12.1", "indexmap", "itoa", "libc", "log", "md-5", "memchr", + "num-bigint", "once_cell", "paste", "percent-encoding", "rand 0.8.5", + "rsa", "serde", "serde_json", "sha-1", - "sha2", + "sha2 0.10.2", "smallvec", "sqlformat", "sqlx-rt", @@ -3873,17 +4149,17 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee35713129561f5e55c554bba1c378e2a7e67f81257b7311183de98c50e6f94" +checksum = "bc0fba2b0cae21fc00fe6046f8baa4c7fcb49e379f0f592b04696607f69ed2e1" dependencies = [ "dotenv", "either", - "heck", + "heck 0.4.0", "once_cell", "proc-macro2", "quote", - "sha2", + "sha2 0.10.2", "sqlx-core", "sqlx-rt", "syn", @@ -3892,9 +4168,9 @@ dependencies = [ [[package]] name = "sqlx-rt" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b555e70fbbf84e269ec3858b7a6515bcfe7a166a7cc9c636dd6efd20431678b6" +checksum = "4db708cd3e459078f85f39f96a00960bd841f66ee2a669e90bf36907f5a79aae" dependencies = [ "native-tls", "once_cell", @@ -3922,9 +4198,9 @@ dependencies = [ [[package]] name = "state" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cf4f5369e6d3044b5e365c9690f451516ac8f0954084622b49ea3fde2f6de5" +checksum = "dbe866e1e51e8260c9eed836a042a5e7f6726bb2b411dffeaa712e19c388f23b" dependencies = [ "loom", ] @@ -4026,7 +4302,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro-error", "proc-macro2", "quote", @@ -4050,13 +4326,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.90" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704df27628939572cd88d33f171cd6f896f4eaca85252c6e0a72d8d8287ee86f" +checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -4124,18 +4400,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -4168,7 +4444,7 @@ dependencies = [ "http-types", "kv-log-macro", "log", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "route-recognizer", "serde", "serde_json", @@ -4176,11 +4452,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] @@ -4194,7 +4471,7 @@ dependencies = [ "libc", "standback", "stdweb", - "time-macros", + "time-macros 0.1.1", "version_check", "winapi", ] @@ -4205,8 +4482,10 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ + "itoa", "libc", "num_threads", + "time-macros 0.2.4", ] [[package]] @@ -4219,6 +4498,12 @@ dependencies = [ "time-macros-impl", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "time-macros-impl" version = "0.1.2" @@ -4234,9 +4519,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -4249,9 +4534,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ "bytes", "libc", @@ -4260,7 +4545,7 @@ dependencies = [ "num_cpus", "once_cell", "parking_lot 0.12.0", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "signal-hook-registry", "socket2", "tokio-macros", @@ -4306,42 +4591,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", ] [[package]] name = "tokio-util" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" dependencies = [ "bytes", "futures-core", "futures-sink", "log", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" dependencies = [ "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", + "tracing", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] @@ -4355,9 +4641,9 @@ dependencies = [ "futures-core", "futures-util", "pin-project", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", - "tokio-util 0.7.1", + "tokio-util 0.7.2", "tower-layer", "tower-service", "tracing", @@ -4365,9 +4651,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.2.5" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" +checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" dependencies = [ "bitflags", "bytes", @@ -4376,7 +4662,7 @@ dependencies = [ "http", "http-body", "http-range-header", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tower", "tower-layer", "tower-service", @@ -4396,22 +4682,22 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tracing-attributes", "tracing-core", ] [[package]] name = "tracing-attributes" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -4420,9 +4706,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -4430,9 +4716,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", @@ -4441,9 +4727,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "ansi_term", "lazy_static", @@ -4471,18 +4757,18 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ubyte" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42756bb9e708855de2f8a98195643dff31a97f0485d90d8467b39dc24be9e8fe" +checksum = "a58e29f263341a29bb79e14ad7fda5f63b1c7e48929bad4c685d7876b1d04e94" dependencies = [ "serde", ] [[package]] name = "uncased" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baeed7327e25054889b9bd4f975f32e5f4c5d434042d59ab6cd4142c0a76ed0" +checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" dependencies = [ "serde", "version_check", @@ -4499,9 +4785,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" [[package]] name = "unicode-normalization" @@ -4526,9 +4818,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "unicode_categories" @@ -4582,8 +4874,14 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + +[[package]] +name = "uuid" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0" dependencies = [ - "getrandom 0.2.5", + "getrandom 0.2.6", "serde", ] @@ -4595,9 +4893,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.0.0-alpha.8" +version = "1.0.0-alpha.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79923f7731dc61ebfba3633098bf3ac533bbd35ccd8c57e7088d9a5eebe0263f" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ "ctor", "erased-serde", @@ -4690,9 +4988,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" @@ -4702,9 +5000,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "serde", @@ -4714,9 +5012,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -4729,9 +5027,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4741,9 +5039,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4751,9 +5049,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -4764,15 +5062,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -4860,9 +5158,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -4873,33 +5171,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "winreg" diff --git a/api/Cargo.toml b/api/Cargo.toml index 6eb1b92b7..09676bb46 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -15,7 +15,7 @@ anyhow = "1.0.54" libloading = "0.7.3" futures = "0.3" fqdn = "0.1" -uuid = { version = "0.8.2", features = ["v4"] } +uuid = { version = "1.0.0", features = ["v4"] } async-mutex = "1.4.0" serde = "1.0.136" structopt = "0.3.26" diff --git a/api/src/database.rs b/api/src/database.rs index 4542f55fb..b87399e54 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -5,7 +5,7 @@ use aws_smithy_types::tristate::TriState; use lazy_static::lazy_static; use rand::Rng; use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; -use shuttle_service::error::CustomError; +use shuttle_service::{database::AwsRdsEngine, error::CustomError}; use sqlx::postgres::{PgPool, PgPoolOptions}; use std::time::Duration; use tokio::time::sleep; @@ -16,6 +16,8 @@ const PUBLIC_PG_IP: &str = "localhost"; #[cfg(not(debug_assertions))] const PUBLIC_PG_IP: &'static str = "pg.shuttle.rs"; +const AWS_RDS_CLASS: &str = "db.t4g.micro"; + lazy_static! { static ref SUDO_POSTGRES_CONNECTION_STRING: String = format!( "postgres://postgres:{}@localhost", @@ -135,7 +137,10 @@ impl State { self.info.clone() } - pub(crate) async fn aws_rds(&mut self) -> Result { + pub(crate) async fn aws_rds( + &mut self, + engine: AwsRdsEngine, + ) -> Result { if self.info.is_some() { // Safe to unwrap since we just confirmed it is `Some` return Ok(self.info.clone().unwrap()); @@ -145,10 +150,7 @@ impl State { let username = self.project.to_string().replace("-", "_"); let password = generate_role_password(); - let engine = "postgres"; - let class = "db.t3.micro"; let instance_name = format!("{}-{}", self.project, engine); - let db_name = "postgres"; let instance = client .modify_db_instance() @@ -156,7 +158,6 @@ impl State { .master_user_password(&password) .send() .await; - debug!("got describe response"); let mut instance = match instance { Ok(instance) => instance @@ -165,19 +166,19 @@ impl State { .clone(), Err(SdkError::ServiceError { err, .. }) => { if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { - debug!("creating new"); + debug!("creating new AWS RDS for {}", self.project); client .create_db_instance() .db_instance_identifier(&instance_name) .master_username(username) .master_user_password(&password) - .engine(engine) - .db_instance_class(class) + .engine(engine.to_string()) + .db_instance_class(AWS_RDS_CLASS) .allocated_storage(20) .backup_retention_period(0) .publicly_accessible(true) - .db_name(db_name) + .db_name(engine.to_string()) .send() .await .map_err(shuttle_service::error::CustomError::new)? diff --git a/api/src/factory.rs b/api/src/factory.rs index a174790ac..edac84fa3 100644 --- a/api/src/factory.rs +++ b/api/src/factory.rs @@ -32,7 +32,7 @@ impl Factory for ShuttleFactory { .request() .await .map_err(shuttle_service::error::CustomError::new)?, - Type::AwsRdsPostgres => self.database.aws_rds().await?, + Type::AwsRds(engine) => self.database.aws_rds(engine).await?, }; let conn_str = db_info.connection_string_private(); diff --git a/api/src/main.rs b/api/src/main.rs index 36d1b9d68..08caa0a09 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -144,7 +144,7 @@ async fn project_secrets( .await?; if let Some(database_deployment) = &deployment.database_deployment { - let conn_str = database_deployment.connection_string("localhost"); + let conn_str = database_deployment.connection_string_private(); let conn = sqlx::PgPool::connect(&conn_str) .await .map_err(|e| DeploymentApiError::Internal(e.to_string()))?; diff --git a/common/Cargo.toml b/common/Cargo.toml index 1003ab06d..97c5034fb 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -10,7 +10,7 @@ description = "Common library for the shuttle platform (https://www.shuttle.rs/) [dependencies] serde = "1.0.136" serde_json = "1.0.79" -uuid = { version = "0.8.2", features = ["v4", "serde"] } +uuid = { version = "1.0.0", features = ["v4", "serde"] } rocket = "0.5.0-rc.1" anyhow = "1.0.55" lazy_static = "1.4.0" diff --git a/service/Cargo.toml b/service/Cargo.toml index a4a6abc6d..9616c527d 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -16,6 +16,7 @@ libloading = { version = "0.7.3", optional = true } log = "0.4" sync_wrapper = { version = "0.1", optional = true } axum = { version = "0.5", optional = true } +paste = "1.0.7" rocket = { version = "0.5.0-rc.1", optional = true } tide = { version = "0.16.0", optional = true } tower = { version = "0.4", features = ["make"], optional = true } @@ -40,7 +41,9 @@ loader = ["libloading"] sqlx-integration = ["sqlx/runtime-tokio-native-tls"] sqlx-postgres = ["sqlx-integration", "sqlx/postgres"] -sqlx-aws = ["sqlx-integration", "sqlx/postgres"] +sqlx-aws-postgres = ["sqlx-integration", "sqlx/postgres"] +sqlx-aws-mysql = ["sqlx-integration", "sqlx/mysql"] +sqlx-aws-mariadb = ["sqlx-integration", "sqlx/mysql"] secrets = ["sqlx-postgres"] diff --git a/service/src/database.rs b/service/src/database.rs index bffd489cd..f9e839cb3 100644 --- a/service/src/database.rs +++ b/service/src/database.rs @@ -1,38 +1,87 @@ +use std::fmt::Display; + pub enum Type { - AwsRdsPostgres, + AwsRds(AwsRdsEngine), Shared, } -#[cfg(feature = "sqlx-aws")] +pub enum AwsRdsEngine { + Postgres, + MySql, + MariaDB, +} + +impl Display for AwsRdsEngine { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + &Self::MariaDB => write!(f, "mariadb"), + &Self::MySql => write!(f, "mysql"), + &Self::Postgres => write!(f, "postgres"), + } + } +} + +macro_rules! aws_engine { + ($feature:expr, $pool_path:path, $options_path:path, $struct_ident:ident) => { + paste! { + #[cfg(feature = $feature)] + #[doc = "A resource connected to an AWS RDS " $struct_ident " instance"] + pub struct $struct_ident(pub $pool_path); + + #[cfg(feature = $feature)] + #[doc = "Gets a `sqlx::Pool` connected to an AWS RDS " $struct_ident " instance"] + #[async_trait] + impl GetResource<$struct_ident> for &mut dyn Factory { + async fn get_resource(self, runtime: &Runtime) -> Result<$struct_ident, crate::Error> { + let connection_string = self + .get_sql_connection_string(Type::AwsRds(AwsRdsEngine::$struct_ident)) + .await?; + + // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end + let pool = runtime + .spawn(async move { + $options_path::new() + .min_connections(1) + .max_connections(5) + .connect(&connection_string) + .await + }) + .await + .map_err(CustomError::new)? + .map_err(CustomError::new)?; + + Ok($struct_ident(pool)) + } + } + } + }; +} + pub mod aws_rds { - use super::Type; + use super::{AwsRdsEngine, Type}; use crate::{error::CustomError, Factory, GetResource}; use async_trait::async_trait; - use sqlx::PgPool; + use paste::paste; use tokio::runtime::Runtime; - pub struct Postgres(pub PgPool); - - /// Get an `sqlx::PgPool` connected to an AWS RDS Postgres instance - #[async_trait] - impl GetResource for &mut dyn Factory { - async fn get_resource(self, runtime: &Runtime) -> Result { - let connection_string = self.get_sql_connection_string(Type::AwsRdsPostgres).await?; - - // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end - let pool = runtime - .spawn(async move { - sqlx::postgres::PgPoolOptions::new() - .min_connections(1) - .max_connections(5) - .connect(&connection_string) - .await - }) - .await - .map_err(CustomError::new)? - .map_err(CustomError::new)?; + aws_engine!( + "sqlx-aws-postgres", + sqlx::PgPool, + sqlx::postgres::PgPoolOptions, + Postgres + ); - Ok(Postgres(pool)) - } - } + aws_engine!( + "sqlx-aws-mysql", + sqlx::MySqlPool, + sqlx::mysql::MySqlPoolOptions, + MySql + ); + + aws_engine!( + "sqlx-aws-mariadb", + sqlx::MySqlPool, + sqlx::mysql::MySqlPoolOptions, + MariaDB + ); } diff --git a/service/src/secrets.rs b/service/src/secrets.rs index f5e14e8a0..2e3a42288 100644 --- a/service/src/secrets.rs +++ b/service/src/secrets.rs @@ -4,7 +4,9 @@ use lazy_static::lazy_static; use regex::Regex; use sqlx::postgres::PgArguments; use sqlx::query::Query; -use sqlx::{ColumnIndex, Database, Decode, Execute, Executor, Postgres, Row, Type}; +use sqlx::{ + ColumnIndex, Database, Decode, Encode, Execute, Executor, IntoArguments, Postgres, Row, Type, +}; use crate::error::Error; @@ -24,11 +26,13 @@ fn check_and_lower_secret_key(key: &str) -> Result { /// should you prefer). The table in question is created if it is found to not exist every time /// either [`get_secret`] or [`set_secret`] is called. #[async_trait] -pub trait SecretStore +pub trait SecretStore where - DB: Database, + DB: Database, + Arguments: for<'q> IntoArguments<'q, DB>, for<'c> &'c Self: Executor<'c, Database = DB>, - for<'c> String: Decode<'c, DB> + Type, + for<'c> String: Decode<'c, DB> + Encode<'c, DB> + Type, + for<'c> &'c str: Encode<'c, DB> + Type, for<'c> usize: ColumnIndex<::Row>, for<'c> Query<'c, Postgres, PgArguments>: Execute<'c, DB>, { @@ -67,7 +71,7 @@ where } #[async_trait] -impl SecretStore for sqlx::PgPool { +impl SecretStore for sqlx::PgPool { const GET_QUERY: &'static str = "SELECT value FROM secrets WHERE key = $1"; const SET_QUERY: &'static str = "INSERT INTO secrets (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = $2"; From 90d9103a855b993e624332967397577db84f79c9 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 16 May 2022 09:40:34 +0200 Subject: [PATCH 05/61] refactor: add engine to DB info --- api/src/database.rs | 2 ++ common/src/lib.rs | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index b87399e54..8628db044 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -123,6 +123,7 @@ impl State { } let info = DatabaseReadyInfo::new( + "postgres".to_string(), role_name, role_password, database_name, @@ -235,6 +236,7 @@ impl State { .expect("endpoint to have an address"); let info = DatabaseReadyInfo::new( + engine.to_string(), instance .master_username .expect("instance to have a username"), diff --git a/common/src/lib.rs b/common/src/lib.rs index ca5153219..ce3716a07 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -98,6 +98,7 @@ impl Display for DeploymentMeta { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DatabaseReadyInfo { + engine: String, role_name: String, role_password: String, database_name: String, @@ -107,6 +108,7 @@ pub struct DatabaseReadyInfo { impl DatabaseReadyInfo { pub fn new( + engine: String, role_name: String, role_password: String, database_name: String, @@ -114,6 +116,7 @@ impl DatabaseReadyInfo { address_public: String, ) -> Self { Self { + engine, role_name, role_password, database_name, @@ -123,14 +126,22 @@ impl DatabaseReadyInfo { } pub fn connection_string_private(&self) -> String { format!( - "postgres://{}:{}@{}/{}", - self.role_name, self.role_password, self.address_private, self.database_name + "{}://{}:{}@{}/{}", + self.engine, + self.role_name, + self.role_password, + self.address_private, + self.database_name ) } pub fn connection_string_public(&self) -> String { format!( - "postgres://{}:{}@{}/{}", - self.role_name, self.role_password, self.address_public, self.database_name + "{}://{}:{}@{}/{}", + self.engine, + self.role_name, + self.role_password, + self.address_public, + self.database_name ) } } From 4e0adc5840734e4d35409f2688bb3d53801da336 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 16 May 2022 09:43:16 +0200 Subject: [PATCH 06/61] refactor: 'master' for username --- api/src/database.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 8628db044..1ad54fb48 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -17,6 +17,7 @@ const PUBLIC_PG_IP: &str = "localhost"; const PUBLIC_PG_IP: &'static str = "pg.shuttle.rs"; const AWS_RDS_CLASS: &str = "db.t4g.micro"; +const MASTER_USERNAME: &str = "master"; lazy_static! { static ref SUDO_POSTGRES_CONNECTION_STRING: String = format!( @@ -149,7 +150,6 @@ impl State { let client = &self.context.rds_client; - let username = self.project.to_string().replace("-", "_"); let password = generate_role_password(); let instance_name = format!("{}-{}", self.project, engine); @@ -172,7 +172,7 @@ impl State { client .create_db_instance() .db_instance_identifier(&instance_name) - .master_username(username) + .master_username(MASTER_USERNAME) .master_user_password(&password) .engine(engine.to_string()) .db_instance_class(AWS_RDS_CLASS) From 6e20ccf7e30a045ebb5aaa0f4b35b0ef137e7315 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 07:23:04 +0200 Subject: [PATCH 07/61] refactor: clippy suggestions --- service/src/database.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service/src/database.rs b/service/src/database.rs index f9e839cb3..62e22bb09 100644 --- a/service/src/database.rs +++ b/service/src/database.rs @@ -14,9 +14,9 @@ pub enum AwsRdsEngine { impl Display for AwsRdsEngine { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - &Self::MariaDB => write!(f, "mariadb"), - &Self::MySql => write!(f, "mysql"), - &Self::Postgres => write!(f, "postgres"), + Self::MariaDB => write!(f, "mariadb"), + Self::MySql => write!(f, "mysql"), + Self::Postgres => write!(f, "postgres"), } } } From 6e244c546e85a804c805123a14a9b14ba172c9de Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 08:59:41 +0200 Subject: [PATCH 08/61] refactor: better debug logs --- api/src/database.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 1ad54fb48..fe079c740 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -143,16 +143,19 @@ impl State { &mut self, engine: AwsRdsEngine, ) -> Result { + println!("getting rds"); if self.info.is_some() { // Safe to unwrap since we just confirmed it is `Some` return Ok(self.info.clone().unwrap()); } + println!("getting client"); let client = &self.context.rds_client; let password = generate_role_password(); let instance_name = format!("{}-{}", self.project, engine); + println!("getting modified instance"); let instance = client .modify_db_instance() .db_instance_identifier(&instance_name) @@ -160,6 +163,7 @@ impl State { .send() .await; + println!("checking status"); let mut instance = match instance { Ok(instance) => instance .db_instance @@ -187,14 +191,14 @@ impl State { .expect("to be able to create instance") } else { return Err(shuttle_service::Error::Custom(anyhow!( - "got unexpected error from AWS: {}", + "got unexpected error from AWS RDS service: {}", err ))); } } Err(unexpected) => { return Err(shuttle_service::Error::Custom(anyhow!( - "got unexpected error from AWS: {}", + "got unexpected error from AWS during API call: {}", unexpected ))) } From 366d583b8d97c112c183ea6e5aa5ef6618c84e22 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 09:16:04 +0200 Subject: [PATCH 09/61] refactor: set region --- api/src/database.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/database.rs b/api/src/database.rs index fe079c740..218b589e1 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use aws_config::timeout; +use aws_config::{meta::region::RegionProviderChain, timeout}; use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, types::SdkError}; use aws_smithy_types::tristate::TriState; use lazy_static::lazy_static; @@ -275,8 +275,10 @@ impl Context { let api_timeout_config = timeout::Api::new().with_call_timeout(TriState::Set(Duration::from_secs(5))); let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); + let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); let aws_config = aws_config::from_env() .timeout_config(timeout_config) + .region(region_provider) .load() .await; From 9d9c2a07fe01a7d5584d702bd318e0b09e6ecc55 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 11:08:02 +0200 Subject: [PATCH 10/61] refactor: increase tracing --- api/src/database.rs | 1 + api/src/main.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/api/src/database.rs b/api/src/database.rs index 218b589e1..1293ac537 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -150,6 +150,7 @@ impl State { } println!("getting client"); + error!("getting client"); let client = &self.context.rds_client; let password = generate_role_password(); diff --git a/api/src/main.rs b/api/src/main.rs index 08caa0a09..f2f591aed 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -181,6 +181,7 @@ fn main() -> Result<(), Box> { async fn rocket() -> Rocket { env_logger::Builder::new() .filter_module("rocket", log::LevelFilter::Warn) + .filter_module("aws_config", log::LevelFilter::Trace) .filter_module("_", log::LevelFilter::Warn) .filter_module("api", log::LevelFilter::Debug) .init(); From 2b541dc7fd0002b69e05ea9d2bc11e06e002b15b Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 12:52:53 +0200 Subject: [PATCH 11/61] refactor: try more timeouts --- api/src/database.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 1293ac537..a3883febd 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -273,9 +273,15 @@ impl Context { .connect_timeout(Duration::from_secs(60)) .connect_lazy(&SUDO_POSTGRES_CONNECTION_STRING)?; - let api_timeout_config = - timeout::Api::new().with_call_timeout(TriState::Set(Duration::from_secs(5))); - let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); + let api_timeout_config = timeout::Api::new() + .with_call_timeout(TriState::Set(Duration::from_secs(120))) + .with_call_attempt_timeout(TriState::Set(Duration::from_secs(120))); + let http_timeout_config = timeout::Http::new() + .with_connect_timeout(TriState::Set(Duration::from_secs(120))) + .with_read_timeout(TriState::Set(Duration::from_secs(120))); + let timeout_config = timeout::Config::new() + .with_api_timeouts(api_timeout_config) + .with_http_timeouts(http_timeout_config); let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); let aws_config = aws_config::from_env() .timeout_config(timeout_config) From b6bd166985a790fe19f68e79bc586faf5fddde61 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 15:22:58 +0200 Subject: [PATCH 12/61] refactor: try manual profile --- api/src/database.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index a3883febd..26c523d33 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -1,5 +1,8 @@ use anyhow::anyhow; -use aws_config::{meta::region::RegionProviderChain, timeout}; +use aws_config::{ + default_provider::credentials::DefaultCredentialsChain, meta::region::RegionProviderChain, + timeout, +}; use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, types::SdkError}; use aws_smithy_types::tristate::TriState; use lazy_static::lazy_static; @@ -276,16 +279,18 @@ impl Context { let api_timeout_config = timeout::Api::new() .with_call_timeout(TriState::Set(Duration::from_secs(120))) .with_call_attempt_timeout(TriState::Set(Duration::from_secs(120))); - let http_timeout_config = timeout::Http::new() - .with_connect_timeout(TriState::Set(Duration::from_secs(120))) - .with_read_timeout(TriState::Set(Duration::from_secs(120))); - let timeout_config = timeout::Config::new() - .with_api_timeouts(api_timeout_config) - .with_http_timeouts(http_timeout_config); + let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); + + let credentials_provider = DefaultCredentialsChain::builder() + .profile_name("BackendAPIRole") + .build() + .await; + let aws_config = aws_config::from_env() .timeout_config(timeout_config) .region(region_provider) + .credentials_provider(credentials_provider) .load() .await; From 3ed7c8bca62eb73bbc6d91cb968016058dfe4637 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 15:33:16 +0200 Subject: [PATCH 13/61] refactor: try manual imds provider --- api/src/database.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 26c523d33..a28298b91 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -1,6 +1,8 @@ use anyhow::anyhow; use aws_config::{ - default_provider::credentials::DefaultCredentialsChain, meta::region::RegionProviderChain, + environment::EnvironmentVariableCredentialsProvider, + imds, + meta::{credentials::CredentialsProviderChain, region::RegionProviderChain}, timeout, }; use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, types::SdkError}; @@ -282,15 +284,18 @@ impl Context { let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); - let credentials_provider = DefaultCredentialsChain::builder() - .profile_name("BackendAPIRole") - .build() - .await; + let env_provider = EnvironmentVariableCredentialsProvider::new(); + let imds_provider = imds::credentials::Builder::default() + .profile("BackendAPIRole") + .build(); + + let chained_provider = CredentialsProviderChain::first_try("Environment", env_provider) + .or_else("Ec2InstanceMetadata", imds_provider); let aws_config = aws_config::from_env() .timeout_config(timeout_config) .region(region_provider) - .credentials_provider(credentials_provider) + .credentials_provider(chained_provider) .load() .await; From c7444f34d7fd3bb1b397af8c593e545337f30292 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 19 May 2022 15:38:16 +0200 Subject: [PATCH 14/61] refactor: try manual client --- api/src/database.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/src/database.rs b/api/src/database.rs index a28298b91..5a1bdabcd 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -285,8 +285,14 @@ impl Context { let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); let env_provider = EnvironmentVariableCredentialsProvider::new(); + let imds_client = imds::client::Builder::default() + .connect_timeout(Duration::from_secs(30)) + .build() + .await + .unwrap(); let imds_provider = imds::credentials::Builder::default() .profile("BackendAPIRole") + .imds_client(imds_client) .build(); let chained_provider = CredentialsProviderChain::first_try("Environment", env_provider) From 268dcbb873af7393703779a30566191b9ee86b35 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 08:59:15 +0200 Subject: [PATCH 15/61] refactor: set read timeout --- api/src/database.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/database.rs b/api/src/database.rs index 5a1bdabcd..d540efdcb 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -287,6 +287,7 @@ impl Context { let env_provider = EnvironmentVariableCredentialsProvider::new(); let imds_client = imds::client::Builder::default() .connect_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(30)) .build() .await .unwrap(); From 17c61fa7cf48555834c8dd74ec86e6022850db6c Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 09:08:47 +0200 Subject: [PATCH 16/61] refactor: try endpoint override --- api/src/database.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/database.rs b/api/src/database.rs index d540efdcb..e73bf79f2 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -7,6 +7,7 @@ use aws_config::{ }; use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, types::SdkError}; use aws_smithy_types::tristate::TriState; +use hyper::Uri; use lazy_static::lazy_static; use rand::Rng; use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; @@ -288,6 +289,7 @@ impl Context { let imds_client = imds::client::Builder::default() .connect_timeout(Duration::from_secs(30)) .read_timeout(Duration::from_secs(30)) + .endpoint(Uri::from_static("http://169.254.169.254")) .build() .await .unwrap(); From 966c233431390b75816d8e72b4f9f4389d85b0ab Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 09:19:34 +0200 Subject: [PATCH 17/61] refactor: trying aws update --- Cargo.lock | 72 ++++++++++++++++++++++++-------------------------- api/Cargo.toml | 6 ++--- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6aa0eccc4..f22e17ebd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,9 +457,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91154572c69defdbe4ee7e7c86a0d1ad1f8c49655bedb7c6be61a7c1dc43105" +checksum = "b58e10d03d2180f9fce2fe72ffa007ccf44b4714b127d8429e78c3e55fe345c0" dependencies = [ "aws-http", "aws-sdk-sso", @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "aws-endpoint" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "515bd0f038107827309daa28612941ff559e71a6e96335e336d4fdf4caffb34b" +checksum = "6eadd9ea45f65689fbd088b22a2aaed363582fab5dcd335a1a7317640436952f" dependencies = [ "aws-smithy-http", "aws-types", @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1378c2c2430a063076621ec8c6435cdbd97b3e053111aebb52c9333fc793f32c" +checksum = "80fd2c81e2782d1f57a532543524c59b9c32dcc9132a7a2d308864c05b645e98" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -512,9 +512,9 @@ dependencies = [ [[package]] name = "aws-sdk-rds" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8660d4b8cc0675ed96b9816be78b183a3adc0545239cc110b8cbbb02dbf1dec7" +checksum = "a3aec9e3ca4a565ea1098399cca17754d3dcd94ad19deb9b3acbdb39df78aae2" dependencies = [ "aws-endpoint", "aws-http", @@ -535,9 +535,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d74bff9a790eceb16a7825b11c4d9526fe6d3649349ba04beb5cb4770b7822b4" +checksum = "0c856987e02db8c5d81cb97e2a24609b05a6289db3d861c5b7d73081d6ecc70d" dependencies = [ "aws-endpoint", "aws-http", @@ -557,9 +557,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e879f3619e0b444218ab76d28364d57b81820ad792fd58088ab675237e4d7b5f" +checksum = "dbf0a83f4382e47778399d0e0ae38761eee2fbad1d3bdde21358bd00cea46f36" dependencies = [ "aws-endpoint", "aws-http", @@ -579,23 +579,22 @@ dependencies = [ [[package]] name = "aws-sig-auth" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b44c7698294a89fabbadb538560ad34b6fdd26e926dee3c5e710928c3093fcf0" +checksum = "27079eb4062be105e44ab4638ade63cf6448c2df5cb093d885faab8082fd5c9d" dependencies = [ "aws-sigv4", "aws-smithy-http", "aws-types", "http", - "thiserror", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe0aee7539298b8447e1a60d2d6de329fb1619f116c5f488e0b3aa136d49696" +checksum = "a0be4a7223c244f3c57426087204cf66e04d1e73adb62571090613889cf6ee57" dependencies = [ "aws-smithy-http", "form_urlencoded", @@ -611,9 +610,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dcbf7d119f514a627d236412626645c4378b126e30dc61db9de3e069fa1676" +checksum = "7cafe7e459caf2e2f834a77062f2c47f7df1956cf3be3060b23bf0721f17e1a4" dependencies = [ "futures-util", "pin-project-lite 0.2.9", @@ -623,9 +622,9 @@ dependencies = [ [[package]] name = "aws-smithy-client" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511ac6c65f2a89cfcd74fe78aa6d07216095a53cbaeab493b17f6df82cd65b86" +checksum = "de133c5629abab72cad7c4eeab522cb9dfa11ab7ace07d051ef7c29bb54fd370" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -647,9 +646,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d800c8684fa567cdf1abd9654c7997b2a887e7b06022938756193472ec7ec251" +checksum = "119f6f903f5c27308c732f6c7e460aee91e083b3229db2b21fa40436fc60eda7" dependencies = [ "aws-smithy-types", "bytes", @@ -662,15 +661,15 @@ dependencies = [ "percent-encoding", "pin-project", "tokio", - "tokio-util 0.6.10", + "tokio-util 0.7.2", "tracing", ] [[package]] name = "aws-smithy-http-tower" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8017959786cce64e690214d303d062c97fcd38a68df7cb444255e534c9bbce49" +checksum = "991855d24079be0b7485a5753f987f060a153757c7f21729d705f2834366c096" dependencies = [ "aws-smithy-http", "bytes", @@ -683,18 +682,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3796e2a4a3b7d15db2fd5aec2de9220919332648f0a56a77b5c53caf4a9653fa" +checksum = "81c96aa965ba386c7206c0913c6cbdfd27cf6b276e8834a7408b67f04994e647" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76694d1a2dacefa347b921c61bf64b5cc493a898971b3c18fa636ce1788ceabe" +checksum = "14ec1e497d5e75d42787c9440bbbed5fcdb02b0bdc41fd8b7fe270449e66deff" dependencies = [ "aws-smithy-types", "urlencoding", @@ -702,9 +701,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7f957a2250cc0fa4ccf155e00aeac9a81f600df7cd4ecc910c75030e6534f5" +checksum = "b470753cf7e2ce6b7c55f66cb1d17576654878cd41d72bd863a38336569543f4" dependencies = [ "itoa", "num-integer", @@ -714,19 +713,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b01d77433f248d9a2b08f519b403d6aa8435bf144b5d8585862f8dd599eb843" +checksum = "d795da895bc0d8a2ba44550ee5ec928af06db8ecdfd0e42b88d046e97e73dd94" dependencies = [ - "thiserror", "xmlparser", ] [[package]] name = "aws-types" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "394d5c945b747ab3292b94509b78c91191aacfd1deacbcd58371d6f61f8be78a" +checksum = "e5641c99c740574f7a7e70c03773f5ead9f9daf2d95ce6c790ddb4747e000bb0" dependencies = [ "aws-smithy-async", "aws-smithy-client", diff --git a/api/Cargo.toml b/api/Cargo.toml index 09676bb46..969023754 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -5,9 +5,9 @@ edition = "2021" [dependencies] rocket = { version = "0.5.0-rc.1", features = ["uuid", "serde_json", "json"] } -aws-config = "0.11" -aws-sdk-rds = "0.11" -aws-smithy-types = "0.41" +aws-config = "0.12" +aws-sdk-rds = "0.12" +aws-smithy-types = "0.42" cargo = "0.59.0" cargo-util = "0.1" chrono = "0.4" From c206d5ac3153543f170db4e9a19809d9ba7ca8de Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 10:56:00 +0200 Subject: [PATCH 18/61] refactor: finally found it --- api/src/database.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index e73bf79f2..010c8f863 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -285,26 +285,9 @@ impl Context { let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); - let env_provider = EnvironmentVariableCredentialsProvider::new(); - let imds_client = imds::client::Builder::default() - .connect_timeout(Duration::from_secs(30)) - .read_timeout(Duration::from_secs(30)) - .endpoint(Uri::from_static("http://169.254.169.254")) - .build() - .await - .unwrap(); - let imds_provider = imds::credentials::Builder::default() - .profile("BackendAPIRole") - .imds_client(imds_client) - .build(); - - let chained_provider = CredentialsProviderChain::first_try("Environment", env_provider) - .or_else("Ec2InstanceMetadata", imds_provider); - let aws_config = aws_config::from_env() .timeout_config(timeout_config) .region(region_provider) - .credentials_provider(chained_provider) .load() .await; From 310f833b1f918b2e03f022f41e45f56b6776a25e Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 11:00:59 +0200 Subject: [PATCH 19/61] refactor: remove region --- api/src/database.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 010c8f863..f818e1973 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -283,11 +283,9 @@ impl Context { .with_call_timeout(TriState::Set(Duration::from_secs(120))) .with_call_attempt_timeout(TriState::Set(Duration::from_secs(120))); let timeout_config = timeout::Config::new().with_api_timeouts(api_timeout_config); - let region_provider = RegionProviderChain::default_provider().or_else("eu-west-2"); let aws_config = aws_config::from_env() .timeout_config(timeout_config) - .region(region_provider) .load() .await; From ed6ee422e82b5f23439a5ba09cd3548c810a9d30 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 11:09:54 +0200 Subject: [PATCH 20/61] bug: increase hop limit --- terraform/modules/shuttle/service.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index b511f939f..bf5173e93 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -83,6 +83,14 @@ resource "aws_instance" "backend" { iam_instance_profile = aws_iam_instance_profile.backend.id + metadata_options { + http_endpoint = "enabled" + # Our api runs in a container and therefore has an extra hop limit + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#imds-considerations + http_put_response_hop_limit = 2 + http_tokens = "required" + } + network_interface { network_interface_id = aws_network_interface.backend.id device_index = 0 From 589487f1c17ba4a23d0d2e93f531ece3bfa8b454 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 11:32:47 +0200 Subject: [PATCH 21/61] feat: add policy to handle RDS --- terraform/modules/shuttle/service.tf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index bf5173e93..ccb1cf895 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -37,6 +37,25 @@ resource "aws_iam_role" "backend" { ] } EOF + + inline_policy { + name = "Handle_RDS" + policy = jsonencode( + { + Statement = [ + { + Action = [ + "rds:CreateDBInstance", + "rds:ModifyDBInstance", + ] + Effect = "Allow" + Resource = "arn:aws:rds:*:833239102462:db:*" + }, + ] + Version = "2012-10-17" + } + ) + } } resource "aws_lb_target_group_attachment" "api" { From 537a18cd07ce8bc4516196c8d16e7d59bef8143f Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 13:26:06 +0200 Subject: [PATCH 22/61] refactor: try subnet group --- api/src/database.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/database.rs b/api/src/database.rs index f818e1973..d5056f19e 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -191,6 +191,7 @@ impl State { .backup_retention_period(0) .publicly_accessible(true) .db_name(engine.to_string()) + .set_db_subnet_group_name(Some("shuttle_rds".to_string())) .send() .await .map_err(shuttle_service::error::CustomError::new)? From 25d8adafccae9245dd96591ae5c1890cf3a06d21 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 13:44:44 +0200 Subject: [PATCH 23/61] refactor: more permission fixes --- terraform/modules/shuttle/service.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index ccb1cf895..e7b6a72d3 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -46,10 +46,14 @@ EOF { Action = [ "rds:CreateDBInstance", + "rds:DescribeDBInstances", "rds:ModifyDBInstance", ] - Effect = "Allow" - Resource = "arn:aws:rds:*:833239102462:db:*" + Effect = "Allow" + Resource = [ + "arn:aws:rds:*:833239102462:db:*", + "arn:aws:rds:*:833239102462:subgrp:shuttle_rds", + ] }, ] Version = "2012-10-17" From 0023a18e47627cf8897b134f5198ae0adbca79c2 Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 13:52:34 +0200 Subject: [PATCH 24/61] feat: add rds subnet --- terraform/modules/shuttle/rds.tf | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 terraform/modules/shuttle/rds.tf diff --git a/terraform/modules/shuttle/rds.tf b/terraform/modules/shuttle/rds.tf new file mode 100644 index 000000000..7c5ffd1e3 --- /dev/null +++ b/terraform/modules/shuttle/rds.tf @@ -0,0 +1,5 @@ +resource "aws_db_subnet_group" "managed" { + name = "shuttle_rds" + description = "Subnet for RDS instances managed by shuttle" + subnet_ids = [aws_subnet.backend_a.id, aws_subnet.backend_b.id] +} From f26519ade9e878fa8f1d787d3f6b00bb996f630c Mon Sep 17 00:00:00 2001 From: chesedo Date: Fri, 20 May 2022 17:21:28 +0200 Subject: [PATCH 25/61] refactor: cleanup --- api/src/database.rs | 16 +++++++--------- api/src/main.rs | 1 - 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index d5056f19e..23baa2ad2 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -24,6 +24,7 @@ const PUBLIC_PG_IP: &'static str = "pg.shuttle.rs"; const AWS_RDS_CLASS: &str = "db.t4g.micro"; const MASTER_USERNAME: &str = "master"; +const RDS_SUBNET_GROUP: &str = "shuttle_rds"; lazy_static! { static ref SUDO_POSTGRES_CONNECTION_STRING: String = format!( @@ -149,20 +150,17 @@ impl State { &mut self, engine: AwsRdsEngine, ) -> Result { - println!("getting rds"); if self.info.is_some() { // Safe to unwrap since we just confirmed it is `Some` return Ok(self.info.clone().unwrap()); } - println!("getting client"); - error!("getting client"); let client = &self.context.rds_client; let password = generate_role_password(); let instance_name = format!("{}-{}", self.project, engine); - println!("getting modified instance"); + debug!("trying to get AWS RDS instance: {instance_name}"); let instance = client .modify_db_instance() .db_instance_identifier(&instance_name) @@ -170,7 +168,6 @@ impl State { .send() .await; - println!("checking status"); let mut instance = match instance { Ok(instance) => instance .db_instance @@ -178,7 +175,7 @@ impl State { .clone(), Err(SdkError::ServiceError { err, .. }) => { if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { - debug!("creating new AWS RDS for {}", self.project); + debug!("creating new AWS RDS {instance_name}"); client .create_db_instance() @@ -188,10 +185,10 @@ impl State { .engine(engine.to_string()) .db_instance_class(AWS_RDS_CLASS) .allocated_storage(20) - .backup_retention_period(0) + .backup_retention_period(0) // Disable backups .publicly_accessible(true) .db_name(engine.to_string()) - .set_db_subnet_group_name(Some("shuttle_rds".to_string())) + .set_db_subnet_group_name(Some(RDS_SUBNET_GROUP.to_string())) .send() .await .map_err(shuttle_service::error::CustomError::new)? @@ -213,7 +210,7 @@ impl State { }; // Wait for up - debug!("waiting for password update"); + debug!("waiting for AWS RDS instance to be available"); sleep(Duration::from_secs(30)).await; loop { instance = client @@ -280,6 +277,7 @@ impl Context { .connect_timeout(Duration::from_secs(60)) .connect_lazy(&SUDO_POSTGRES_CONNECTION_STRING)?; + // Default timeout is too long so lowering it let api_timeout_config = timeout::Api::new() .with_call_timeout(TriState::Set(Duration::from_secs(120))) .with_call_attempt_timeout(TriState::Set(Duration::from_secs(120))); diff --git a/api/src/main.rs b/api/src/main.rs index f2f591aed..08caa0a09 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -181,7 +181,6 @@ fn main() -> Result<(), Box> { async fn rocket() -> Rocket { env_logger::Builder::new() .filter_module("rocket", log::LevelFilter::Warn) - .filter_module("aws_config", log::LevelFilter::Trace) .filter_module("_", log::LevelFilter::Warn) .filter_module("api", log::LevelFilter::Debug) .init(); From 1aab7a4dc68442dbdce01558ba12312d50bdd43b Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 23 May 2022 15:02:10 +0200 Subject: [PATCH 26/61] feat: switch to attribute annotation --- codegen/src/lib.rs | 153 +++++++++++++++++++++++++++++----------- service/src/database.rs | 16 +++-- service/src/lib.rs | 22 ++++-- 3 files changed, 139 insertions(+), 52 deletions(-) diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 927feadc8..5cd7613a4 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -1,12 +1,14 @@ use proc_macro::TokenStream; use quote::{quote, ToTokens}; -use syn::{parse_macro_input, parse_quote, FnArg, Ident, ItemFn, Pat, ReturnType, Stmt}; +use syn::{ + parse_macro_input, parse_quote, Attribute, FnArg, Ident, ItemFn, Pat, Path, ReturnType, Stmt, +}; #[proc_macro_attribute] pub fn main(_attr: TokenStream, item: TokenStream) -> TokenStream { - let fn_decl = parse_macro_input!(item as ItemFn); + let mut fn_decl = parse_macro_input!(item as ItemFn); - let wrapper = Wrapper::from_item_fn(&fn_decl); + let wrapper = Wrapper::from_item_fn(&mut fn_decl); let expanded = quote! { #wrapper @@ -35,36 +37,33 @@ pub fn main(_attr: TokenStream, item: TokenStream) -> TokenStream { struct Wrapper { fn_ident: Ident, fn_output: ReturnType, - fn_inputs: Vec, + fn_inputs: Vec, +} + +#[derive(Debug, PartialEq)] +struct Input { + ident: Ident, + builder: Option, } impl Wrapper { - fn from_item_fn(item_fn: &ItemFn) -> Self { + fn from_item_fn(item_fn: &mut ItemFn) -> Self { let inputs: Vec<_> = item_fn .sig .inputs - .iter() + .iter_mut() .filter_map(|input| match input { FnArg::Receiver(_) => None, FnArg::Typed(typed) => Some(typed), }) .filter_map(|typed| match typed.pat.as_ref() { - Pat::Ident(ident) => Some(ident), - Pat::TupleStruct(ident) => { - match ident - .pat - .elems - .first() - .as_ref() - .expect("a single pattern item") - { - Pat::Ident(ident) => Some(ident), - _ => None, - } - } + Pat::Ident(ident) => Some((ident, typed.attrs.drain(..).collect())), _ => None, }) - .map(|pat_ident| pat_ident.ident.clone()) + .map(|(pat_ident, attrs)| Input { + ident: pat_ident.ident.clone(), + builder: attribute_to_string(attrs), + }) .collect(); Self { @@ -75,11 +74,22 @@ impl Wrapper { } } +fn attribute_to_string(attrs: Vec) -> Option { + if attrs.is_empty() { + return None; + } + + let builder = attrs[0].path.clone(); + + Some(builder) +} + impl ToTokens for Wrapper { fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { let fn_output = &self.fn_output; let fn_ident = &self.fn_ident; - let fn_inputs = &self.fn_inputs; + let fn_inputs: Vec<_> = self.fn_inputs.iter().map(|i| i.ident.clone()).collect(); + let fn_inputs_attrs: Vec<_> = self.fn_inputs.iter().map(|i| i.builder.clone()).collect(); let factory_ident: Ident = if self.fn_inputs.is_empty() { parse_quote!(_factory) @@ -91,7 +101,7 @@ impl ToTokens for Wrapper { None } else { Some(parse_quote!( - use shuttle_service::GetResource; + use shuttle_service::ResourceBuilder; )) }; @@ -110,7 +120,7 @@ impl ToTokens for Wrapper { }).await.unwrap(); - #(let #fn_inputs = #factory_ident.get_resource(runtime).await?;)* + #(let #fn_inputs = shuttle_service::#fn_inputs_attrs::new().build(#factory_ident, runtime).await?;)* runtime.spawn(#fn_ident(#(#fn_inputs),*)).await.unwrap() } @@ -126,20 +136,20 @@ mod tests { use quote::quote; use syn::{parse_quote, Ident, ReturnType}; - use crate::Wrapper; + use crate::{Input, Wrapper}; #[test] fn from_missing_return() { - let input = parse_quote!( + let mut input = parse_quote!( async fn simple() {} ); - let actual = Wrapper::from_item_fn(&input); + let actual = Wrapper::from_item_fn(&mut input); let expected_ident: Ident = parse_quote!(simple); assert_eq!(actual.fn_ident, expected_ident); assert_eq!(actual.fn_output, ReturnType::Default); - assert_eq!(actual.fn_inputs, Vec::::new()); + assert_eq!(actual.fn_inputs, Vec::::new()); } #[test] @@ -172,17 +182,17 @@ mod tests { #[test] fn from_with_return() { - let input = parse_quote!( + let mut input = parse_quote!( async fn complex() -> Result<(), Box> {} ); - let actual = Wrapper::from_item_fn(&input); + let actual = Wrapper::from_item_fn(&mut input); let expected_ident: Ident = parse_quote!(complex); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); assert_eq!(actual.fn_ident, expected_ident); assert_eq!(actual.fn_output, expected_output); - assert_eq!(actual.fn_inputs, Vec::::new()); + assert_eq!(actual.fn_inputs, Vec::::new()); } #[test] @@ -215,14 +225,17 @@ mod tests { #[test] fn from_with_inputs() { - let input = parse_quote!( + let mut input = parse_quote!( async fn complex(pool: PgPool) -> Result<(), Box> {} ); - let actual = Wrapper::from_item_fn(&input); + let actual = Wrapper::from_item_fn(&mut input); let expected_ident: Ident = parse_quote!(complex); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); - let expected_inputs: Vec = vec![parse_quote!(pool)]; + let expected_inputs: Vec = vec![Input { + ident: parse_quote!(pool), + builder: None, + }]; assert_eq!(actual.fn_ident, expected_ident); assert_eq!(actual.fn_output, expected_output); @@ -234,7 +247,16 @@ mod tests { let input = Wrapper { fn_ident: parse_quote!(complex), fn_output: parse_quote!(-> Result<(), Box>), - fn_inputs: vec![parse_quote!(pool), parse_quote!(redis)], + fn_inputs: vec![ + Input { + ident: parse_quote!(pool), + builder: None, + }, + Input { + ident: parse_quote!(redis), + builder: None, + }, + ], }; let actual = quote!(#input); @@ -244,7 +266,7 @@ mod tests { runtime: &shuttle_service::Runtime, logger: shuttle_service::Logger, ) -> Result<(), Box > { - use shuttle_service::GetResource; + use shuttle_service::ResourceBuilder; runtime.spawn_blocking(move || { shuttle_service::log::set_boxed_logger(Box::new(logger)) @@ -263,18 +285,67 @@ mod tests { } #[test] - fn from_tuple_input() { - let input = parse_quote!( - async fn tuple(Postgres(pool): Postgres) -> Result<(), Box> {} + fn from_input_attribute() { + let mut input = parse_quote!( + async fn meta(#[aws::rds] pool: PgPool) -> Result<(), Box> {} ); - let actual = Wrapper::from_item_fn(&input); - let expected_ident: Ident = parse_quote!(tuple); + let actual = Wrapper::from_item_fn(&mut input); + let expected_ident: Ident = parse_quote!(meta); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); - let expected_inputs: Vec = vec![parse_quote!(pool)]; + let expected_inputs: Vec = vec![Input { + ident: parse_quote!(pool), + builder: Some(parse_quote!(aws::rds)), + }]; assert_eq!(actual.fn_ident, expected_ident); assert_eq!(actual.fn_output, expected_output); assert_eq!(actual.fn_inputs, expected_inputs); + + // Make sure attributes was removed from input + if let syn::FnArg::Typed(param) = input.sig.inputs.first().unwrap() { + assert!( + param.attrs.is_empty(), + "some attributes were not removed: {:?}", + param.attrs + ); + } else { + panic!("expected first input to be typed") + } + } + + #[test] + fn output_with_input_attributes() { + let input = Wrapper { + fn_ident: parse_quote!(complex), + fn_output: parse_quote!(-> Result<(), Box>), + fn_inputs: vec![Input { + ident: parse_quote!(pool), + builder: Some(parse_quote!(aws::rds)), + }], + }; + + let actual = quote!(#input); + let expected = quote! { + async fn __shuttle_wrapper( + factory: &mut dyn shuttle_service::Factory, + runtime: &shuttle_service::Runtime, + logger: shuttle_service::Logger, + ) -> Result<(), Box > { + use shuttle_service::ResourceBuilder; + + runtime.spawn_blocking(move || { + shuttle_service::log::set_boxed_logger(Box::new(logger)) + .map(|()| shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info)) + .expect("logger set should succeed"); + }).await.unwrap(); + + let pool = shuttle_service::aws::rds::new().build(factory, runtime).await?; + + runtime.spawn(complex(pool)).await.unwrap() + } + }; + + assert_eq!(actual.to_string(), expected.to_string()); } } diff --git a/service/src/database.rs b/service/src/database.rs index 62e22bb09..732f4ebf0 100644 --- a/service/src/database.rs +++ b/service/src/database.rs @@ -26,14 +26,18 @@ macro_rules! aws_engine { paste! { #[cfg(feature = $feature)] #[doc = "A resource connected to an AWS RDS " $struct_ident " instance"] - pub struct $struct_ident(pub $pool_path); + pub struct $struct_ident; #[cfg(feature = $feature)] #[doc = "Gets a `sqlx::Pool` connected to an AWS RDS " $struct_ident " instance"] #[async_trait] - impl GetResource<$struct_ident> for &mut dyn Factory { - async fn get_resource(self, runtime: &Runtime) -> Result<$struct_ident, crate::Error> { - let connection_string = self + impl ResourceBuilder<$pool_path> for $struct_ident { + fn new() -> Self { + Self {} + } + + async fn build(self, factory: &mut dyn Factory, runtime: &Runtime) -> Result<$pool_path, crate::Error> { + let connection_string = factory .get_sql_connection_string(Type::AwsRds(AwsRdsEngine::$struct_ident)) .await?; @@ -50,7 +54,7 @@ macro_rules! aws_engine { .map_err(CustomError::new)? .map_err(CustomError::new)?; - Ok($struct_ident(pool)) + Ok(pool) } } } @@ -59,7 +63,7 @@ macro_rules! aws_engine { pub mod aws_rds { use super::{AwsRdsEngine, Type}; - use crate::{error::CustomError, Factory, GetResource}; + use crate::{error::CustomError, Factory, ResourceBuilder}; use async_trait::async_trait; use paste::paste; use tokio::runtime::Runtime; diff --git a/service/src/lib.rs b/service/src/lib.rs index f1a65c545..edf543d09 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -265,18 +265,30 @@ pub trait Factory: Send + Sync { /// Some resources cannot cross the boundary between the api runtime and the runtime of services. These resources /// should be created on the passed in runtime. #[async_trait] -pub trait GetResource { - async fn get_resource(self, runtime: &Runtime) -> Result; +pub trait ResourceBuilder { + fn new() -> Self; + async fn build(self, factory: &mut dyn Factory, runtime: &Runtime) -> Result; } +#[cfg(feature = "sqlx-postgres")] +pub struct Postgres; + /// Get an `sqlx::PgPool` from any factory #[cfg(feature = "sqlx-postgres")] #[async_trait] -impl GetResource for &mut dyn Factory { - async fn get_resource(self, runtime: &Runtime) -> Result { +impl ResourceBuilder for Postgres { + fn new() -> Self { + Self {} + } + + async fn build( + self, + factory: &mut dyn Factory, + runtime: &Runtime, + ) -> Result { use error::CustomError; - let connection_string = self + let connection_string = factory .get_sql_connection_string(database::Type::Shared) .await?; From e21e00563d8119748122ac60d6566c1b813d31de Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 23 May 2022 15:15:09 +0200 Subject: [PATCH 27/61] refactor: move to aws::rds --- service/src/aws/mod.rs | 1 + service/src/aws/rds.rs | 69 +++++++++++++++++++++++++++++++++++++++++ service/src/database.rs | 69 ----------------------------------------- service/src/lib.rs | 7 +++++ 4 files changed, 77 insertions(+), 69 deletions(-) create mode 100644 service/src/aws/mod.rs create mode 100644 service/src/aws/rds.rs diff --git a/service/src/aws/mod.rs b/service/src/aws/mod.rs new file mode 100644 index 000000000..83d80c0c8 --- /dev/null +++ b/service/src/aws/mod.rs @@ -0,0 +1 @@ +pub mod rds; diff --git a/service/src/aws/rds.rs b/service/src/aws/rds.rs new file mode 100644 index 000000000..dc3d2fbf4 --- /dev/null +++ b/service/src/aws/rds.rs @@ -0,0 +1,69 @@ +use crate::{ + database::{AwsRdsEngine, Type}, + error::CustomError, + Factory, ResourceBuilder, +}; +use async_trait::async_trait; +use paste::paste; +use tokio::runtime::Runtime; + +macro_rules! aws_engine { + ($feature:expr, $pool_path:path, $options_path:path, $struct_ident:ident) => { + paste! { + #[cfg(feature = $feature)] + #[doc = "A resource connected to an AWS RDS " $struct_ident " instance"] + pub struct $struct_ident; + + #[cfg(feature = $feature)] + #[doc = "Gets a `sqlx::Pool` connected to an AWS RDS " $struct_ident " instance"] + #[async_trait] + impl ResourceBuilder<$pool_path> for $struct_ident { + fn new() -> Self { + Self {} + } + + async fn build(self, factory: &mut dyn Factory, runtime: &Runtime) -> Result<$pool_path, crate::Error> { + let connection_string = factory + .get_sql_connection_string(Type::AwsRds(AwsRdsEngine::$struct_ident)) + .await?; + + // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end + let pool = runtime + .spawn(async move { + $options_path::new() + .min_connections(1) + .max_connections(5) + .connect(&connection_string) + .await + }) + .await + .map_err(CustomError::new)? + .map_err(CustomError::new)?; + + Ok(pool) + } + } + } + }; +} + +aws_engine!( + "sqlx-aws-postgres", + sqlx::PgPool, + sqlx::postgres::PgPoolOptions, + Postgres +); + +aws_engine!( + "sqlx-aws-mysql", + sqlx::MySqlPool, + sqlx::mysql::MySqlPoolOptions, + MySql +); + +aws_engine!( + "sqlx-aws-mariadb", + sqlx::MySqlPool, + sqlx::mysql::MySqlPoolOptions, + MariaDB +); diff --git a/service/src/database.rs b/service/src/database.rs index 732f4ebf0..b128b2e60 100644 --- a/service/src/database.rs +++ b/service/src/database.rs @@ -20,72 +20,3 @@ impl Display for AwsRdsEngine { } } } - -macro_rules! aws_engine { - ($feature:expr, $pool_path:path, $options_path:path, $struct_ident:ident) => { - paste! { - #[cfg(feature = $feature)] - #[doc = "A resource connected to an AWS RDS " $struct_ident " instance"] - pub struct $struct_ident; - - #[cfg(feature = $feature)] - #[doc = "Gets a `sqlx::Pool` connected to an AWS RDS " $struct_ident " instance"] - #[async_trait] - impl ResourceBuilder<$pool_path> for $struct_ident { - fn new() -> Self { - Self {} - } - - async fn build(self, factory: &mut dyn Factory, runtime: &Runtime) -> Result<$pool_path, crate::Error> { - let connection_string = factory - .get_sql_connection_string(Type::AwsRds(AwsRdsEngine::$struct_ident)) - .await?; - - // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end - let pool = runtime - .spawn(async move { - $options_path::new() - .min_connections(1) - .max_connections(5) - .connect(&connection_string) - .await - }) - .await - .map_err(CustomError::new)? - .map_err(CustomError::new)?; - - Ok(pool) - } - } - } - }; -} - -pub mod aws_rds { - use super::{AwsRdsEngine, Type}; - use crate::{error::CustomError, Factory, ResourceBuilder}; - use async_trait::async_trait; - use paste::paste; - use tokio::runtime::Runtime; - - aws_engine!( - "sqlx-aws-postgres", - sqlx::PgPool, - sqlx::postgres::PgPoolOptions, - Postgres - ); - - aws_engine!( - "sqlx-aws-mysql", - sqlx::MySqlPool, - sqlx::mysql::MySqlPoolOptions, - MySql - ); - - aws_engine!( - "sqlx-aws-mariadb", - sqlx::MySqlPool, - sqlx::mysql::MySqlPoolOptions, - MariaDB - ); -} diff --git a/service/src/lib.rs b/service/src/lib.rs index edf543d09..2c79f32db 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -185,6 +185,13 @@ pub mod secrets; #[cfg(feature = "secrets")] pub use secrets::SecretStore; +#[cfg(any( + feature = "sqlx-aws-mariadb", + feature = "sqlx-aws-mysql", + feature = "sqlx-aws-postgres" +))] +pub mod aws; + #[cfg(feature = "codegen")] extern crate shuttle_codegen; #[cfg(feature = "codegen")] From ff6cbef8bbc021b9451080f722c76b7d4dfe0d5f Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 23 May 2022 15:16:03 +0200 Subject: [PATCH 28/61] feat: aws::rds example --- examples/tide/postgres/Cargo.toml | 15 ++++++++ examples/tide/postgres/Shuttle.toml | 1 + examples/tide/postgres/schema.sql | 6 +++ examples/tide/postgres/src/lib.rs | 58 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 examples/tide/postgres/Cargo.toml create mode 100644 examples/tide/postgres/Shuttle.toml create mode 100644 examples/tide/postgres/schema.sql create mode 100644 examples/tide/postgres/src/lib.rs diff --git a/examples/tide/postgres/Cargo.toml b/examples/tide/postgres/Cargo.toml new file mode 100644 index 000000000..cf5d07b19 --- /dev/null +++ b/examples/tide/postgres/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "postgres" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tide = "0.16.0" +serde = { version = "1.0", features = ["derive"] } +sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } +shuttle-service = { version = "0.2", features = ["sqlx-aws-postgres", "web-tide"] } diff --git a/examples/tide/postgres/Shuttle.toml b/examples/tide/postgres/Shuttle.toml new file mode 100644 index 000000000..4ffbeb3e5 --- /dev/null +++ b/examples/tide/postgres/Shuttle.toml @@ -0,0 +1 @@ +name = "postgres-tide-app" diff --git a/examples/tide/postgres/schema.sql b/examples/tide/postgres/schema.sql new file mode 100644 index 000000000..460e7c23d --- /dev/null +++ b/examples/tide/postgres/schema.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS todos; + +CREATE TABLE todos ( + id serial PRIMARY KEY, + note TEXT NOT NULL +); diff --git a/examples/tide/postgres/src/lib.rs b/examples/tide/postgres/src/lib.rs new file mode 100644 index 000000000..b36f5846a --- /dev/null +++ b/examples/tide/postgres/src/lib.rs @@ -0,0 +1,58 @@ +use serde::{Deserialize, Serialize}; +use shuttle_service::error::CustomError; +use sqlx::{Executor, FromRow, PgPool}; +use tide::{Body, Request}; + +async fn retrieve(req: Request) -> tide::Result { + let id = req.param("id")?; + let todo: Todo = sqlx::query_as("SELECT * FROM todos WHERE id = $1") + .bind(id) + .fetch_one(&req.state().pool) + .await?; + + Body::from_json(&todo).map(core::convert::Into::into) +} + +async fn add(mut req: Request) -> tide::Result { + let data: TodoNew = req.body_json().await?; + let todo: Todo = sqlx::query_as("INSERT INTO todos(note) VALUES ($1) RETURNING id, note") + .bind(&data.note) + .fetch_one(&req.state().pool) + .await?; + + Body::from_json(&todo).map(core::convert::Into::into) +} + +#[derive(Clone)] +struct MyState { + pool: PgPool, +} + +#[shuttle_service::main] +async fn tide( + #[aws::rds::Postgres] pool: PgPool, +) -> Result, shuttle_service::Error> { + pool.execute(include_str!("../schema.sql")) + .await + .map_err(CustomError::new)?; + + let state = MyState { pool }; + let mut app = tide::with_state(state); + + app.with(tide::log::LogMiddleware::new()); + app.at("/todo/:id").get(retrieve); + app.at("/todo").post(add); + + Ok(app) +} + +#[derive(Deserialize)] +struct TodoNew { + pub note: String, +} + +#[derive(Serialize, FromRow)] +struct Todo { + pub id: i32, + pub note: String, +} From f088eff0545339f14be8cd5452649fd3071bef74 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 23 May 2022 15:37:46 +0200 Subject: [PATCH 29/61] refactor: move to shared::Postgres --- examples/rocket/postgres/src/lib.rs | 12 ++++++--- examples/url-shortener/src/lib.rs | 4 +-- service/src/lib.rs | 42 +++-------------------------- service/src/shared.rs | 39 +++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 45 deletions(-) create mode 100644 service/src/shared.rs diff --git a/examples/rocket/postgres/src/lib.rs b/examples/rocket/postgres/src/lib.rs index faede01bf..2e61dc89a 100644 --- a/examples/rocket/postgres/src/lib.rs +++ b/examples/rocket/postgres/src/lib.rs @@ -1,12 +1,12 @@ #[macro_use] extern crate rocket; -use shuttle_service::SecretStore; -use shuttle_service::error::CustomError; use rocket::response::status::BadRequest; use rocket::serde::json::Json; use rocket::State; use serde::{Deserialize, Serialize}; +use shuttle_service::error::CustomError; +use shuttle_service::SecretStore; use sqlx::{Executor, FromRow, PgPool}; #[get("/")] @@ -37,7 +37,11 @@ async fn add( #[get("/secret")] async fn secret(state: &State) -> Result> { // get secret defined in `Secrets.toml` file. - state.pool.get_secret("MY_API_KEY").await.map_err(|e| BadRequest(Some(e.to_string()))) + state + .pool + .get_secret("MY_API_KEY") + .await + .map_err(|e| BadRequest(Some(e.to_string()))) } struct MyState { @@ -45,7 +49,7 @@ struct MyState { } #[shuttle_service::main] -async fn rocket(pool: PgPool) -> shuttle_service::ShuttleRocket { +async fn rocket(#[shared::Postgres] pool: PgPool) -> shuttle_service::ShuttleRocket { pool.execute(include_str!("../schema.sql")) .await .map_err(CustomError::new)?; diff --git a/examples/url-shortener/src/lib.rs b/examples/url-shortener/src/lib.rs index 84475cbfc..a2a3d5537 100644 --- a/examples/url-shortener/src/lib.rs +++ b/examples/url-shortener/src/lib.rs @@ -6,7 +6,7 @@ use rocket::{ response::{status, Redirect}, routes, Build, Rocket, State, }; -use serde::{Serialize}; +use serde::Serialize; use shuttle_service::error::CustomError; use sqlx::migrate::Migrator; use sqlx::{FromRow, PgPool}; @@ -71,7 +71,7 @@ async fn shorten(url: String, state: &State) -> Result Result, shuttle_service::Error> { +async fn rocket(#[shared::Postgres] pool: PgPool) -> Result, shuttle_service::Error> { MIGRATOR.run(&pool).await.map_err(CustomError::new)?; let state = AppState { pool }; diff --git a/service/src/lib.rs b/service/src/lib.rs index 2c79f32db..cbaf6d443 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -180,6 +180,9 @@ pub use error::Error; pub mod logger; +#[cfg(feature = "sqlx-postgres")] +pub mod shared; + #[cfg(feature = "secrets")] pub mod secrets; #[cfg(feature = "secrets")] @@ -277,45 +280,6 @@ pub trait ResourceBuilder { async fn build(self, factory: &mut dyn Factory, runtime: &Runtime) -> Result; } -#[cfg(feature = "sqlx-postgres")] -pub struct Postgres; - -/// Get an `sqlx::PgPool` from any factory -#[cfg(feature = "sqlx-postgres")] -#[async_trait] -impl ResourceBuilder for Postgres { - fn new() -> Self { - Self {} - } - - async fn build( - self, - factory: &mut dyn Factory, - runtime: &Runtime, - ) -> Result { - use error::CustomError; - - let connection_string = factory - .get_sql_connection_string(database::Type::Shared) - .await?; - - // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end - let pool = runtime - .spawn(async move { - sqlx::postgres::PgPoolOptions::new() - .min_connections(1) - .max_connections(5) - .connect(&connection_string) - .await - }) - .await - .map_err(CustomError::new)? - .map_err(CustomError::new)?; - - Ok(pool) - } -} - /// A tokio handle the service was started on pub type ServeHandle = JoinHandle>; diff --git a/service/src/shared.rs b/service/src/shared.rs new file mode 100644 index 000000000..a1708fcf9 --- /dev/null +++ b/service/src/shared.rs @@ -0,0 +1,39 @@ +use tokio::runtime::Runtime; + +use crate::{database, error::CustomError, Factory, ResourceBuilder}; +use async_trait::async_trait; + +pub struct Postgres; + +/// Get an `sqlx::PgPool` from any factory +#[async_trait] +impl ResourceBuilder for Postgres { + fn new() -> Self { + Self {} + } + + async fn build( + self, + factory: &mut dyn Factory, + runtime: &Runtime, + ) -> Result { + let connection_string = factory + .get_sql_connection_string(database::Type::Shared) + .await?; + + // A sqlx Pool cannot cross runtime boundaries, so make sure to create the Pool on the service end + let pool = runtime + .spawn(async move { + sqlx::postgres::PgPoolOptions::new() + .min_connections(1) + .max_connections(5) + .connect(&connection_string) + .await + }) + .await + .map_err(CustomError::new)? + .map_err(CustomError::new)?; + + Ok(pool) + } +} From 0c550e06c194f76c54ae157cde1735954b486fa2 Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 07:10:14 +0200 Subject: [PATCH 30/61] refactor: improve rds waiting --- api/src/database.rs | 84 +++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/api/src/database.rs b/api/src/database.rs index 23baa2ad2..cad7f0dad 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -1,13 +1,7 @@ use anyhow::anyhow; -use aws_config::{ - environment::EnvironmentVariableCredentialsProvider, - imds, - meta::{credentials::CredentialsProviderChain, region::RegionProviderChain}, - timeout, -}; -use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, types::SdkError}; +use aws_config::timeout; +use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, model::DbInstance, types::SdkError, Client}; use aws_smithy_types::tristate::TriState; -use hyper::Uri; use lazy_static::lazy_static; use rand::Rng; use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; @@ -168,11 +162,10 @@ impl State { .send() .await; - let mut instance = match instance { - Ok(instance) => instance - .db_instance - .expect("aws response should contain an instance") - .clone(), + match instance { + Ok(_) => { + wait_for_instance(client, &&instance_name, "resetting-master-credentials").await?; + } Err(SdkError::ServiceError { err, .. }) => { if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { debug!("creating new AWS RDS {instance_name}"); @@ -193,7 +186,9 @@ impl State { .await .map_err(shuttle_service::error::CustomError::new)? .db_instance - .expect("to be able to create instance") + .expect("to be able to create instance"); + + wait_for_instance(client, &&instance_name, "creating").await?; } else { return Err(shuttle_service::Error::Custom(anyhow!( "got unexpected error from AWS RDS service: {}", @@ -210,33 +205,7 @@ impl State { }; // Wait for up - debug!("waiting for AWS RDS instance to be available"); - sleep(Duration::from_secs(30)).await; - loop { - instance = client - .describe_db_instances() - .db_instance_identifier(&instance_name) - .send() - .await - .map_err(CustomError::new)? - .db_instances - .expect("aws to return instances") - .get(0) - .expect("to find the instance just created or modified") - .clone(); - - let status = instance - .db_instance_status - .as_ref() - .expect("instance to have a status") - .clone(); - - if status == "available" { - break; - } - - sleep(Duration::from_secs(1)).await; - } + let instance = wait_for_instance(client, &&instance_name, "available").await?; let address = instance .endpoint @@ -263,6 +232,39 @@ impl State { } } +async fn wait_for_instance( + client: &Client, + name: &str, + wait_for: &str, +) -> Result { + debug!("waiting for {name} to enter {wait_for} state"); + loop { + let instance = client + .describe_db_instances() + .db_instance_identifier(name) + .send() + .await + .map_err(CustomError::new)? + .db_instances + .expect("aws to return instances") + .get(0) + .expect("to find the instance just created or modified") + .clone(); + + let status = instance + .db_instance_status + .as_ref() + .expect("instance to have a status") + .clone(); + + if status == wait_for { + return Ok(instance); + } + + sleep(Duration::from_secs(1)).await; + } +} + #[derive(Clone)] pub struct Context { sudo_pool: PgPool, From cca3a59ed4aa3cb162c89c5fefe24aded53f5ead Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 08:21:32 +0200 Subject: [PATCH 31/61] feat: open acl ports for acl --- terraform/modules/shuttle/networking.tf | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/terraform/modules/shuttle/networking.tf b/terraform/modules/shuttle/networking.tf index 275451b9b..8c7f05224 100644 --- a/terraform/modules/shuttle/networking.tf +++ b/terraform/modules/shuttle/networking.tf @@ -8,26 +8,26 @@ resource "aws_internet_gateway" "public" { vpc_id = aws_vpc.backend.id } -resource "aws_network_acl" "unreasonable" { - vpc_id = aws_vpc.backend.id - - egress { - protocol = "tcp" - rule_no = 100 - action = "allow" - cidr_block = "10.0.0.0/16" - from_port = 0 - to_port = 65535 - } +resource "aws_network_acl_rule" "postgres" { + network_acl_id = aws_vpc.backend.default_network_acl_id + rule_number = 10 + egress = false + protocol = "tcp" + rule_action = "allow" + cidr_block = "0.0.0.0/0" + from_port = 5432 + to_port = 5432 +} - ingress { - protocol = "tcp" - rule_no = 100 - action = "allow" - cidr_block = "10.0.0.0/16" - from_port = 0 - to_port = 65535 - } +resource "aws_network_acl_rule" "mysql" { + network_acl_id = aws_vpc.backend.default_network_acl_id + rule_number = 11 + egress = false + protocol = "tcp" + rule_action = "allow" + cidr_block = "0.0.0.0/0" + from_port = 3306 + to_port = 3306 } resource "aws_security_group" "unreasonable" { From 9f8f4ac7c1c85f9fbc5765b9da1df2f4b4917b53 Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 08:27:43 +0200 Subject: [PATCH 32/61] bug: running tokio inside tide (async-std) --- Cargo.lock | 2 ++ api/users.toml | 1 + examples/tide/postgres/src/lib.rs | 4 ++-- service/Cargo.toml | 6 ++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f22e17ebd..b26fd94d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,6 +241,7 @@ dependencies = [ "futures-lite", "num_cpus", "once_cell", + "tokio", ] [[package]] @@ -3967,6 +3968,7 @@ name = "shuttle-service" version = "0.3.1" dependencies = [ "anyhow", + "async-std", "async-trait", "axum", "chrono", diff --git a/api/users.toml b/api/users.toml index 32852fb78..c062affbf 100644 --- a/api/users.toml +++ b/api/users.toml @@ -7,4 +7,5 @@ projects = [ 'authentication-rocket-app', 'hello-world-tide-app', 'hello-world-tower-app', + 'postgres-tide-app', ] diff --git a/examples/tide/postgres/src/lib.rs b/examples/tide/postgres/src/lib.rs index b36f5846a..88c0beef6 100644 --- a/examples/tide/postgres/src/lib.rs +++ b/examples/tide/postgres/src/lib.rs @@ -4,7 +4,7 @@ use sqlx::{Executor, FromRow, PgPool}; use tide::{Body, Request}; async fn retrieve(req: Request) -> tide::Result { - let id = req.param("id")?; + let id: i32 = req.param("id")?.parse()?; let todo: Todo = sqlx::query_as("SELECT * FROM todos WHERE id = $1") .bind(id) .fetch_one(&req.state().pool) @@ -40,8 +40,8 @@ async fn tide( let mut app = tide::with_state(state); app.with(tide::log::LogMiddleware::new()); - app.at("/todo/:id").get(retrieve); app.at("/todo").post(add); + app.at("/todo/:id").get(retrieve); Ok(app) } diff --git a/service/Cargo.toml b/service/Cargo.toml index 9616c527d..c68bb528f 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -51,3 +51,9 @@ web-axum = ["axum", "sync_wrapper"] web-rocket = ["rocket"] web-tide = ["tide"] web-tower = ["tower", "hyper"] + +# Tide does not have tokio support. So make sure async-std is compatible with tokio +# https://github.com/http-rs/tide/issues/791 +[dependencies.async-std] +version = "1" +features = ["tokio1"] From 9bf234b15c2b95bdb2cf5c9d1ac059052aae5363 Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 10:05:17 +0200 Subject: [PATCH 33/61] docs: service configuration using attributes --- service/src/lib.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/service/src/lib.rs b/service/src/lib.rs index cbaf6d443..80da7ad96 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -105,7 +105,7 @@ //! } //! //! #[shuttle_service::main] -//! async fn rocket(pool: PgPool) -> ShuttleRocket { +//! async fn rocket(#[shared::Postgres] pool: PgPool) -> ShuttleRocket { //! let state = MyState(pool); //! let rocket = rocket::build().manage(state).mount("/", routes![hello]); //! @@ -225,7 +225,7 @@ extern crate shuttle_codegen; /// | [`Server`](https://docs.rs/tide/latest/tide/struct.Server.html) | web-tide | [tide](https://docs.rs/tide/0.16.0) | 0.16.0 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/tide/hello-world) | /// /// # Getting shuttle managed services -/// The shuttle is able to manage service dependencies for you. These services are passed in as inputs to your main function: +/// The shuttle is able to manage service dependencies for you. These services are passed in as inputs to your main function and are configured using attributes: /// ```rust,no_run /// use sqlx::PgPool; /// use shuttle_service::ShuttleRocket; @@ -233,7 +233,7 @@ extern crate shuttle_codegen; /// struct MyState(PgPool); /// /// #[shuttle_service::main] -/// async fn rocket(pool: PgPool) -> ShuttleRocket { +/// async fn rocket(#[shared::Postgres] pool: PgPool) -> ShuttleRocket { /// let state = MyState(pool); /// let rocket = rocket::build().manage(state); /// @@ -242,11 +242,14 @@ extern crate shuttle_codegen; /// ``` /// /// ## shuttle managed dependencies -/// The following dependencies can be managed by shuttle - remember to enable their feature flags for the `shuttle-service` dependency in `Cargo.toml`: -/// -/// | Argument type | Feature flag | Dependency | Example | -/// | ------------------------------------------------------------- | ------------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -/// | [`PgPool`](https://docs.rs/sqlx/latest/sqlx/type.PgPool.html) | sqlx-postgres | A PostgresSql instance accessed using [sqlx](https://docs.rs/sqlx) | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/rocket/postgres) | +/// The following dependencies can be managed by shuttle - remember to enable their feature flags for the `shuttle-service` dependency in `Cargo.toml` and configure them using an attribute annotation: +/// +/// | Argument type | Feature flag | Attribute | Dependency | Example | +/// | ------------------------------------------------------------------- | ----------------- | -------------------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | +/// | [`PgPool`](https://docs.rs/sqlx/latest/sqlx/type.PgPool.html) | sqlx-postgres | `shared::Postgres` | A shared PostgresSql instance accessed using [sqlx](https://docs.rs/sqlx) | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/rocket/postgres) | +/// | [`MySqlPool`](https://docs.rs/sqlx/latest/sqlx/type.MySqlPool.html) | sqlx-aws-mariadb | `aws::rds::MariaDB` | An AWS RDS MariaDB instance tied to your instance and accessed using [sqlx](https://docs.rs/sqlx) | | +/// | [`MySqlPool`](https://docs.rs/sqlx/latest/sqlx/type.MySqlPool.html) | sqlx-aws-mysql | `aws::rds::MySql` | An AWS RDS MySql instance tied to your instance and accessed using [sqlx](https://docs.rs/sqlx) | | +/// | [`PgPool`](https://docs.rs/sqlx/latest/sqlx/type.PgPool.html) | sqlx-aws-postgres | `aws::rds::Postgres` | An AWS RDS Postgres instance tied to your instance and accessed using [sqlx](https://docs.rs/sqlx) | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/tide/postgres) | pub use shuttle_codegen::main; use tokio::task::JoinHandle; From f7950e9d9c2084826fcce2b15283a0fa8c83bc95 Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 10:13:30 +0200 Subject: [PATCH 34/61] bug: fix uuid/v4 in test --- Cargo.lock | 3 +++ service/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b26fd94d4..5b3e066d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4874,6 +4874,9 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.6", +] [[package]] name = "uuid" diff --git a/service/Cargo.toml b/service/Cargo.toml index c68bb528f..7d4fb68d2 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -32,7 +32,7 @@ shuttle-common = { version = "0.3.0", path = "../common" } [dev-dependencies] portpicker = "0.1.1" -uuid = "0.8.2" +uuid = { version = "0.8.2", features = ["v4"] } [features] default = ["codegen"] From 9de5977704cfa6053aa5928e68047ccc214e30f5 Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 10:22:47 +0200 Subject: [PATCH 35/61] refactor: clippy suggestions --- Cargo.lock | 19 +++++-------------- api/src/database.rs | 6 +++--- api/src/main.rs | 2 +- service/Cargo.toml | 2 +- 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b3e066d4..ca20de704 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,7 +163,7 @@ dependencies = [ "structopt", "tokio", "toml", - "uuid 1.0.0", + "uuid", ] [[package]] @@ -3566,7 +3566,7 @@ dependencies = [ "tokio-stream", "tokio-util 0.7.2", "ubyte", - "uuid 1.0.0", + "uuid", "version_check", "yansi", ] @@ -3612,7 +3612,7 @@ dependencies = [ "time 0.3.9", "tokio", "uncased", - "uuid 1.0.0", + "uuid", ] [[package]] @@ -3960,7 +3960,7 @@ dependencies = [ "rocket", "serde", "serde_json", - "uuid 1.0.0", + "uuid", ] [[package]] @@ -3988,7 +3988,7 @@ dependencies = [ "tide", "tokio", "tower", - "uuid 0.8.2", + "uuid", ] [[package]] @@ -4869,15 +4869,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.6", -] - [[package]] name = "uuid" version = "1.0.0" diff --git a/api/src/database.rs b/api/src/database.rs index cad7f0dad..856046589 100644 --- a/api/src/database.rs +++ b/api/src/database.rs @@ -164,7 +164,7 @@ impl State { match instance { Ok(_) => { - wait_for_instance(client, &&instance_name, "resetting-master-credentials").await?; + wait_for_instance(client, &instance_name, "resetting-master-credentials").await?; } Err(SdkError::ServiceError { err, .. }) => { if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { @@ -188,7 +188,7 @@ impl State { .db_instance .expect("to be able to create instance"); - wait_for_instance(client, &&instance_name, "creating").await?; + wait_for_instance(client, &instance_name, "creating").await?; } else { return Err(shuttle_service::Error::Custom(anyhow!( "got unexpected error from AWS RDS service: {}", @@ -205,7 +205,7 @@ impl State { }; // Wait for up - let instance = wait_for_instance(client, &&instance_name, "available").await?; + let instance = wait_for_instance(client, &instance_name, "available").await?; let address = instance .endpoint diff --git a/api/src/main.rs b/api/src/main.rs index 08caa0a09..b200a03ea 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -171,7 +171,7 @@ fn main() -> Result<(), Box> { .build() .unwrap() .block_on(async { - rocket().await.launch().await?; + let _rocket = rocket().await.launch().await?; Ok(()) }) diff --git a/service/Cargo.toml b/service/Cargo.toml index 7d4fb68d2..4cb96a3fb 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -32,7 +32,7 @@ shuttle-common = { version = "0.3.0", path = "../common" } [dev-dependencies] portpicker = "0.1.1" -uuid = { version = "0.8.2", features = ["v4"] } +uuid = { version = "1.0.0", features = ["v4"] } [features] default = ["codegen"] From 428f1d713a355892dda7dea556e2aeb99bfe0edd Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 10:26:13 +0200 Subject: [PATCH 36/61] tests: update sqlx tests --- service/tests/resources/sqlx-pool/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/service/tests/resources/sqlx-pool/src/lib.rs b/service/tests/resources/sqlx-pool/src/lib.rs index 1c6d8b5b1..a27f00422 100644 --- a/service/tests/resources/sqlx-pool/src/lib.rs +++ b/service/tests/resources/sqlx-pool/src/lib.rs @@ -1,5 +1,5 @@ use shuttle_service::error::CustomError; -use shuttle_service::{GetResource, IntoService, Logger, Runtime, ServeHandle, Service}; +use shuttle_service::{IntoService, Logger, ResourceBuilder, Runtime, ServeHandle, Service}; use sqlx::PgPool; #[macro_use] @@ -66,7 +66,9 @@ impl Service for PoolService { .await .unwrap(); - let pool = factory.get_resource(&self.runtime).await?; + let pool = shuttle_service::shared::Postgres::new() + .build(factory, &self.runtime) + .await?; self.pool = Some(pool); From 74fc059eddfea2877164f9e9db3505bc5709b218 Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 12:57:31 +0200 Subject: [PATCH 37/61] refactor: make attribute config required --- codegen/src/lib.rs | 129 +++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 80 deletions(-) diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 5cd7613a4..e5f25c1b8 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -8,7 +8,11 @@ use syn::{ pub fn main(_attr: TokenStream, item: TokenStream) -> TokenStream { let mut fn_decl = parse_macro_input!(item as ItemFn); - let wrapper = Wrapper::from_item_fn(&mut fn_decl); + let wrapper = match Wrapper::from_item_fn(&mut fn_decl) { + Ok(wrapper) => wrapper, + Err(error) => return error.into_compile_error().into(), + }; + let expanded = quote! { #wrapper @@ -42,12 +46,15 @@ struct Wrapper { #[derive(Debug, PartialEq)] struct Input { + /// The identifier for a resource input ident: Ident, - builder: Option, + + /// The shuttle_service path to the builder for this resource + builder: Path, } impl Wrapper { - fn from_item_fn(item_fn: &mut ItemFn) -> Self { + fn from_item_fn(item_fn: &mut ItemFn) -> syn::Result { let inputs: Vec<_> = item_fn .sig .inputs @@ -60,28 +67,31 @@ impl Wrapper { Pat::Ident(ident) => Some((ident, typed.attrs.drain(..).collect())), _ => None, }) - .map(|(pat_ident, attrs)| Input { - ident: pat_ident.ident.clone(), - builder: attribute_to_string(attrs), + .map(|(pat_ident, attrs)| { + Ok(Input { + ident: pat_ident.ident.clone(), + builder: attribute_to_path(attrs) + .map_err(|err| syn::Error::new_spanned(pat_ident, err))?, + }) }) - .collect(); + .collect::>>()?; - Self { + Ok(Self { fn_ident: item_fn.sig.ident.clone(), fn_output: item_fn.sig.output.clone(), fn_inputs: inputs, - } + }) } } -fn attribute_to_string(attrs: Vec) -> Option { +fn attribute_to_path(attrs: Vec) -> Result { if attrs.is_empty() { - return None; + return Err("resource needs an attribute configuration".to_string()); } let builder = attrs[0].path.clone(); - Some(builder) + Ok(builder) } impl ToTokens for Wrapper { @@ -144,7 +154,7 @@ mod tests { async fn simple() {} ); - let actual = Wrapper::from_item_fn(&mut input); + let actual = Wrapper::from_item_fn(&mut input).unwrap(); let expected_ident: Ident = parse_quote!(simple); assert_eq!(actual.fn_ident, expected_ident); @@ -186,7 +196,7 @@ mod tests { async fn complex() -> Result<(), Box> {} ); - let actual = Wrapper::from_item_fn(&mut input); + let actual = Wrapper::from_item_fn(&mut input).unwrap(); let expected_ident: Ident = parse_quote!(complex); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); @@ -226,20 +236,34 @@ mod tests { #[test] fn from_with_inputs() { let mut input = parse_quote!( - async fn complex(pool: PgPool) -> Result<(), Box> {} + async fn complex( + #[shared::Postgres] pool: PgPool, + ) -> Result<(), Box> { + } ); - let actual = Wrapper::from_item_fn(&mut input); + let actual = Wrapper::from_item_fn(&mut input).unwrap(); let expected_ident: Ident = parse_quote!(complex); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); let expected_inputs: Vec = vec![Input { ident: parse_quote!(pool), - builder: None, + builder: parse_quote!(shared::Postgres), }]; assert_eq!(actual.fn_ident, expected_ident); assert_eq!(actual.fn_output, expected_output); assert_eq!(actual.fn_inputs, expected_inputs); + + // Make sure attributes was removed from input + if let syn::FnArg::Typed(param) = input.sig.inputs.first().unwrap() { + assert!( + param.attrs.is_empty(), + "some attributes were not removed: {:?}", + param.attrs + ); + } else { + panic!("expected first input to be typed") + } } #[test] @@ -250,11 +274,11 @@ mod tests { fn_inputs: vec![ Input { ident: parse_quote!(pool), - builder: None, + builder: parse_quote!(shared::Postgres), }, Input { ident: parse_quote!(redis), - builder: None, + builder: parse_quote!(shared::Redis), }, ], }; @@ -274,8 +298,8 @@ mod tests { .expect("logger set should succeed"); }).await.unwrap(); - let pool = factory.get_resource(runtime).await?; - let redis = factory.get_resource(runtime).await?; + let pool = shuttle_service::shared::Postgres::new().build(factory, runtime).await?; + let redis = shuttle_service::shared::Redis::new().build(factory, runtime).await?; runtime.spawn(complex(pool, redis)).await.unwrap() } @@ -285,67 +309,12 @@ mod tests { } #[test] - fn from_input_attribute() { + #[should_panic(expected = "resource needs an attribute configuration")] + fn from_with_inputs_missing_attribute() { let mut input = parse_quote!( - async fn meta(#[aws::rds] pool: PgPool) -> Result<(), Box> {} + async fn complex(pool: PgPool) {} ); - let actual = Wrapper::from_item_fn(&mut input); - let expected_ident: Ident = parse_quote!(meta); - let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); - let expected_inputs: Vec = vec![Input { - ident: parse_quote!(pool), - builder: Some(parse_quote!(aws::rds)), - }]; - - assert_eq!(actual.fn_ident, expected_ident); - assert_eq!(actual.fn_output, expected_output); - assert_eq!(actual.fn_inputs, expected_inputs); - - // Make sure attributes was removed from input - if let syn::FnArg::Typed(param) = input.sig.inputs.first().unwrap() { - assert!( - param.attrs.is_empty(), - "some attributes were not removed: {:?}", - param.attrs - ); - } else { - panic!("expected first input to be typed") - } - } - - #[test] - fn output_with_input_attributes() { - let input = Wrapper { - fn_ident: parse_quote!(complex), - fn_output: parse_quote!(-> Result<(), Box>), - fn_inputs: vec![Input { - ident: parse_quote!(pool), - builder: Some(parse_quote!(aws::rds)), - }], - }; - - let actual = quote!(#input); - let expected = quote! { - async fn __shuttle_wrapper( - factory: &mut dyn shuttle_service::Factory, - runtime: &shuttle_service::Runtime, - logger: shuttle_service::Logger, - ) -> Result<(), Box > { - use shuttle_service::ResourceBuilder; - - runtime.spawn_blocking(move || { - shuttle_service::log::set_boxed_logger(Box::new(logger)) - .map(|()| shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info)) - .expect("logger set should succeed"); - }).await.unwrap(); - - let pool = shuttle_service::aws::rds::new().build(factory, runtime).await?; - - runtime.spawn(complex(pool)).await.unwrap() - } - }; - - assert_eq!(actual.to_string(), expected.to_string()); + Wrapper::from_item_fn(&mut input).unwrap(); } } From 689ed270001e5c27009e81dbbca6f961483e0e6d Mon Sep 17 00:00:00 2001 From: chesedo Date: Tue, 24 May 2022 13:06:35 +0200 Subject: [PATCH 38/61] refactor: make transition error better --- codegen/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index e5f25c1b8..677cd9511 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -86,7 +86,10 @@ impl Wrapper { fn attribute_to_path(attrs: Vec) -> Result { if attrs.is_empty() { - return Err("resource needs an attribute configuration".to_string()); + return Err( + "resource needs an attribute configuration\nTry adding `#[shared::Postgres]`" + .to_string(), + ); } let builder = attrs[0].path.clone(); From c8dc739dde0e66f81bfa1e073fb8456001cecbf8 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 30 May 2022 11:54:55 +0200 Subject: [PATCH 39/61] tests: trybuild --- Cargo.lock | 16 ++++++++++++++++ codegen/Cargo.toml | 3 ++- codegen/src/lib.rs | 6 ++++++ codegen/tests/ui/missing-attribute.rs | 2 ++ codegen/tests/ui/missing-attribute.stderr | 12 ++++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 codegen/tests/ui/missing-attribute.rs create mode 100644 codegen/tests/ui/missing-attribute.stderr diff --git a/Cargo.lock b/Cargo.lock index ca20de704..b7b61e06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3947,6 +3947,7 @@ dependencies = [ "proc-macro2", "quote", "syn", + "trybuild", ] [[package]] @@ -4749,6 +4750,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "trybuild" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc92f558afb6d1d7c6f175eb8d615b8ef49c227543e68e19c123d4ee43d8a7d" +dependencies = [ + "glob", + "once_cell", + "serde", + "serde_derive", + "serde_json", + "termcolor", + "toml", +] + [[package]] name = "typenum" version = "1.15.0" diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 7858b1609..e94129be3 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -13,7 +13,8 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.36" quote = "1.0.17" -syn = { version = "1.0.90", features = ["full"] } +syn = { version = "1.0.90", features = ["full", "extra-traits"] } [dev-dependencies] pretty_assertions = "1.2" +trybuild = "1.0" diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 677cd9511..3ed5d9054 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -320,4 +320,10 @@ mod tests { Wrapper::from_item_fn(&mut input).unwrap(); } + + #[test] + fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/*.rs"); + } } diff --git a/codegen/tests/ui/missing-attribute.rs b/codegen/tests/ui/missing-attribute.rs new file mode 100644 index 000000000..243578628 --- /dev/null +++ b/codegen/tests/ui/missing-attribute.rs @@ -0,0 +1,2 @@ +#[shuttle_codegen::main] +async fn missing_attriute(pool: PgPool) {} diff --git a/codegen/tests/ui/missing-attribute.stderr b/codegen/tests/ui/missing-attribute.stderr new file mode 100644 index 000000000..cf178d851 --- /dev/null +++ b/codegen/tests/ui/missing-attribute.stderr @@ -0,0 +1,12 @@ +error: resource needs an attribute configuration + Try adding `#[shared::Postgres]` + --> tests/ui/missing-attribute.rs:2:27 + | +2 | async fn missing_attriute(pool: PgPool) {} + | ^^^^ + +error[E0601]: `main` function not found in crate `$CRATE` + --> tests/ui/missing-attribute.rs:2:43 + | +2 | async fn missing_attriute(pool: PgPool) {} + | ^ consider adding a `main` function to `$DIR/tests/ui/missing-attribute.rs` From 8506b0b19c89c8523db8bc746df78bd9eafd08b7 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 30 May 2022 14:59:14 +0200 Subject: [PATCH 40/61] refactor: better hints --- Cargo.lock | 1 + codegen/Cargo.toml | 1 + codegen/src/lib.rs | 56 ++++++++++------------- codegen/tests/ui/missing-attribute.rs | 2 +- codegen/tests/ui/missing-attribute.stderr | 21 +++++++-- 5 files changed, 43 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7b61e06d..9595e5558 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3944,6 +3944,7 @@ name = "shuttle-codegen" version = "0.3.0" dependencies = [ "pretty_assertions", + "proc-macro-error", "proc-macro2", "quote", "syn", diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index e94129be3..65bc985fa 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -12,6 +12,7 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.36" +proc-macro-error = "1.0" quote = "1.0.17" syn = { version = "1.0.90", features = ["full", "extra-traits"] } diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 3ed5d9054..2ce96bcba 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -1,17 +1,17 @@ use proc_macro::TokenStream; +use proc_macro_error::{emit_error, proc_macro_error}; use quote::{quote, ToTokens}; use syn::{ - parse_macro_input, parse_quote, Attribute, FnArg, Ident, ItemFn, Pat, Path, ReturnType, Stmt, + parse_macro_input, parse_quote, spanned::Spanned, Attribute, FnArg, Ident, ItemFn, Pat, Path, + ReturnType, Stmt, }; +#[proc_macro_error] #[proc_macro_attribute] pub fn main(_attr: TokenStream, item: TokenStream) -> TokenStream { let mut fn_decl = parse_macro_input!(item as ItemFn); - let wrapper = match Wrapper::from_item_fn(&mut fn_decl) { - Ok(wrapper) => wrapper, - Err(error) => return error.into_compile_error().into(), - }; + let wrapper = Wrapper::from_item_fn(&mut fn_decl); let expanded = quote! { #wrapper @@ -54,7 +54,7 @@ struct Input { } impl Wrapper { - fn from_item_fn(item_fn: &mut ItemFn) -> syn::Result { + fn from_item_fn(item_fn: &mut ItemFn) -> Self { let inputs: Vec<_> = item_fn .sig .inputs @@ -67,29 +67,31 @@ impl Wrapper { Pat::Ident(ident) => Some((ident, typed.attrs.drain(..).collect())), _ => None, }) - .map(|(pat_ident, attrs)| { - Ok(Input { - ident: pat_ident.ident.clone(), - builder: attribute_to_path(attrs) - .map_err(|err| syn::Error::new_spanned(pat_ident, err))?, - }) + .filter_map(|(pat_ident, attrs)| { + match attribute_to_path(attrs) { + Ok(builder) => Some(Input { + ident: pat_ident.ident.clone(), + builder, + }), + Err(err) => { + emit_error!(pat_ident, err; hint = pat_ident.span() => "Try adding a config like `#[shared::Postgres]`"); + None + } + } }) - .collect::>>()?; + .collect(); - Ok(Self { + Self { fn_ident: item_fn.sig.ident.clone(), fn_output: item_fn.sig.output.clone(), fn_inputs: inputs, - }) + } } } fn attribute_to_path(attrs: Vec) -> Result { if attrs.is_empty() { - return Err( - "resource needs an attribute configuration\nTry adding `#[shared::Postgres]`" - .to_string(), - ); + return Err("resource needs an attribute configuration".to_string()); } let builder = attrs[0].path.clone(); @@ -157,7 +159,7 @@ mod tests { async fn simple() {} ); - let actual = Wrapper::from_item_fn(&mut input).unwrap(); + let actual = Wrapper::from_item_fn(&mut input); let expected_ident: Ident = parse_quote!(simple); assert_eq!(actual.fn_ident, expected_ident); @@ -199,7 +201,7 @@ mod tests { async fn complex() -> Result<(), Box> {} ); - let actual = Wrapper::from_item_fn(&mut input).unwrap(); + let actual = Wrapper::from_item_fn(&mut input); let expected_ident: Ident = parse_quote!(complex); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); @@ -245,7 +247,7 @@ mod tests { } ); - let actual = Wrapper::from_item_fn(&mut input).unwrap(); + let actual = Wrapper::from_item_fn(&mut input); let expected_ident: Ident = parse_quote!(complex); let expected_output: ReturnType = parse_quote!(-> Result<(), Box>); let expected_inputs: Vec = vec![Input { @@ -311,16 +313,6 @@ mod tests { assert_eq!(actual.to_string(), expected.to_string()); } - #[test] - #[should_panic(expected = "resource needs an attribute configuration")] - fn from_with_inputs_missing_attribute() { - let mut input = parse_quote!( - async fn complex(pool: PgPool) {} - ); - - Wrapper::from_item_fn(&mut input).unwrap(); - } - #[test] fn ui() { let t = trybuild::TestCases::new(); diff --git a/codegen/tests/ui/missing-attribute.rs b/codegen/tests/ui/missing-attribute.rs index 243578628..54ce54049 100644 --- a/codegen/tests/ui/missing-attribute.rs +++ b/codegen/tests/ui/missing-attribute.rs @@ -1,2 +1,2 @@ #[shuttle_codegen::main] -async fn missing_attriute(pool: PgPool) {} +async fn missing_attriute(pool: PgPool, cache: Redis) {} diff --git a/codegen/tests/ui/missing-attribute.stderr b/codegen/tests/ui/missing-attribute.stderr index cf178d851..fafa2434b 100644 --- a/codegen/tests/ui/missing-attribute.stderr +++ b/codegen/tests/ui/missing-attribute.stderr @@ -1,12 +1,23 @@ error: resource needs an attribute configuration - Try adding `#[shared::Postgres]` + + = help: Try adding a config like `#[shared::Postgres]` + --> tests/ui/missing-attribute.rs:2:27 | -2 | async fn missing_attriute(pool: PgPool) {} +2 | async fn missing_attriute(pool: PgPool, cache: Redis) {} | ^^^^ +error: resource needs an attribute configuration + + = help: Try adding a config like `#[shared::Postgres]` + + --> tests/ui/missing-attribute.rs:2:41 + | +2 | async fn missing_attriute(pool: PgPool, cache: Redis) {} + | ^^^^^ + error[E0601]: `main` function not found in crate `$CRATE` - --> tests/ui/missing-attribute.rs:2:43 + --> tests/ui/missing-attribute.rs:2:57 | -2 | async fn missing_attriute(pool: PgPool) {} - | ^ consider adding a `main` function to `$DIR/tests/ui/missing-attribute.rs` +2 | async fn missing_attriute(pool: PgPool, cache: Redis) {} + | ^ consider adding a `main` function to `$DIR/tests/ui/missing-attribute.rs` From ad00744719d272cdecb4041d7d13966d266db106 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 30 May 2022 15:27:34 +0200 Subject: [PATCH 41/61] refactor: builders vec --- codegen/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 2ce96bcba..117f587a6 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -104,7 +104,7 @@ impl ToTokens for Wrapper { let fn_output = &self.fn_output; let fn_ident = &self.fn_ident; let fn_inputs: Vec<_> = self.fn_inputs.iter().map(|i| i.ident.clone()).collect(); - let fn_inputs_attrs: Vec<_> = self.fn_inputs.iter().map(|i| i.builder.clone()).collect(); + let fn_inputs_builder: Vec<_> = self.fn_inputs.iter().map(|i| i.builder.clone()).collect(); let factory_ident: Ident = if self.fn_inputs.is_empty() { parse_quote!(_factory) @@ -135,7 +135,7 @@ impl ToTokens for Wrapper { }).await.unwrap(); - #(let #fn_inputs = shuttle_service::#fn_inputs_attrs::new().build(#factory_ident, runtime).await?;)* + #(let #fn_inputs = shuttle_service::#fn_inputs_builder::new().build(#factory_ident, runtime).await?;)* runtime.spawn(#fn_ident(#(#fn_inputs),*)).await.unwrap() } From b95ac5a6b89ac8cd3d24a96da559ed11748d6d14 Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 30 May 2022 15:28:05 +0200 Subject: [PATCH 42/61] refactor: update examples to main --- examples/tide/postgres/Cargo.toml | 2 +- examples/tide/postgres/src/lib.rs | 6 ++---- examples/url-shortener/src/lib.rs | 6 +++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/tide/postgres/Cargo.toml b/examples/tide/postgres/Cargo.toml index cf5d07b19..8e4929c27 100644 --- a/examples/tide/postgres/Cargo.toml +++ b/examples/tide/postgres/Cargo.toml @@ -12,4 +12,4 @@ crate-type = ["cdylib"] tide = "0.16.0" serde = { version = "1.0", features = ["derive"] } sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } -shuttle-service = { version = "0.2", features = ["sqlx-aws-postgres", "web-tide"] } +shuttle-service = { version = "0.3", features = ["sqlx-aws-postgres", "web-tide"] } diff --git a/examples/tide/postgres/src/lib.rs b/examples/tide/postgres/src/lib.rs index 88c0beef6..3bd88bcf5 100644 --- a/examples/tide/postgres/src/lib.rs +++ b/examples/tide/postgres/src/lib.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use shuttle_service::error::CustomError; +use shuttle_service::{error::CustomError, ShuttleTide}; use sqlx::{Executor, FromRow, PgPool}; use tide::{Body, Request}; @@ -29,9 +29,7 @@ struct MyState { } #[shuttle_service::main] -async fn tide( - #[aws::rds::Postgres] pool: PgPool, -) -> Result, shuttle_service::Error> { +async fn tide(#[aws::rds::Postgres] pool: PgPool) -> ShuttleTide { pool.execute(include_str!("../schema.sql")) .await .map_err(CustomError::new)?; diff --git a/examples/url-shortener/src/lib.rs b/examples/url-shortener/src/lib.rs index a2a3d5537..c38d3f93a 100644 --- a/examples/url-shortener/src/lib.rs +++ b/examples/url-shortener/src/lib.rs @@ -4,10 +4,10 @@ extern crate rocket; use rocket::{ http::Status, response::{status, Redirect}, - routes, Build, Rocket, State, + routes, State, }; use serde::Serialize; -use shuttle_service::error::CustomError; +use shuttle_service::{error::CustomError, ShuttleRocket}; use sqlx::migrate::Migrator; use sqlx::{FromRow, PgPool}; use url::Url; @@ -71,7 +71,7 @@ async fn shorten(url: String, state: &State) -> Result Result, shuttle_service::Error> { +async fn rocket(#[shared::Postgres] pool: PgPool) -> ShuttleRocket { MIGRATOR.run(&pool).await.map_err(CustomError::new)?; let state = AppState { pool }; From c592963c206733f1d667db30bd4c527df319a05f Mon Sep 17 00:00:00 2001 From: chesedo Date: Mon, 30 May 2022 15:41:36 +0200 Subject: [PATCH 43/61] refactor: make tf use account id --- terraform/modules/shuttle/locals.tf | 3 +++ terraform/modules/shuttle/service.tf | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/terraform/modules/shuttle/locals.tf b/terraform/modules/shuttle/locals.tf index a36d29e4f..c6922bf59 100644 --- a/terraform/modules/shuttle/locals.tf +++ b/terraform/modules/shuttle/locals.tf @@ -1,4 +1,7 @@ +data "aws_caller_identity" "current" {} + locals { + account_id = data.aws_caller_identity.current.account_id data_dir = "/opt/shuttle" docker_image = "public.ecr.aws/shuttle/backend" } diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index e7b6a72d3..bb63c2b8f 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -51,8 +51,8 @@ EOF ] Effect = "Allow" Resource = [ - "arn:aws:rds:*:833239102462:db:*", - "arn:aws:rds:*:833239102462:subgrp:shuttle_rds", + "arn:aws:rds:*:${local.account_id}:db:*", + "arn:aws:rds:*:${local.account_id}:subgrp:shuttle_rds", ] }, ] From 556a455a770f7409dda109f51c25c0163a9e0be6 Mon Sep 17 00:00:00 2001 From: chesedo Date: Wed, 15 Jun 2022 14:45:26 +0200 Subject: [PATCH 44/61] feat: add AWS RDS to proto --- Cargo.lock | 194 ++++++++++++++++++++++++++--------- api/src/deployment.rs | 5 - api/src/factory.rs | 16 +-- api/src/main.rs | 3 +- cargo-shuttle/src/factory.rs | 12 ++- common/src/database.rs | 10 ++ common/src/lib.rs | 1 + proto/provisioner.proto | 20 ++++ proto/src/lib.rs | 41 +++++++- provisioner/src/error.rs | 13 +++ provisioner/src/lib.rs | 162 +++++++++++++++++++++++++++-- provisioner/src/main.rs | 2 +- service/src/database.rs | 22 ---- service/src/lib.rs | 4 +- 14 files changed, 408 insertions(+), 97 deletions(-) create mode 100644 common/src/database.rs delete mode 100644 service/src/database.rs diff --git a/Cargo.lock b/Cargo.lock index 5a9769225..1c8866df2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,9 +139,6 @@ dependencies = [ "anyhow", "async-mutex", "async-trait", - "aws-config", - "aws-sdk-rds", - "aws-smithy-types", "base64 0.13.0", "cargo", "cargo-util", @@ -888,14 +885,14 @@ dependencies = [ "hyper", "hyperlocal", "log", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "serde", "serde_derive", "serde_json", "serde_urlencoded", "thiserror", "tokio", - "tokio-util 0.7.1", + "tokio-util 0.7.2", "url", "winapi", ] @@ -1556,6 +1553,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "crypto-bigint" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.3" @@ -1598,6 +1605,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct 0.6.1", +] + [[package]] name = "ctor" version = "0.1.22" @@ -2490,18 +2506,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite 0.2.8", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "hyper-rustls" version = "0.22.1" @@ -2512,11 +2516,23 @@ dependencies = [ "futures-util", "hyper", "log", - "rustls", + "rustls 0.19.1", "rustls-native-certs", "tokio", "tokio-rustls", - "webpki", + "webpki 0.21.4", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite 0.2.9", + "tokio", + "tokio-io-timeout", ] [[package]] @@ -3034,8 +3050,36 @@ dependencies = [ ] [[package]] -name = "ntapi" -version = "0.3.7" +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566d173b2f9406afbc5510a90925d5a2cd80cae4605631f1212303df265de011" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ @@ -3581,6 +3625,9 @@ dependencies = [ name = "provisioner" version = "0.1.0" dependencies = [ + "aws-config", + "aws-sdk-rds", + "aws-smithy-types", "clap 3.1.18", "ctor", "lazy_static", @@ -3980,6 +4027,26 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56770675ebc04927ded3e60633437841581c285dc6236109ea25fbf3beb7b59e" +[[package]] +name = "rsa" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf22754c49613d2b3b119f0e5d46e34a2c628a937e3024b8762de4e7d8c710b" +dependencies = [ + "byteorder", + "digest 0.10.3", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.3", + "smallvec", + "subtle", + "zeroize", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -4022,6 +4089,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct 0.6.1", + "webpki 0.21.4", +] + [[package]] name = "rustls" version = "0.20.6" @@ -4030,8 +4110,20 @@ checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls 0.19.1", + "schannel", + "security-framework", ] [[package]] @@ -4077,6 +4169,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sct" version = "0.7.0" @@ -4450,12 +4552,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.3" @@ -5007,7 +5103,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tokio", ] @@ -5038,9 +5134,9 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "rustls", + "rustls 0.19.1", "tokio", - "webpki", + "webpki 0.21.4", ] [[package]] @@ -5169,7 +5265,7 @@ dependencies = [ "prost-derive", "tokio", "tokio-stream", - "tokio-util 0.7.1", + "tokio-util 0.7.2", "tower", "tower-layer", "tower-service", @@ -5201,6 +5297,8 @@ dependencies = [ "indexmap", "pin-project", "pin-project-lite 0.2.9", + "rand 0.8.5", + "slab", "tokio", "tokio-util 0.7.2", "tower-layer", @@ -5210,9 +5308,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" +checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" dependencies = [ "bitflags", "bytes", @@ -5221,7 +5319,7 @@ dependencies = [ "http", "http-body", "http-range-header", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tower-layer", "tower-service", ] @@ -5239,7 +5337,7 @@ dependencies = [ "http", "http-body", "http-range-header", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tower", "tower-layer", "tower-service", @@ -5397,12 +5495,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" -[[package]] -name = "unicode-ident" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" - [[package]] name = "unicode-normalization" version = "0.1.19" @@ -5463,12 +5555,12 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls", + "rustls 0.20.6", "serde", "serde_json", "socks", "url", - "webpki", + "webpki 0.22.0", "webpki-roots", ] @@ -5703,6 +5795,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki" version = "0.22.0" @@ -5719,7 +5821,7 @@ version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" dependencies = [ - "webpki", + "webpki 0.22.0", ] [[package]] diff --git a/api/src/deployment.rs b/api/src/deployment.rs index 28977ab14..7854ae39a 100644 --- a/api/src/deployment.rs +++ b/api/src/deployment.rs @@ -160,7 +160,6 @@ impl Deployment { let mut factory = ShuttleFactory::new( context.provisioner_client.clone(), - context.provisioner_address.clone(), meta.project.clone(), ); let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port); @@ -289,7 +288,6 @@ pub(crate) struct DeploymentSystem { job_queue: JobQueue, router: Arc, fqdn: String, - pub(crate) provisioner_address: String, } const JOB_QUEUE_SIZE: usize = 200; @@ -340,7 +338,6 @@ pub(crate) struct Context { build_system: Box, deployments: Arc>, provisioner_client: ProvisionerClient, - provisioner_address: String, } impl DeploymentSystem { @@ -384,7 +381,6 @@ impl DeploymentSystem { build_system, deployments: deployments.clone(), provisioner_client, - provisioner_address: provisioner_address.clone(), }; let job_queue = JobQueue::new(context, tx).await; @@ -400,7 +396,6 @@ impl DeploymentSystem { job_queue, router, fqdn, - provisioner_address, } } diff --git a/api/src/factory.rs b/api/src/factory.rs index f76e33f2e..3a84683c0 100644 --- a/api/src/factory.rs +++ b/api/src/factory.rs @@ -1,25 +1,24 @@ use async_trait::async_trait; -use proto::provisioner::{provisioner_client::ProvisionerClient, DatabaseRequest}; +use proto::provisioner::{ + database_request::DbType, provisioner_client::ProvisionerClient, DatabaseRequest, +}; use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; -use shuttle_service::Factory; +use shuttle_service::{database::Type, Factory}; use tonic::{transport::Channel, Request}; pub(crate) struct ShuttleFactory { project_name: ProjectName, provisioner_client: ProvisionerClient, - provisioner_address: String, info: Option, } impl ShuttleFactory { pub(crate) fn new( provisioner_client: ProvisionerClient, - provisioner_address: String, project_name: ProjectName, ) -> Self { Self { provisioner_client, - provisioner_address, project_name, info: None, } @@ -37,11 +36,14 @@ impl Factory for ShuttleFactory { db_type: Type, ) -> Result { if let Some(ref info) = self.info { - return Ok(info.connection_string(&self.provisioner_address)); + return Ok(info.connection_string_private()); } + let db_type: DbType = db_type.into(); + let request = Request::new(DatabaseRequest { project_name: self.project_name.to_string(), + db_type: Some(db_type), }); let response = self @@ -52,7 +54,7 @@ impl Factory for ShuttleFactory { .into_inner(); let info: DatabaseReadyInfo = response.into(); - let conn_str = info.connection_string(&self.provisioner_address); + let conn_str = info.connection_string_private(); self.info = Some(info); debug!("giving a sql connection string: {}", conn_str); diff --git a/api/src/main.rs b/api/src/main.rs index f14170793..a93c99342 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -148,8 +148,7 @@ async fn project_secrets( .await?; if let Some(database_deployment) = &deployment.database_deployment { - let conn_str = - database_deployment.connection_string(&state.deployment_manager.provisioner_address); + let conn_str = database_deployment.connection_string_private(); let conn = sqlx::PgPool::connect(&conn_str) .await .map_err(|e| DeploymentApiError::Internal(e.to_string()))?; diff --git a/cargo-shuttle/src/factory.rs b/cargo-shuttle/src/factory.rs index bfa3fb043..ca94e03da 100644 --- a/cargo-shuttle/src/factory.rs +++ b/cargo-shuttle/src/factory.rs @@ -16,7 +16,7 @@ use crossterm::{ use futures::StreamExt; use portpicker::pick_unused_port; use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; -use shuttle_service::{error::CustomError, Factory}; +use shuttle_service::{database::Type, error::CustomError, Factory}; use std::{collections::HashMap, io::stdout, time::Duration}; use tokio::time::sleep; @@ -39,7 +39,10 @@ const PG_IMAGE: &str = "postgres:11"; #[async_trait] impl Factory for LocalFactory { - async fn get_sql_connection_string(&mut self) -> Result { + async fn get_sql_connection_string( + &mut self, + db_type: Type, + ) -> Result { trace!("getting sql string for project '{}'", self.project); let container_name = format!("shuttle_{}_postgres", self.project); @@ -128,13 +131,16 @@ impl Factory for LocalFactory { self.wait_for_ready(&container_name).await?; let db_info = DatabaseReadyInfo::new( + "postgres".to_string(), "postgres".to_string(), PG_PASSWORD.to_string(), "postgres".to_string(), port, + "localhost".to_string(), + "localhost".to_string(), ); - let conn_str = db_info.connection_string("localhost"); + let conn_str = db_info.connection_string_private(); println!( "{:>12} can be reached at {}\n", diff --git a/common/src/database.rs b/common/src/database.rs new file mode 100644 index 000000000..6618b8ae8 --- /dev/null +++ b/common/src/database.rs @@ -0,0 +1,10 @@ +pub enum Type { + AwsRds(AwsRdsEngine), + Shared, +} + +pub enum AwsRdsEngine { + Postgres, + MySql, + MariaDB, +} diff --git a/common/src/lib.rs b/common/src/lib.rs index bc2e88c87..ebbdeb50e 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,3 +1,4 @@ +pub mod database; pub mod project; use std::{ diff --git a/proto/provisioner.proto b/proto/provisioner.proto index f93705325..fe0dd704b 100644 --- a/proto/provisioner.proto +++ b/proto/provisioner.proto @@ -7,10 +7,30 @@ service Provisioner { message DatabaseRequest { string project_name = 1; + oneof db_type { + string shared = 10; + AwsRds AwsRds = 11; + }; +} + +message AwsRds { + oneof engine { + RdsConfig postgres = 1; + RdsConfig mysql = 2; + RdsConfig mariadb = 3; + } +} + +message RdsConfig { + } message DatabaseResponse { string username = 1; string password = 2; string database_name = 3; + string engine = 4; + string address_private = 5; + string address_public = 6; + string port = 7; } diff --git a/proto/src/lib.rs b/proto/src/lib.rs index cb2bac417..cc1d3919e 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -1,16 +1,53 @@ pub mod provisioner { - use shuttle_common::DatabaseReadyInfo; + use std::fmt::Display; + + use shuttle_common::{ + database::{self, AwsRdsEngine}, + DatabaseReadyInfo, + }; tonic::include_proto!("provisioner"); impl From for DatabaseReadyInfo { fn from(response: DatabaseResponse) -> Self { DatabaseReadyInfo::new( + response.engine, response.username, response.password, response.database_name, - "5432".to_string(), + response.port, + response.address_private, + response.address_public, ) } } + + impl From for database_request::DbType { + fn from(db_type: database::Type) -> Self { + match db_type { + database::Type::Shared => database_request::DbType::Shared(String::new()), + database::Type::AwsRds(engine) => { + let config = RdsConfig {}; + let engine = match engine { + AwsRdsEngine::Postgres => aws_rds::Engine::Postgres(config), + AwsRdsEngine::MariaDB => aws_rds::Engine::Mariadb(config), + AwsRdsEngine::MySql => aws_rds::Engine::Mysql(config), + }; + database_request::DbType::AwsRds(AwsRds { + engine: Some(engine), + }) + } + } + } + } + + impl Display for aws_rds::Engine { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Mariadb(_) => write!(f, "mariadb"), + Self::Mysql(_) => write!(f, "mysql"), + Self::Postgres(_) => write!(f, "postgres"), + } + } + } } diff --git a/provisioner/src/error.rs b/provisioner/src/error.rs index 6e91436d7..7ebfb34a9 100644 --- a/provisioner/src/error.rs +++ b/provisioner/src/error.rs @@ -1,3 +1,7 @@ +use aws_sdk_rds::{ + error::{CreateDBInstanceError, DescribeDBInstancesError}, + types::SdkError, +}; use thiserror::Error; use tonic::Status; use tracing::error; @@ -15,6 +19,15 @@ pub enum Error { #[error("unexpected error")] Unexpected(#[from] sqlx::Error), + + #[error("failed to create RDS instance")] + CreateRDSInstance(#[from] SdkError), + + #[error("failed to get description of RDS instance")] + DescribeRDSInstance(#[from] SdkError), + + #[error["plain error"]] + Plain(String), } unsafe impl Send for Error {} diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs index 687f5ee87..b300de929 100644 --- a/provisioner/src/lib.rs +++ b/provisioner/src/lib.rs @@ -1,25 +1,38 @@ use std::time::Duration; pub use args::Args; +use aws_config::timeout; +use aws_sdk_rds::{error::ModifyDBInstanceErrorKind, model::DbInstance, types::SdkError, Client}; +use aws_smithy_types::tristate::TriState; pub use error::Error; use proto::provisioner::provisioner_server::Provisioner; pub use proto::provisioner::provisioner_server::ProvisionerServer; -use proto::provisioner::{DatabaseRequest, DatabaseResponse}; +use proto::provisioner::{ + aws_rds, database_request::DbType, AwsRds, DatabaseRequest, DatabaseResponse, +}; use rand::Rng; use sqlx::{postgres::PgPoolOptions, PgPool}; +use tokio::time::sleep; use tonic::{Request, Response, Status}; -use tracing::info; +use tracing::{debug, info}; mod args; mod error; +const PRIVATE_PG_IP: &str = "provisioner"; +const PUBLIC_PG_IP: &str = "pg.shuttle.rs"; + +const AWS_RDS_CLASS: &str = "db.t4g.micro"; +const MASTER_USERNAME: &str = "master"; +const RDS_SUBNET_GROUP: &str = "shuttle_rds"; + pub struct MyProvisioner { pool: PgPool, rds_client: aws_sdk_rds::Client, } impl MyProvisioner { - pub fn new(uri: &str) -> sqlx::Result { + pub async fn new(uri: &str) -> sqlx::Result { let pool = PgPoolOptions::new() .min_connections(4) .max_connections(12) @@ -42,14 +55,18 @@ impl MyProvisioner { Ok(Self { pool, rds_client }) } - pub async fn request_shared_db(&self, project_name: &str) -> Result { + async fn request_shared_db(&self, project_name: &str) -> Result { let (username, password) = self.shared_role(project_name).await?; let database_name = self.shared_db(project_name, &username).await?; Ok(DatabaseResponse { + engine: "postgres".to_string(), username, password, database_name, + address_private: PRIVATE_PG_IP.to_string(), + address_public: PUBLIC_PG_IP.to_string(), + port: "3306".to_string(), }) } @@ -111,6 +128,90 @@ impl MyProvisioner { Ok(database_name) } + + async fn request_aws_rds( + &self, + project_name: &str, + engine: aws_rds::Engine, + ) -> Result { + let client = &self.rds_client; + + let password = generate_password(); + let instance_name = format!("{}-{}", project_name, engine); + + debug!("trying to get AWS RDS instance: {instance_name}"); + let instance = client + .modify_db_instance() + .db_instance_identifier(&instance_name) + .master_user_password(&password) + .send() + .await; + + match instance { + Ok(_) => { + wait_for_instance(client, &instance_name, "resetting-master-credentials").await?; + } + Err(SdkError::ServiceError { err, .. }) => { + if let ModifyDBInstanceErrorKind::DbInstanceNotFoundFault(_) = err.kind { + debug!("creating new AWS RDS {instance_name}"); + + client + .create_db_instance() + .db_instance_identifier(&instance_name) + .master_username(MASTER_USERNAME) + .master_user_password(&password) + .engine(engine.to_string()) + .db_instance_class(AWS_RDS_CLASS) + .allocated_storage(20) + .backup_retention_period(0) // Disable backups + .publicly_accessible(true) + .db_name(engine.to_string()) + .set_db_subnet_group_name(Some(RDS_SUBNET_GROUP.to_string())) + .send() + .await? + .db_instance + .expect("to be able to create instance"); + + wait_for_instance(client, &instance_name, "creating").await?; + } else { + return Err(Error::Plain(format!( + "got unexpected error from AWS RDS service: {}", + err + ))); + } + } + Err(unexpected) => { + return Err(Error::Plain(format!( + "got unexpected error from AWS during API call: {}", + unexpected + ))) + } + }; + + // Wait for up + let instance = wait_for_instance(client, &instance_name, "available").await?; + + // TODO: find private IP somehow + let address = instance + .endpoint + .expect("instance to have an endpoint") + .address + .expect("endpoint to have an address"); + + Ok(DatabaseResponse { + engine: engine.to_string(), + username: instance + .master_username + .expect("instance to have a username"), + password, + database_name: instance + .db_name + .expect("instance to have a default database"), + address_private: address.clone(), + address_public: address, + port: engine_to_port(engine), + }) + } } #[tonic::async_trait] @@ -120,9 +221,16 @@ impl Provisioner for MyProvisioner { &self, request: Request, ) -> Result, Status> { - let reply = self - .request_shared_db(&request.into_inner().project_name) - .await?; + let request = request.into_inner(); + let db_type = request.db_type.unwrap(); + + let reply = match db_type { + DbType::Shared(_) => self.request_shared_db(&request.project_name).await?, + DbType::AwsRds(AwsRds { engine }) => { + self.request_aws_rds(&request.project_name, engine.expect("oneof to be set")) + .await? + } + }; Ok(Response::new(reply)) } @@ -135,3 +243,43 @@ fn generate_password() -> String { .map(char::from) .collect() } + +async fn wait_for_instance( + client: &Client, + name: &str, + wait_for: &str, +) -> Result { + debug!("waiting for {name} to enter {wait_for} state"); + loop { + let instance = client + .describe_db_instances() + .db_instance_identifier(name) + .send() + .await? + .db_instances + .expect("aws to return instances") + .get(0) + .expect("to find the instance just created or modified") + .clone(); + + let status = instance + .db_instance_status + .as_ref() + .expect("instance to have a status") + .clone(); + + if status == wait_for { + return Ok(instance); + } + + sleep(Duration::from_secs(1)).await; + } +} + +fn engine_to_port(engine: aws_rds::Engine) -> String { + match engine { + aws_rds::Engine::Postgres(_) => "5432".to_string(), + aws_rds::Engine::Mariadb(_) => "3306".to_string(), + aws_rds::Engine::Mysql(_) => "3306".to_string(), + } +} diff --git a/provisioner/src/main.rs b/provisioner/src/main.rs index 37a196adb..cc538225f 100644 --- a/provisioner/src/main.rs +++ b/provisioner/src/main.rs @@ -15,7 +15,7 @@ async fn main() -> Result<(), Box> { } = Args::parse(); let addr = SocketAddr::new(ip, port); - let provisioner = MyProvisioner::new(&shared_pg_uri).unwrap(); + let provisioner = MyProvisioner::new(&shared_pg_uri).await.unwrap(); println!("starting provisioner on {}", addr); Server::builder() diff --git a/service/src/database.rs b/service/src/database.rs deleted file mode 100644 index b128b2e60..000000000 --- a/service/src/database.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::fmt::Display; - -pub enum Type { - AwsRds(AwsRdsEngine), - Shared, -} - -pub enum AwsRdsEngine { - Postgres, - MySql, - MariaDB, -} - -impl Display for AwsRdsEngine { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::MariaDB => write!(f, "mariadb"), - Self::MySql => write!(f, "mysql"), - Self::Postgres => write!(f, "postgres"), - } - } -} diff --git a/service/src/lib.rs b/service/src/lib.rs index 685e2d36d..681791b00 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -222,13 +222,13 @@ use async_trait::async_trait; pub use log; pub use tokio::runtime::Runtime; -pub mod database; - pub mod error; pub use error::Error; pub mod logger; +pub use shuttle_common::database; + #[cfg(feature = "sqlx-postgres")] pub mod shared; From 5ff34836ec4fe3adced08b4cabfd9eb59977abfb Mon Sep 17 00:00:00 2001 From: chesedo Date: Wed, 15 Jun 2022 17:46:27 +0200 Subject: [PATCH 45/61] feat: local runs for AWS RDS --- cargo-shuttle/src/factory.rs | 135 ++++++++++++++++++++++++++++------- 1 file changed, 109 insertions(+), 26 deletions(-) diff --git a/cargo-shuttle/src/factory.rs b/cargo-shuttle/src/factory.rs index ca94e03da..3d415b1c7 100644 --- a/cargo-shuttle/src/factory.rs +++ b/cargo-shuttle/src/factory.rs @@ -15,7 +15,7 @@ use crossterm::{ }; use futures::StreamExt; use portpicker::pick_unused_port; -use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; +use shuttle_common::{database::AwsRdsEngine, project::ProjectName, DatabaseReadyInfo}; use shuttle_service::{database::Type, error::CustomError, Factory}; use std::{collections::HashMap, io::stdout, time::Duration}; use tokio::time::sleep; @@ -34,9 +34,6 @@ impl LocalFactory { } } -const PG_PASSWORD: &str = "password"; -const PG_IMAGE: &str = "postgres:11"; - #[async_trait] impl Factory for LocalFactory { async fn get_sql_connection_string( @@ -44,7 +41,19 @@ impl Factory for LocalFactory { db_type: Type, ) -> Result { trace!("getting sql string for project '{}'", self.project); - let container_name = format!("shuttle_{}_postgres", self.project); + + let EngineConfig { + r#type, + image, + engine, + username, + password, + database_name, + port, + env, + is_ready_cmd, + } = db_type_to_config(db_type); + let container_name = format!("shuttle_{}_{}", self.project, r#type); let container = match self.docker.inspect_container(&container_name, None).await { Ok(container) => { @@ -54,9 +63,7 @@ impl Factory for LocalFactory { Err(bollard::errors::Error::DockerResponseServerError { status_code, .. }) if status_code == 404 => { - self.pull_image(PG_IMAGE) - .await - .expect("failed to pull image"); + self.pull_image(&image).await.expect("failed to pull image"); trace!("will create DB container {container_name}"); let options = Some(CreateContainerOptions { name: container_name.clone(), @@ -64,7 +71,7 @@ impl Factory for LocalFactory { let mut port_bindings = HashMap::new(); let host_port = pick_unused_port().expect("system to have a free port"); port_bindings.insert( - "5432/tcp".to_string(), + port.clone(), Some(vec![PortBinding { host_port: Some(host_port.to_string()), ..Default::default() @@ -75,10 +82,9 @@ impl Factory for LocalFactory { ..Default::default() }; - let password_env = format!("POSTGRES_PASSWORD={PG_PASSWORD}"); let config = Config { - image: Some(PG_IMAGE), - env: Some(vec![&password_env]), + image: Some(image), + env, host_config: Some(host_config), ..Default::default() }; @@ -104,10 +110,10 @@ impl Factory for LocalFactory { .expect("container to have host config") .port_bindings .expect("port bindings on container") - .get("5432/tcp") - .expect("a '5432/tcp' port bindings entry") + .get(&port) + .expect("a port bindings entry") .as_ref() - .expect("a '5432/tcp' port bindings") + .expect("a port bindings") .first() .expect("at least one port binding") .host_port @@ -128,13 +134,13 @@ impl Factory for LocalFactory { .expect("failed to start none running container"); } - self.wait_for_ready(&container_name).await?; + self.wait_for_ready(&container_name, is_ready_cmd).await?; let db_info = DatabaseReadyInfo::new( - "postgres".to_string(), - "postgres".to_string(), - PG_PASSWORD.to_string(), - "postgres".to_string(), + engine, + username, + password, + database_name, port, "localhost".to_string(), "localhost".to_string(), @@ -153,12 +159,16 @@ impl Factory for LocalFactory { } impl LocalFactory { - async fn wait_for_ready(&self, container_name: &str) -> Result<(), shuttle_service::Error> { + async fn wait_for_ready( + &self, + container_name: &str, + is_ready_cmd: Vec, + ) -> Result<(), shuttle_service::Error> { loop { trace!("waiting for '{container_name}' to be ready for connections"); let config = CreateExecOptions { - cmd: Some(vec!["pg_isready"]), + cmd: Some(is_ready_cmd.clone()), attach_stdout: Some(true), ..Default::default() }; @@ -177,12 +187,12 @@ impl LocalFactory { if let bollard::exec::StartExecResults::Attached { mut output, .. } = ready_result { while let Some(line) = output.next().await { - if let bollard::container::LogOutput::StdOut { message } = + trace!("line: {:?}", line); + + if let bollard::container::LogOutput::StdOut { .. } = line.expect("output to have a log line") { - if message.ends_with(b"accepting connections\n") { - return Ok(()); - } + return Ok(()); } } } @@ -260,3 +270,76 @@ fn print_layers(layers: &Vec) { )) .expect("to reset cursor position"); } + +struct EngineConfig { + r#type: String, + image: String, + engine: String, + username: String, + password: String, + database_name: String, + port: String, + env: Option>, + is_ready_cmd: Vec, +} + +fn db_type_to_config(db_type: Type) -> EngineConfig { + match db_type { + Type::Shared => EngineConfig { + r#type: "shared_postgres".to_string(), + image: "postgres:11".to_string(), + engine: "postgres".to_string(), + username: "postgres".to_string(), + password: "postgres".to_string(), + database_name: "postgres".to_string(), + port: "5432/tcp".to_string(), + env: Some(vec!["POSTGRES_PASSWORD=postgres".to_string()]), + is_ready_cmd: vec!["pg_isready".to_string()], + }, + Type::AwsRds(AwsRdsEngine::Postgres) => EngineConfig { + r#type: "aws_rds_postgres".to_string(), + image: "postgres:13.4".to_string(), + engine: "postgres".to_string(), + username: "postgres".to_string(), + password: "postgres".to_string(), + database_name: "postgres".to_string(), + port: "5432/tcp".to_string(), + env: Some(vec!["POSTGRES_PASSWORD=postgres".to_string()]), + is_ready_cmd: vec!["pg_isready".to_string()], + }, + Type::AwsRds(AwsRdsEngine::MariaDB) => EngineConfig { + r#type: "aws_rds_mariadb".to_string(), + image: "mariadb:10.6.7".to_string(), + engine: "mariadb".to_string(), + username: "root".to_string(), + password: "mariadb".to_string(), + database_name: "mysql".to_string(), + port: "3306/tcp".to_string(), + env: Some(vec!["MARIADB_ROOT_PASSWORD=mariadb".to_string()]), + is_ready_cmd: vec![ + "mysql".to_string(), + "-pmariadb".to_string(), + "--silent".to_string(), + "-e".to_string(), + "show databases;".to_string(), + ], + }, + Type::AwsRds(AwsRdsEngine::MySql) => EngineConfig { + r#type: "aws_rds_mysql".to_string(), + image: "mysql:8.0.28".to_string(), + engine: "mysql".to_string(), + username: "root".to_string(), + password: "mysql".to_string(), + database_name: "mysql".to_string(), + port: "3306/tcp".to_string(), + env: Some(vec!["MYSQL_ROOT_PASSWORD=mysql".to_string()]), + is_ready_cmd: vec![ + "mysql".to_string(), + "-pmysql".to_string(), + "--silent".to_string(), + "-e".to_string(), + "show databases;".to_string(), + ], + }, + } +} From 4c548429bac8da265f2491e4077b568df391c186 Mon Sep 17 00:00:00 2001 From: chesedo Date: Wed, 15 Jun 2022 17:59:26 +0200 Subject: [PATCH 46/61] refactor: sort Cargo.toml --- codegen/Cargo.toml | 2 +- examples/tide/postgres/Cargo.toml | 5 ++--- service/Cargo.toml | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index cfa862863..68f475f06 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -10,8 +10,8 @@ description = "Proc-macro code generator for the shuttle.rs service" proc-macro = true [dependencies] -proc-macro2 = "1.0.39" proc-macro-error = "1.0" +proc-macro2 = "1.0.39" quote = "1.0.18" syn = { version = "1.0.96", features = ["full", "extra-traits"] } diff --git a/examples/tide/postgres/Cargo.toml b/examples/tide/postgres/Cargo.toml index 8e4929c27..80bd1ae84 100644 --- a/examples/tide/postgres/Cargo.toml +++ b/examples/tide/postgres/Cargo.toml @@ -5,11 +5,10 @@ edition = "2021" [lib] crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tide = "0.16.0" serde = { version = "1.0", features = ["derive"] } -sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } shuttle-service = { version = "0.3", features = ["sqlx-aws-postgres", "web-tide"] } +sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } +tide = "0.16.0" diff --git a/service/Cargo.toml b/service/Cargo.toml index 161cc67a2..4873055f7 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -29,19 +29,25 @@ tide = { version = "0.16.0", optional = true } tokio = { version = "1.19.2", features = ["rt", "rt-multi-thread"] } tower = { version = "0.4.12", features = ["make"], optional = true } +# Tide does not have tokio support. So make sure async-std is compatible with tokio +# https://github.com/http-rs/tide/issues/791 +[dependencies.async-std] +version = "1" +features = ["tokio1"] + [dependencies.shuttle-codegen] version = "0.3.1" path = "../codegen" optional = true -[dependencies.shuttle-common] -version = "0.3.1" -path = "../common" - [dev-dependencies] portpicker = "0.1.1" uuid = { version = "1.1.1", features = ["v4"] } +[dependencies.shuttle-common] +version = "0.3.1" +path = "../common" + [features] default = ["codegen"] codegen = ["shuttle-codegen"] @@ -59,9 +65,3 @@ web-axum = ["axum", "sync_wrapper"] web-rocket = ["rocket"] web-tide = ["tide"] web-tower = ["tower", "hyper"] - -# Tide does not have tokio support. So make sure async-std is compatible with tokio -# https://github.com/http-rs/tide/issues/791 -[dependencies.async-std] -version = "1" -features = ["tokio1"] From 9dea2531acfa06fb94f338eba2ee58cd7ff17c19 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 08:22:35 +0200 Subject: [PATCH 47/61] tests: provisioner await --- provisioner/src/lib.rs | 2 +- provisioner/tests/provisioner.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs index b300de929..8eebac047 100644 --- a/provisioner/src/lib.rs +++ b/provisioner/src/lib.rs @@ -55,7 +55,7 @@ impl MyProvisioner { Ok(Self { pool, rds_client }) } - async fn request_shared_db(&self, project_name: &str) -> Result { + pub async fn request_shared_db(&self, project_name: &str) -> Result { let (username, password) = self.shared_role(project_name).await?; let database_name = self.shared_db(project_name, &username).await?; diff --git a/provisioner/tests/provisioner.rs b/provisioner/tests/provisioner.rs index cb04fe8da..294416f40 100644 --- a/provisioner/tests/provisioner.rs +++ b/provisioner/tests/provisioner.rs @@ -110,7 +110,7 @@ fn exec(query: &str) -> String { #[tokio::test] async fn shared_db_role_does_not_exist() { - let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); assert_eq!( exec("SELECT rolname FROM pg_roles WHERE rolname = 'user-not_exist'"), @@ -127,7 +127,7 @@ async fn shared_db_role_does_not_exist() { #[tokio::test] async fn shared_db_role_does_exist() { - let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); exec("CREATE ROLE \"user-exist\" WITH LOGIN PASSWORD 'temp'"); assert_eq!( @@ -149,7 +149,7 @@ async fn shared_db_role_does_exist() { expected = "CreateRole(\"error returned from database: cannot insert multiple commands into a prepared statement\"" )] async fn injection_safe() { - let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); provisioner .request_shared_db("new\"; CREATE ROLE \"injected") @@ -159,7 +159,7 @@ async fn injection_safe() { #[tokio::test] async fn shared_db_missing() { - let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); assert_eq!( exec("SELECT datname FROM pg_database WHERE datname = 'db-missing'"), @@ -176,7 +176,7 @@ async fn shared_db_missing() { #[tokio::test] async fn shared_db_filled() { - let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); exec("CREATE ROLE \"user-filled\" WITH LOGIN PASSWORD 'temp'"); exec("CREATE DATABASE \"db-filled\" OWNER 'user-filled'"); From 7b406f2a8934da49cd0850b1ba42862b984810c4 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 08:22:58 +0200 Subject: [PATCH 48/61] refactor: undo secrets patch --- service/src/secrets.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/service/src/secrets.rs b/service/src/secrets.rs index 2e3a42288..f5e14e8a0 100644 --- a/service/src/secrets.rs +++ b/service/src/secrets.rs @@ -4,9 +4,7 @@ use lazy_static::lazy_static; use regex::Regex; use sqlx::postgres::PgArguments; use sqlx::query::Query; -use sqlx::{ - ColumnIndex, Database, Decode, Encode, Execute, Executor, IntoArguments, Postgres, Row, Type, -}; +use sqlx::{ColumnIndex, Database, Decode, Execute, Executor, Postgres, Row, Type}; use crate::error::Error; @@ -26,13 +24,11 @@ fn check_and_lower_secret_key(key: &str) -> Result { /// should you prefer). The table in question is created if it is found to not exist every time /// either [`get_secret`] or [`set_secret`] is called. #[async_trait] -pub trait SecretStore +pub trait SecretStore where - DB: Database, - Arguments: for<'q> IntoArguments<'q, DB>, + DB: Database, for<'c> &'c Self: Executor<'c, Database = DB>, - for<'c> String: Decode<'c, DB> + Encode<'c, DB> + Type, - for<'c> &'c str: Encode<'c, DB> + Type, + for<'c> String: Decode<'c, DB> + Type, for<'c> usize: ColumnIndex<::Row>, for<'c> Query<'c, Postgres, PgArguments>: Execute<'c, DB>, { @@ -71,7 +67,7 @@ where } #[async_trait] -impl SecretStore for sqlx::PgPool { +impl SecretStore for sqlx::PgPool { const GET_QUERY: &'static str = "SELECT value FROM secrets WHERE key = $1"; const SET_QUERY: &'static str = "INSERT INTO secrets (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = $2"; From de19f4163d2c4adfe9d315e1565dc7f94e51e20c Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 09:05:24 +0200 Subject: [PATCH 49/61] refactor: fix pg_isready --- cargo-shuttle/src/factory.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cargo-shuttle/src/factory.rs b/cargo-shuttle/src/factory.rs index 3d415b1c7..12c1432fe 100644 --- a/cargo-shuttle/src/factory.rs +++ b/cargo-shuttle/src/factory.rs @@ -9,7 +9,7 @@ use bollard::{ }; use colored::Colorize; use crossterm::{ - cursor::MoveUp, + cursor::{MoveDown, MoveUp}, terminal::{Clear, ClearType}, QueueableCommand, }; @@ -170,6 +170,7 @@ impl LocalFactory { let config = CreateExecOptions { cmd: Some(is_ready_cmd.clone()), attach_stdout: Some(true), + attach_stderr: Some(true), ..Default::default() }; @@ -229,6 +230,13 @@ impl LocalFactory { print_layers(&layers); } + // Undo last MoveUps + stdout() + .queue(MoveDown( + layers.len().try_into().expect("to convert usize to u16"), + )) + .expect("to reset cursor position"); + Ok(()) } } @@ -294,7 +302,11 @@ fn db_type_to_config(db_type: Type) -> EngineConfig { database_name: "postgres".to_string(), port: "5432/tcp".to_string(), env: Some(vec!["POSTGRES_PASSWORD=postgres".to_string()]), - is_ready_cmd: vec!["pg_isready".to_string()], + is_ready_cmd: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "pg_isready | grep 'accepting connections'".to_string(), + ], }, Type::AwsRds(AwsRdsEngine::Postgres) => EngineConfig { r#type: "aws_rds_postgres".to_string(), @@ -305,7 +317,11 @@ fn db_type_to_config(db_type: Type) -> EngineConfig { database_name: "postgres".to_string(), port: "5432/tcp".to_string(), env: Some(vec!["POSTGRES_PASSWORD=postgres".to_string()]), - is_ready_cmd: vec!["pg_isready".to_string()], + is_ready_cmd: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "pg_isready | grep 'accepting connections'".to_string(), + ], }, Type::AwsRds(AwsRdsEngine::MariaDB) => EngineConfig { r#type: "aws_rds_mariadb".to_string(), From 21f6bf8b49162c998efe2808c0edf6cd790c6296 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 09:26:23 +0200 Subject: [PATCH 50/61] refactor: args for public and private PG addresses --- Cargo.lock | 1 + provisioner/Cargo.toml | 1 + provisioner/docker/supervisord.conf | 2 +- provisioner/src/args.rs | 27 ++++++++++++++++--- provisioner/src/lib.rs | 20 ++++++++------ provisioner/src/main.rs | 6 ++++- provisioner/tests/provisioner.rs | 20 ++++++++++---- terraform/modules/shuttle/service.tf | 1 + .../system/shuttle-provisioner.service.tftpl | 1 + 9 files changed, 61 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c8866df2..2fbc935f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3630,6 +3630,7 @@ dependencies = [ "aws-smithy-types", "clap 3.1.18", "ctor", + "fqdn", "lazy_static", "portpicker", "prost", diff --git a/provisioner/Cargo.toml b/provisioner/Cargo.toml index d52995ac8..252cebd95 100644 --- a/provisioner/Cargo.toml +++ b/provisioner/Cargo.toml @@ -10,6 +10,7 @@ aws-config = "0.12" aws-sdk-rds = "0.12" aws-smithy-types = "0.42" clap = { version = "3.1.18", features = ["derive", "env"] } +fqdn = "0.1.9" prost = "0.10.4" rand = "0.8.5" sqlx = { version = "0.5.13", features = ["postgres", "runtime-tokio-native-tls"] } diff --git a/provisioner/docker/supervisord.conf b/provisioner/docker/supervisord.conf index d085dcf30..7fc92f4c4 100644 --- a/provisioner/docker/supervisord.conf +++ b/provisioner/docker/supervisord.conf @@ -8,7 +8,7 @@ startsecs=20 autorestart=true [program:shuttle-provisioner] -command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s +command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %{ENV_FQDN} redirect_stderr=true environment=RUST_BACKTRACE="1",RUST_LOG="debug" startretries=3 diff --git a/provisioner/src/args.rs b/provisioner/src/args.rs index 3bf09c09e..567cbc38d 100644 --- a/provisioner/src/args.rs +++ b/provisioner/src/args.rs @@ -1,19 +1,40 @@ -use std::net::{IpAddr, Ipv4Addr}; +use std::{ + net::{IpAddr, Ipv4Addr}, + str::FromStr, +}; use clap::Parser; +use fqdn::FQDN; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] pub struct Args { /// Address to bind provisioner on - #[clap(short, long, env = "PROVISIONER_IP", default_value_t = Ipv4Addr::LOCALHOST.into())] + #[clap(long, env = "PROVISIONER_IP", default_value_t = Ipv4Addr::LOCALHOST.into())] pub ip: IpAddr, /// Port to start provisioner on - #[clap(short, long, env = "PROVISIONER_PORT", default_value_t = 5001)] + #[clap(long, env = "PROVISIONER_PORT", default_value_t = 5001)] pub port: u16, /// URI to connect to Postgres for managing shared DB resources #[clap(short, long, env = "PROVISIONER_PG_URI", hide_env_values = true)] pub shared_pg_uri: String, + + /// Fully qualified domain name this provisioner instance is reachable at + #[clap(long, env = "PROVISIONER_FQDN", parse(try_from_str = parse_fqdn))] + pub fqdn: FQDN, + + /// Address this provisioner can be reached at on the internal network + #[clap( + short, + long, + env = "PROVISIONER_ADDRESS", + default_value = "provisioner" + )] + pub internal_address: String, +} + +fn parse_fqdn(src: &str) -> Result { + FQDN::from_str(src).map_err(|e| format!("{e:?}")) } diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs index 8eebac047..7b78d86a5 100644 --- a/provisioner/src/lib.rs +++ b/provisioner/src/lib.rs @@ -19,9 +19,6 @@ use tracing::{debug, info}; mod args; mod error; -const PRIVATE_PG_IP: &str = "provisioner"; -const PUBLIC_PG_IP: &str = "pg.shuttle.rs"; - const AWS_RDS_CLASS: &str = "db.t4g.micro"; const MASTER_USERNAME: &str = "master"; const RDS_SUBNET_GROUP: &str = "shuttle_rds"; @@ -29,15 +26,17 @@ const RDS_SUBNET_GROUP: &str = "shuttle_rds"; pub struct MyProvisioner { pool: PgPool, rds_client: aws_sdk_rds::Client, + fqdn: String, + internal_address: String, } impl MyProvisioner { - pub async fn new(uri: &str) -> sqlx::Result { + pub async fn new(db_uri: &str, fqdn: String, internal_address: String) -> sqlx::Result { let pool = PgPoolOptions::new() .min_connections(4) .max_connections(12) .connect_timeout(Duration::from_secs(60)) - .connect_lazy(uri)?; + .connect_lazy(db_uri)?; // Default timeout is too long so lowering it let api_timeout_config = timeout::Api::new() @@ -52,7 +51,12 @@ impl MyProvisioner { let rds_client = aws_sdk_rds::Client::new(&aws_config); - Ok(Self { pool, rds_client }) + Ok(Self { + pool, + rds_client, + fqdn, + internal_address, + }) } pub async fn request_shared_db(&self, project_name: &str) -> Result { @@ -64,8 +68,8 @@ impl MyProvisioner { username, password, database_name, - address_private: PRIVATE_PG_IP.to_string(), - address_public: PUBLIC_PG_IP.to_string(), + address_private: self.internal_address.clone(), + address_public: self.fqdn.clone(), port: "3306".to_string(), }) } diff --git a/provisioner/src/main.rs b/provisioner/src/main.rs index cc538225f..5575a3b80 100644 --- a/provisioner/src/main.rs +++ b/provisioner/src/main.rs @@ -12,10 +12,14 @@ async fn main() -> Result<(), Box> { ip, port, shared_pg_uri, + fqdn, + internal_address, } = Args::parse(); let addr = SocketAddr::new(ip, port); - let provisioner = MyProvisioner::new(&shared_pg_uri).await.unwrap(); + let provisioner = MyProvisioner::new(&shared_pg_uri, fqdn.to_string(), internal_address) + .await + .unwrap(); println!("starting provisioner on {}", addr); Server::builder() diff --git a/provisioner/tests/provisioner.rs b/provisioner/tests/provisioner.rs index 294416f40..7276672f2 100644 --- a/provisioner/tests/provisioner.rs +++ b/provisioner/tests/provisioner.rs @@ -110,7 +110,9 @@ fn exec(query: &str) -> String { #[tokio::test] async fn shared_db_role_does_not_exist() { - let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); + let provisioner = MyProvisioner::new(&PG.uri, "fqdn".to_string(), "internal".to_string()) + .await + .unwrap(); assert_eq!( exec("SELECT rolname FROM pg_roles WHERE rolname = 'user-not_exist'"), @@ -127,7 +129,9 @@ async fn shared_db_role_does_not_exist() { #[tokio::test] async fn shared_db_role_does_exist() { - let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); + let provisioner = MyProvisioner::new(&PG.uri, "fqdn".to_string(), "internal".to_string()) + .await + .unwrap(); exec("CREATE ROLE \"user-exist\" WITH LOGIN PASSWORD 'temp'"); assert_eq!( @@ -149,7 +153,9 @@ async fn shared_db_role_does_exist() { expected = "CreateRole(\"error returned from database: cannot insert multiple commands into a prepared statement\"" )] async fn injection_safe() { - let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); + let provisioner = MyProvisioner::new(&PG.uri, "fqdn".to_string(), "internal".to_string()) + .await + .unwrap(); provisioner .request_shared_db("new\"; CREATE ROLE \"injected") @@ -159,7 +165,9 @@ async fn injection_safe() { #[tokio::test] async fn shared_db_missing() { - let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); + let provisioner = MyProvisioner::new(&PG.uri, "fqdn".to_string(), "internal".to_string()) + .await + .unwrap(); assert_eq!( exec("SELECT datname FROM pg_database WHERE datname = 'db-missing'"), @@ -176,7 +184,9 @@ async fn shared_db_missing() { #[tokio::test] async fn shared_db_filled() { - let provisioner = MyProvisioner::new(&PG.uri).await.unwrap(); + let provisioner = MyProvisioner::new(&PG.uri, "fqdn".to_string(), "internal".to_string()) + .await + .unwrap(); exec("CREATE ROLE \"user-filled\" WITH LOGIN PASSWORD 'temp'"); exec("CREATE DATABASE \"db-filled\" OWNER 'user-filled'"); diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index 32b039b2d..8be8f2ae1 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -154,6 +154,7 @@ locals { data_dir = local.data_dir, docker_image = local.docker_provisioner_image, pg_password = var.postgres_password, + fqdn = var.pg_fqdn } ) } diff --git a/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl b/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl index bd212375d..1ba546fde 100644 --- a/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl +++ b/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl @@ -15,6 +15,7 @@ ExecStart=/usr/bin/docker run --rm \ -e PG_PORT=5432 \ -p 5432:5432 \ -e PORT=5001 \ + -e FQDN=${fqdn} \ -e PG_PASSWORD=${pg_password} \ -e PG_DATA=/opt/shuttle/postgres \ -v ${data_dir}/conf/postgres:/etc/postgresql/11/shuttle:rw \ From 96ac06428f8455a5e87f8978754dc0bb9828c3aa Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 09:46:16 +0200 Subject: [PATCH 51/61] refactor: clippy suggestions --- service/tests/loader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/tests/loader.rs b/service/tests/loader.rs index 52df8a8cb..065ce4c44 100644 --- a/service/tests/loader.rs +++ b/service/tests/loader.rs @@ -3,7 +3,7 @@ mod helpers; use helpers::{build_so_create_loader, PostgresInstance}; use shuttle_service::loader::LoaderError; -use shuttle_service::{Error, Factory}; +use shuttle_service::{database, Error, Factory}; use std::net::{Ipv4Addr, SocketAddr}; use std::process::exit; From dec2e23188019290d3c7028691b33e6d2a9100d7 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 11:55:41 +0200 Subject: [PATCH 52/61] bug: supervisord args --- provisioner/docker/supervisord.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/docker/supervisord.conf b/provisioner/docker/supervisord.conf index 7fc92f4c4..11066fc0c 100644 --- a/provisioner/docker/supervisord.conf +++ b/provisioner/docker/supervisord.conf @@ -8,7 +8,7 @@ startsecs=20 autorestart=true [program:shuttle-provisioner] -command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %{ENV_FQDN} +command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %{ENV_FQDN}s redirect_stderr=true environment=RUST_BACKTRACE="1",RUST_LOG="debug" startretries=3 From 3a9020caf4dda98c324cbc29b5aa0c287bc8e4b5 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 11:57:44 +0200 Subject: [PATCH 53/61] bug: supervisord args v2 --- provisioner/docker/supervisord.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/docker/supervisord.conf b/provisioner/docker/supervisord.conf index 11066fc0c..0a40c0028 100644 --- a/provisioner/docker/supervisord.conf +++ b/provisioner/docker/supervisord.conf @@ -8,7 +8,7 @@ startsecs=20 autorestart=true [program:shuttle-provisioner] -command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %{ENV_FQDN}s +command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %(ENV_FQDN)s redirect_stderr=true environment=RUST_BACKTRACE="1",RUST_LOG="debug" startretries=3 From d56800a74958f47fe3a5389a892c240344ce8104 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 13:03:24 +0200 Subject: [PATCH 54/61] fix: test helper --- docker-compose.yml | 3 ++- e2e/tests/helpers/mod.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d77f6fa6b..d7728b3c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,4 +20,5 @@ services: context: . dockerfile: provisioner/Containerfile.dev environment: - - PORT=5001 \ No newline at end of file + - FQDN=unreacable + - PORT=5001 diff --git a/e2e/tests/helpers/mod.rs b/e2e/tests/helpers/mod.rs index 2ed41c8a4..27e0cc90f 100644 --- a/e2e/tests/helpers/mod.rs +++ b/e2e/tests/helpers/mod.rs @@ -153,6 +153,8 @@ impl Services { "--network", "shuttle-net", "-e", + "FQDN=unreacable", + "-e", "PORT=5001", &provisioner_image, ]); From 93023ea5d8d0705f5f5677668c6958adb7172f7c Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 13:03:42 +0200 Subject: [PATCH 55/61] bug: security group --- terraform/modules/shuttle/api.tf | 2 +- terraform/modules/shuttle/load-balancing.tf | 6 +++--- terraform/modules/shuttle/networking.tf | 2 +- terraform/modules/shuttle/service.tf | 2 +- terraform/modules/shuttle/storage.tf | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/terraform/modules/shuttle/api.tf b/terraform/modules/shuttle/api.tf index c5041c6e9..17713a4fe 100644 --- a/terraform/modules/shuttle/api.tf +++ b/terraform/modules/shuttle/api.tf @@ -23,7 +23,7 @@ resource "aws_apigatewayv2_api_mapping" "backend" { resource "aws_apigatewayv2_vpc_link" "private" { name = "shuttle-api-gateway-vpc-link" - security_group_ids = [aws_security_group.unreasonable.id] + security_group_ids = [aws_default_security_group.default.id] subnet_ids = [aws_subnet.backend_a.id, aws_subnet.backend_b.id] } diff --git a/terraform/modules/shuttle/load-balancing.tf b/terraform/modules/shuttle/load-balancing.tf index 0aa981062..25ac9b29f 100644 --- a/terraform/modules/shuttle/load-balancing.tf +++ b/terraform/modules/shuttle/load-balancing.tf @@ -5,7 +5,7 @@ resource "aws_lb" "api" { load_balancer_type = "application" - security_groups = [aws_security_group.unreasonable.id] + security_groups = [aws_default_security_group.default.id] subnets = [aws_subnet.backend_a.id, aws_subnet.backend_b.id] access_logs { @@ -22,7 +22,7 @@ resource "aws_lb" "db" { load_balancer_type = "network" - //security_groups = [aws_security_group.unreasonable.id] + //security_groups = [aws_default_security_group.default.id] subnets = [aws_subnet.backend_a.id, aws_subnet.backend_b.id] access_logs { @@ -84,7 +84,7 @@ resource "aws_lb" "user" { load_balancer_type = "application" - security_groups = [aws_security_group.unreasonable.id] + security_groups = [aws_default_security_group.default.id] subnets = [aws_subnet.backend_a.id, aws_subnet.backend_b.id] access_logs { diff --git a/terraform/modules/shuttle/networking.tf b/terraform/modules/shuttle/networking.tf index 8c7f05224..b03b1bc4a 100644 --- a/terraform/modules/shuttle/networking.tf +++ b/terraform/modules/shuttle/networking.tf @@ -30,7 +30,7 @@ resource "aws_network_acl_rule" "mysql" { to_port = 3306 } -resource "aws_security_group" "unreasonable" { +resource "aws_default_security_group" "default" { vpc_id = aws_vpc.backend.id ingress { diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index 8be8f2ae1..62df8250c 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -8,7 +8,7 @@ resource "aws_eip" "backend" { } resource "aws_network_interface_sg_attachment" "backend" { - security_group_id = aws_security_group.unreasonable.id + security_group_id = aws_default_security_group.default.id network_interface_id = aws_network_interface.backend.id } diff --git a/terraform/modules/shuttle/storage.tf b/terraform/modules/shuttle/storage.tf index bb47a75f6..352b94d06 100644 --- a/terraform/modules/shuttle/storage.tf +++ b/terraform/modules/shuttle/storage.tf @@ -49,11 +49,11 @@ resource "aws_efs_file_system" "user_data" { resource "aws_efs_mount_target" "user_data_a" { file_system_id = aws_efs_file_system.user_data.id subnet_id = aws_subnet.backend_a.id - security_groups = [aws_security_group.unreasonable.id] + security_groups = [aws_default_security_group.default.id] } resource "aws_efs_mount_target" "user_data_b" { file_system_id = aws_efs_file_system.user_data.id subnet_id = aws_subnet.backend_b.id - security_groups = [aws_security_group.unreasonable.id] + security_groups = [aws_default_security_group.default.id] } From c5133475affbbd8d911277ce5be92953ceac4196 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 13:03:58 +0200 Subject: [PATCH 56/61] refactor: tide postgres version --- examples/tide/postgres/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tide/postgres/Cargo.toml b/examples/tide/postgres/Cargo.toml index 80bd1ae84..95c0274c6 100644 --- a/examples/tide/postgres/Cargo.toml +++ b/examples/tide/postgres/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [dependencies] serde = { version = "1.0", features = ["derive"] } -shuttle-service = { version = "0.3", features = ["sqlx-aws-postgres", "web-tide"] } +shuttle-service = { version = "0.3.3", features = ["sqlx-aws-postgres", "web-tide"] } sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } tide = "0.16.0" From eba0d01dfc3b2505c9d168728caf95b42fdf3e7c Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 15:12:47 +0200 Subject: [PATCH 57/61] refactor: more helper fixes --- e2e/tests/helpers/mod.rs | 2 ++ provisioner/docker/entrypoint.sh | 2 ++ provisioner/docker/supervisord.conf | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/tests/helpers/mod.rs b/e2e/tests/helpers/mod.rs index 27e0cc90f..f2099d986 100644 --- a/e2e/tests/helpers/mod.rs +++ b/e2e/tests/helpers/mod.rs @@ -155,6 +155,8 @@ impl Services { "-e", "FQDN=unreacable", "-e", + &format!("INTERNAL_ADDRESS={provisioner_image}"), + "-e", "PORT=5001", &provisioner_image, ]); diff --git a/provisioner/docker/entrypoint.sh b/provisioner/docker/entrypoint.sh index af1fec058..4cb00ed3a 100755 --- a/provisioner/docker/entrypoint.sh +++ b/provisioner/docker/entrypoint.sh @@ -36,4 +36,6 @@ export PG_HOST=localhost export PG_URI=postgres://postgres:${PG_PASSWORD}@localhost:${PG_PORT}/postgres +export INTERNAL_ADDRESS=${INTERNAL_ADDRESS:provisioner} + exec supervisord -n -c /usr/share/supervisord/supervisord.conf diff --git a/provisioner/docker/supervisord.conf b/provisioner/docker/supervisord.conf index 0a40c0028..17e739ddb 100644 --- a/provisioner/docker/supervisord.conf +++ b/provisioner/docker/supervisord.conf @@ -8,7 +8,7 @@ startsecs=20 autorestart=true [program:shuttle-provisioner] -command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %(ENV_FQDN)s +command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %(ENV_FQDN)s --internal-address %(ENV_INTERNAL_ADDRESS)s redirect_stderr=true environment=RUST_BACKTRACE="1",RUST_LOG="debug" startretries=3 From b7e993dc257c45308011026831e600911b1f0a38 Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 15:20:04 +0200 Subject: [PATCH 58/61] bug: INTERNAL_ADDRESS --- provisioner/docker/entrypoint.sh | 2 +- provisioner/docker/supervisord.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/provisioner/docker/entrypoint.sh b/provisioner/docker/entrypoint.sh index 4cb00ed3a..a269479bb 100755 --- a/provisioner/docker/entrypoint.sh +++ b/provisioner/docker/entrypoint.sh @@ -36,6 +36,6 @@ export PG_HOST=localhost export PG_URI=postgres://postgres:${PG_PASSWORD}@localhost:${PG_PORT}/postgres -export INTERNAL_ADDRESS=${INTERNAL_ADDRESS:provisioner} +export INTERNAL_ADDRESS=${INTERNAL_ADDRESS:-provisioner} exec supervisord -n -c /usr/share/supervisord/supervisord.conf diff --git a/provisioner/docker/supervisord.conf b/provisioner/docker/supervisord.conf index 17e739ddb..0a40c0028 100644 --- a/provisioner/docker/supervisord.conf +++ b/provisioner/docker/supervisord.conf @@ -8,7 +8,7 @@ startsecs=20 autorestart=true [program:shuttle-provisioner] -command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %(ENV_FQDN)s --internal-address %(ENV_INTERNAL_ADDRESS)s +command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s --fqdn %(ENV_FQDN)s redirect_stderr=true environment=RUST_BACKTRACE="1",RUST_LOG="debug" startretries=3 From 33bd7b00a724c3c14d954853c20135712fac1cdb Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 15:34:23 +0200 Subject: [PATCH 59/61] refactor: remove redundant security group link --- terraform/modules/shuttle/service.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index 62df8250c..087b5c3e0 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -7,11 +7,6 @@ resource "aws_eip" "backend" { network_interface = aws_network_interface.backend.id } -resource "aws_network_interface_sg_attachment" "backend" { - security_group_id = aws_default_security_group.default.id - network_interface_id = aws_network_interface.backend.id -} - resource "aws_iam_instance_profile" "backend" { name = "backend-profile" role = aws_iam_role.backend.name From fbb1588765119a16cfb226ad48444c0a6561abad Mon Sep 17 00:00:00 2001 From: chesedo Date: Thu, 16 Jun 2022 16:38:35 +0200 Subject: [PATCH 60/61] bug: PG timeout --- e2e/tests/helpers/mod.rs | 2 +- provisioner/docker/entrypoint.sh | 2 +- provisioner/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/tests/helpers/mod.rs b/e2e/tests/helpers/mod.rs index f2099d986..6c84b4c6a 100644 --- a/e2e/tests/helpers/mod.rs +++ b/e2e/tests/helpers/mod.rs @@ -155,7 +155,7 @@ impl Services { "-e", "FQDN=unreacable", "-e", - &format!("INTERNAL_ADDRESS={provisioner_image}"), + &format!("PROVISIONER_ADDRESS={provisioner_container}"), "-e", "PORT=5001", &provisioner_image, diff --git a/provisioner/docker/entrypoint.sh b/provisioner/docker/entrypoint.sh index a269479bb..ba81a63e5 100755 --- a/provisioner/docker/entrypoint.sh +++ b/provisioner/docker/entrypoint.sh @@ -36,6 +36,6 @@ export PG_HOST=localhost export PG_URI=postgres://postgres:${PG_PASSWORD}@localhost:${PG_PORT}/postgres -export INTERNAL_ADDRESS=${INTERNAL_ADDRESS:-provisioner} +export PROVISIONER_ADDRESS=${PROVISIONER_ADDRESS:-provisioner} exec supervisord -n -c /usr/share/supervisord/supervisord.conf diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs index 7b78d86a5..0a3e882b6 100644 --- a/provisioner/src/lib.rs +++ b/provisioner/src/lib.rs @@ -70,7 +70,7 @@ impl MyProvisioner { database_name, address_private: self.internal_address.clone(), address_public: self.fqdn.clone(), - port: "3306".to_string(), + port: "5432".to_string(), }) } From b54ae81584e4dce854be34237ec9f57db12abdb0 Mon Sep 17 00:00:00 2001 From: Pieter Date: Thu, 16 Jun 2022 18:00:47 +0200 Subject: [PATCH 61/61] Apply suggestions from code review Co-authored-by: Max <42641081+bmoxb@users.noreply.github.com> --- examples/tide/postgres/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tide/postgres/src/lib.rs b/examples/tide/postgres/src/lib.rs index 3bd88bcf5..bf6a481ff 100644 --- a/examples/tide/postgres/src/lib.rs +++ b/examples/tide/postgres/src/lib.rs @@ -10,7 +10,7 @@ async fn retrieve(req: Request) -> tide::Result { .fetch_one(&req.state().pool) .await?; - Body::from_json(&todo).map(core::convert::Into::into) + Body::from_json(&todo).map(Into::into) } async fn add(mut req: Request) -> tide::Result { @@ -20,7 +20,7 @@ async fn add(mut req: Request) -> tide::Result { .fetch_one(&req.state().pool) .await?; - Body::from_json(&todo).map(core::convert::Into::into) + Body::from_json(&todo).map(Into::into) } #[derive(Clone)]