From 68154226650808d795d018989ea269c60c47c96d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Oct 2021 21:42:13 +0200 Subject: [PATCH 01/31] clients: add support for `webpki and native certificate stores` (#533) * Update tokio-rustls requirement from 0.22 to 0.23 Updates the requirements on [tokio-rustls](https://github.com/tokio-rs/tls) to permit the latest version. - [Release notes](https://github.com/tokio-rs/tls/releases) - [Commits](https://github.com/tokio-rs/tls/commits) --- updated-dependencies: - dependency-name: tokio-rustls dependency-type: direct:production ... Signed-off-by: dependabot[bot] * push fixes but requires rustls-native-certs v0.6 * update native certs to 0.6.0 * fix clippy warnings * remove webpki roots support * Revert "remove webpki roots support" This reverts commit 1144d567b343049ab7c967d320fc2fe162ba0f7c. * support both native cert store and webpki * sort deps in Cargo.toml * Update ws-client/src/transport.rs Co-authored-by: David Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Niklas Adolfsson Co-authored-by: David --- http-client/Cargo.toml | 8 ++-- http-client/src/client.rs | 14 ++++-- http-client/src/transport.rs | 23 +++++++-- types/src/client.rs | 10 ++++ ws-client/Cargo.toml | 14 +++--- ws-client/src/client.rs | 5 +- ws-client/src/transport.rs | 90 +++++++++++++++++++++--------------- 7 files changed, 105 insertions(+), 59 deletions(-) diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index 5f29bcf01d..f2fbaf4420 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -11,17 +11,17 @@ documentation = "https://docs.rs/jsonrpsee-http-client" [dependencies] async-trait = "0.1" -hyper-rustls = "0.22" +fnv = "1" hyper = { version = "0.14.10", features = ["client", "http1", "http2", "tcp"] } +hyper-rustls = { version = "0.22", features = ["webpki-tokio"] } jsonrpsee-types = { path = "../types", version = "0.4.1" } jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["http-helpers"] } -tracing = "0.1" serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" -tokio = { version = "1", features = ["time"] } thiserror = "1.0" +tokio = { version = "1", features = ["time"] } +tracing = "0.1" url = "2.2" -fnv = "1" [dev-dependencies] jsonrpsee-test-utils = { path = "../test-utils" } diff --git a/http-client/src/client.rs b/http-client/src/client.rs index 14f72fc065..d1bff8213b 100644 --- a/http-client/src/client.rs +++ b/http-client/src/client.rs @@ -28,7 +28,7 @@ use crate::transport::HttpTransportClient; use crate::types::{ traits::Client, v2::{Id, NotificationSer, ParamsSer, RequestSer, Response, RpcError}, - Error, RequestIdGuard, TEN_MB_SIZE_BYTES, + CertificateStore, Error, RequestIdGuard, TEN_MB_SIZE_BYTES, }; use async_trait::async_trait; use fnv::FnvHashMap; @@ -41,6 +41,7 @@ pub struct HttpClientBuilder { max_request_body_size: u32, request_timeout: Duration, max_concurrent_requests: usize, + certificate_store: CertificateStore, } impl HttpClientBuilder { @@ -62,10 +63,16 @@ impl HttpClientBuilder { self } + /// Set which certificate store to use. + pub fn certificate_store(mut self, certificate_store: CertificateStore) -> Self { + self.certificate_store = certificate_store; + self + } + /// Build the HTTP client with target to connect to. pub fn build(self, target: impl AsRef) -> Result { - let transport = - HttpTransportClient::new(target, self.max_request_body_size).map_err(|e| Error::Transport(e.into()))?; + let transport = HttpTransportClient::new(target, self.max_request_body_size, self.certificate_store) + .map_err(|e| Error::Transport(e.into()))?; Ok(HttpClient { transport, id_guard: RequestIdGuard::new(self.max_concurrent_requests), @@ -80,6 +87,7 @@ impl Default for HttpClientBuilder { max_request_body_size: TEN_MB_SIZE_BYTES, request_timeout: Duration::from_secs(60), max_concurrent_requests: 256, + certificate_store: CertificateStore::Native, } } } diff --git a/http-client/src/transport.rs b/http-client/src/transport.rs index 1fecdd0f18..e46b3a597b 100644 --- a/http-client/src/transport.rs +++ b/http-client/src/transport.rs @@ -9,6 +9,7 @@ use crate::types::error::GenericTransportError; use hyper::client::{Client, HttpConnector}; use hyper_rustls::HttpsConnector; +use jsonrpsee_types::CertificateStore; use jsonrpsee_utils::http_helpers; use thiserror::Error; @@ -27,10 +28,18 @@ pub(crate) struct HttpTransportClient { impl HttpTransportClient { /// Initializes a new HTTP client. - pub(crate) fn new(target: impl AsRef, max_request_body_size: u32) -> Result { + pub(crate) fn new( + target: impl AsRef, + max_request_body_size: u32, + cert_store: CertificateStore, + ) -> Result { let target = url::Url::parse(target.as_ref()).map_err(|e| Error::Url(format!("Invalid URL: {}", e)))?; if target.scheme() == "http" || target.scheme() == "https" { - let connector = HttpsConnector::with_native_roots(); + let connector = match cert_store { + CertificateStore::Native => HttpsConnector::with_native_roots(), + CertificateStore::WebPki => HttpsConnector::with_webpki_roots(), + _ => return Err(Error::InvalidCertficateStore), + }; let client = Client::builder().build::<_, hyper::Body>(connector); Ok(HttpTransportClient { target, client, max_request_body_size }) } else { @@ -99,6 +108,10 @@ pub(crate) enum Error { /// Malformed request. #[error("Malformed request")] Malformed, + + /// Invalid certificate store. + #[error("Invalid certificate store")] + InvalidCertficateStore, } impl From> for Error @@ -116,18 +129,18 @@ where #[cfg(test)] mod tests { - use super::{Error, HttpTransportClient}; + use super::{CertificateStore, Error, HttpTransportClient}; #[test] fn invalid_http_url_rejected() { - let err = HttpTransportClient::new("ws://localhost:9933", 80).unwrap_err(); + let err = HttpTransportClient::new("ws://localhost:9933", 80, CertificateStore::Native).unwrap_err(); assert!(matches!(err, Error::Url(_))); } #[tokio::test] async fn request_limit_works() { let eighty_bytes_limit = 80; - let client = HttpTransportClient::new("http://localhost:9933", 80).unwrap(); + let client = HttpTransportClient::new("http://localhost:9933", 80, CertificateStore::WebPki).unwrap(); assert_eq!(client.max_request_body_size, eighty_bytes_limit); let body = "a".repeat(81); diff --git a/types/src/client.rs b/types/src/client.rs index 9e4ac5bfc8..3f45387417 100644 --- a/types/src/client.rs +++ b/types/src/client.rs @@ -249,3 +249,13 @@ impl RequestIdGuard { }); } } + +/// What certificate store to use +#[derive(Clone, Copy, Debug, PartialEq)] +#[non_exhaustive] +pub enum CertificateStore { + /// Use the native system certificate store + Native, + /// Use WebPKI's certificate store + WebPki, +} diff --git a/ws-client/Cargo.toml b/ws-client/Cargo.toml index 127676fd38..b6943e5a47 100644 --- a/ws-client/Cargo.toml +++ b/ws-client/Cargo.toml @@ -10,25 +10,25 @@ homepage = "https://github.com/paritytech/jsonrpsee" documentation = "https://docs.rs/jsonrpsee-ws-client" [dependencies] -tokio = { version = "1", features = ["net", "time", "rt-multi-thread", "macros"] } -tokio-rustls = "0.22" -tokio-util = { version = "0.6", features = ["compat"] } - async-trait = "0.1" fnv = "1" futures = { version = "0.3.14", default-features = false, features = ["std"] } http = "0.2" jsonrpsee-types = { path = "../types", version = "0.4.1" } pin-project = "1" -rustls-native-certs = "0.5.0" +rustls-native-certs = "0.6.0" serde = "1" serde_json = "1" soketto = "0.7" thiserror = "1" +tokio = { version = "1", features = ["net", "time", "rt-multi-thread", "macros"] } +tokio-rustls = "0.23" +tokio-util = { version = "0.6", features = ["compat"] } tracing = "0.1" +webpki-roots = "0.22.0" [dev-dependencies] env_logger = "0.9" jsonrpsee-test-utils = { path = "../test-utils" } -jsonrpsee-utils = { path = "../utils" } -tokio = { version = "1", features = ["macros"] } \ No newline at end of file +jsonrpsee-utils = { path = "../utils", features = ["client"] } +tokio = { version = "1", features = ["macros"] } diff --git a/ws-client/src/client.rs b/ws-client/src/client.rs index 68c64bd113..962e54d643 100644 --- a/ws-client/src/client.rs +++ b/ws-client/src/client.rs @@ -28,8 +28,8 @@ use crate::transport::{Receiver as WsReceiver, Sender as WsSender, WsHandshakeEr use crate::types::{ traits::{Client, SubscriptionClient}, v2::{Id, Notification, NotificationSer, ParamsSer, RequestSer, Response, RpcError, SubscriptionResponse}, - BatchMessage, Error, FrontToBack, RegisterNotificationMessage, RequestIdGuard, RequestMessage, Subscription, - SubscriptionKind, SubscriptionMessage, TEN_MB_SIZE_BYTES, + BatchMessage, CertificateStore, Error, FrontToBack, RegisterNotificationMessage, RequestIdGuard, RequestMessage, + Subscription, SubscriptionKind, SubscriptionMessage, TEN_MB_SIZE_BYTES, }; use crate::{ helpers::{ @@ -37,7 +37,6 @@ use crate::{ process_notification, process_single_response, process_subscription_response, stop_subscription, }, manager::RequestManager, - transport::CertificateStore, }; use async_trait::async_trait; use futures::{ diff --git a/ws-client/src/transport.rs b/ws-client/src/transport.rs index 987d84d1f0..e16e94e90a 100644 --- a/ws-client/src/transport.rs +++ b/ws-client/src/transport.rs @@ -24,7 +24,7 @@ // IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -use crate::stream::EitherStream; +use crate::{stream::EitherStream, types::CertificateStore}; use futures::io::{BufReader, BufWriter}; use http::Uri; use soketto::connection; @@ -40,12 +40,7 @@ use std::{ }; use thiserror::Error; use tokio::net::TcpStream; -use tokio_rustls::{ - client::TlsStream, - rustls::ClientConfig, - webpki::{DNSNameRef, InvalidDNSNameError}, - TlsConnector, -}; +use tokio_rustls::{client::TlsStream, rustls, webpki::InvalidDnsNameError, TlsConnector}; type TlsOrPlain = EitherStream>; @@ -88,16 +83,6 @@ pub enum Mode { Tls, } -/// What certificate store to use -#[derive(Clone, Copy, Debug, PartialEq)] -#[non_exhaustive] -pub enum CertificateStore { - /// Use the native system certificate store - Native, - /// Use webPki's certificate store - WebPki, -} - /// Error that can happen during the WebSocket handshake. /// /// If multiple IP addresses are attempted, only the last error is returned, similar to how @@ -122,7 +107,7 @@ pub enum WsHandshakeError { /// Invalid DNS name error for TLS #[error("Invalid DNS name: {0}")] - InvalidDnsName(#[source] InvalidDNSNameError), + InvalidDnsName(#[source] InvalidDnsNameError), /// Server rejected the handshake. #[error("Connection rejected with status code: {status_code}")] @@ -186,12 +171,8 @@ impl<'a> WsTransportClientBuilder<'a> { pub async fn build(self) -> Result<(Sender, Receiver), WsHandshakeError> { let connector = match self.target.mode { Mode::Tls => { - let mut client_config = ClientConfig::default(); - if let CertificateStore::Native = self.certificate_store { - client_config.root_store = rustls_native_certs::load_native_certs() - .map_err(|(_, e)| WsHandshakeError::CertificateStore(e))?; - } - Some(Arc::new(client_config).into()) + let tls_connector = build_tls_config(&self.certificate_store)?; + Some(tls_connector) } Mode::Plain => None, }; @@ -250,16 +231,14 @@ impl<'a> WsTransportClientBuilder<'a> { // Absolute URI. if uri.scheme().is_some() { target = uri.try_into()?; - tls_connector = match target.mode { - Mode::Tls => { - let mut client_config = ClientConfig::default(); - if let CertificateStore::Native = self.certificate_store { - client_config.root_store = rustls_native_certs::load_native_certs() - .map_err(|(_, e)| WsHandshakeError::CertificateStore(e))?; - } - Some(Arc::new(client_config).into()) + match target.mode { + Mode::Tls if tls_connector.is_none() => { + tls_connector = Some(build_tls_config(&self.certificate_store)?); + } + Mode::Tls => (), + Mode::Plain => { + tls_connector = None; } - Mode::Plain => None, }; } // Relative URI. @@ -320,8 +299,8 @@ async fn connect( match tls_connector { None => Ok(TlsOrPlain::Plain(socket)), Some(connector) => { - let dns_name = DNSNameRef::try_from_ascii_str(host)?; - let tls_stream = connector.connect(dns_name, socket).await?; + let server_name: rustls::ServerName = host.try_into().map_err(|e| WsHandshakeError::Url(format!("Invalid host: {} {:?}", host, e).into()))?; + let tls_stream = connector.connect(server_name, socket).await?; Ok(TlsOrPlain::Tls(tls_stream)) } } @@ -336,8 +315,8 @@ impl From for WsHandshakeError { } } -impl From for WsHandshakeError { - fn from(err: InvalidDNSNameError) -> WsHandshakeError { +impl From for WsHandshakeError { + fn from(err: InvalidDnsNameError) -> WsHandshakeError { WsHandshakeError::InvalidDnsName(err) } } @@ -390,6 +369,43 @@ impl TryFrom for Target { } } +// NOTE: this is slow and should be used sparingly. +fn build_tls_config(cert_store: &CertificateStore) -> Result { + let mut roots = tokio_rustls::rustls::RootCertStore::empty(); + + match cert_store { + CertificateStore::Native => { + let mut first_error = None; + let certs = rustls_native_certs::load_native_certs().map_err(WsHandshakeError::CertificateStore)?; + for cert in certs { + let cert = rustls::Certificate(cert.0); + if let Err(err) = roots.add(&cert) { + first_error = first_error.or_else(|| Some(io::Error::new(io::ErrorKind::InvalidData, err))); + } + } + if roots.is_empty() { + let err = first_error + .unwrap_or_else(|| io::Error::new(io::ErrorKind::NotFound, "No valid certificate found")); + return Err(WsHandshakeError::CertificateStore(err)); + } + } + CertificateStore::WebPki => { + roots.add_server_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| { + rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(ta.subject, ta.spki, ta.name_constraints) + })); + } + _ => { + let err = io::Error::new(io::ErrorKind::NotFound, "Invalid certificate store"); + return Err(WsHandshakeError::CertificateStore(err)); + } + }; + + let config = + rustls::ClientConfig::builder().with_safe_defaults().with_root_certificates(roots).with_no_client_auth(); + + Ok(Arc::new(config).into()) +} + #[cfg(test)] mod tests { use super::{Mode, Target, Uri, WsHandshakeError}; From 092081a0a2b8904c6ebd2cd99e16c7bc13ffc3ae Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Mon, 1 Nov 2021 12:20:41 +0100 Subject: [PATCH 02/31] fix(ws server): `batch` wait until all methods has been executed. (#542) * reproduce Kian's issue * fix ws server wait until batches has completed * fix nit * clippify * enable benches for ws batch requests * use stream instead of futures::join_all * clippify * address grumbles: better assert --- benches/bench.rs | 6 +-- tests/Cargo.toml | 3 +- tests/tests/helpers.rs | 7 ++++ tests/tests/integration_tests.rs | 15 ++++++++ ws-server/src/server.rs | 64 +++++++++++++++++++------------- 5 files changed, 64 insertions(+), 31 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index 901cecd6cb..f69b7905ea 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -21,16 +21,14 @@ criterion_group!( SyncBencher::http_requests, SyncBencher::batched_http_requests, SyncBencher::websocket_requests, - // TODO: https://github.com/paritytech/jsonrpsee/issues/528 - // SyncBencher::batched_ws_requests, + SyncBencher::batched_ws_requests, ); criterion_group!( async_benches, AsyncBencher::http_requests, AsyncBencher::batched_http_requests, AsyncBencher::websocket_requests, - // TODO: https://github.com/paritytech/jsonrpsee/issues/528 - // AsyncBencher::batched_ws_requests + AsyncBencher::batched_ws_requests ); criterion_group!(subscriptions, AsyncBencher::subscriptions); criterion_main!(types_benches, sync_benches, async_benches, subscriptions); diff --git a/tests/Cargo.toml b/tests/Cargo.toml index e3f7682901..1a450dedf2 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,9 +8,10 @@ license = "MIT" publish = false [dev-dependencies] +env_logger = "0.8" beef = { version = "0.5.1", features = ["impl_serde"] } futures = { version = "0.3.14", default-features = false, features = ["std"] } jsonrpsee = { path = "../jsonrpsee", features = ["full"] } tokio = { version = "1", features = ["full"] } -serde_json = "1" tracing = "0.1" +serde_json = "1" diff --git a/tests/tests/helpers.rs b/tests/tests/helpers.rs index 4c2bafa764..2ee4db9011 100644 --- a/tests/tests/helpers.rs +++ b/tests/tests/helpers.rs @@ -98,6 +98,13 @@ pub async fn websocket_server() -> SocketAddr { let mut module = RpcModule::new(()); module.register_method("say_hello", |_, _| Ok("hello")).unwrap(); + module + .register_async_method("slow_hello", |_, _| async { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + Ok("hello") + }) + .unwrap(); + let addr = server.local_addr().unwrap(); server.start(module).unwrap(); diff --git a/tests/tests/integration_tests.rs b/tests/tests/integration_tests.rs index 2cc7570bd1..118c270a81 100644 --- a/tests/tests/integration_tests.rs +++ b/tests/tests/integration_tests.rs @@ -360,3 +360,18 @@ async fn ws_server_should_stop_subscription_after_client_drop() { // assert that the server received `SubscriptionClosed` after the client was dropped. assert!(matches!(rx.next().await.unwrap(), SubscriptionClosedError { .. })); } + +#[tokio::test] +async fn ws_batch_works() { + let server_addr = websocket_server().await; + let server_url = format!("ws://{}", server_addr); + let client = WsClientBuilder::default().build(&server_url).await.unwrap(); + + let mut batch = Vec::new(); + + batch.push(("say_hello", rpc_params![])); + batch.push(("slow_hello", rpc_params![])); + + let responses: Vec = client.batch_request(batch).await.unwrap(); + assert_eq!(responses, vec!["hello".to_string(), "hello".to_string()]); +} diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index 6a0abf25af..6fccc7b8a3 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -36,8 +36,9 @@ use crate::types::{ TEN_MB_SIZE_BYTES, }; use futures_channel::mpsc; +use futures_util::future::FutureExt; use futures_util::io::{BufReader, BufWriter}; -use futures_util::stream::StreamExt; +use futures_util::stream::{self, StreamExt}; use soketto::handshake::{server::Response, Server as SokettoServer}; use tokio::net::{TcpListener, TcpStream, ToSocketAddrs}; use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; @@ -296,34 +297,45 @@ async fn background_task( } } Some(b'[') => { - if let Ok(batch) = serde_json::from_slice::>(&data) { - if !batch.is_empty() { - // Batch responses must be sent back as a single message so we read the results from each - // request in the batch and read the results off of a new channel, `rx_batch`, and then send the - // complete batch response back to the client over `tx`. - let (tx_batch, mut rx_batch) = mpsc::unbounded::(); - - for fut in batch - .into_iter() - .filter_map(|req| methods.execute_with_resources(&tx_batch, req, conn_id, &resources)) - { - method_executors.add(fut); - } - - // Closes the receiving half of a channel without dropping it. This prevents any further - // messages from being sent on the channel. - rx_batch.close(); - let results = collect_batch_response(rx_batch).await; - if let Err(err) = tx.unbounded_send(results) { - tracing::error!("Error sending batch response to the client: {:?}", err) + // Make sure the following variables are not moved into async closure below. + let d = std::mem::take(&mut data); + let resources = &resources; + let methods = &methods; + let tx2 = tx.clone(); + + let fut = async move { + // Batch responses must be sent back as a single message so we read the results from each + // request in the batch and read the results off of a new channel, `rx_batch`, and then send the + // complete batch response back to the client over `tx`. + let (tx_batch, mut rx_batch) = mpsc::unbounded(); + if let Ok(batch) = serde_json::from_slice::>(&d) { + if !batch.is_empty() { + let methods_stream = + stream::iter(batch.into_iter().filter_map(|req| { + methods.execute_with_resources(&tx_batch, req, conn_id, resources) + })); + + let results = methods_stream + .for_each_concurrent(None, |item| item) + .then(|_| { + rx_batch.close(); + collect_batch_response(rx_batch) + }) + .await; + + if let Err(err) = tx2.unbounded_send(results) { + tracing::error!("Error sending batch response to the client: {:?}", err) + } + } else { + send_error(Id::Null, &tx2, ErrorCode::InvalidRequest.into()); } } else { - send_error(Id::Null, &tx, ErrorCode::InvalidRequest.into()); + let (id, code) = prepare_error(&d); + send_error(id, &tx2, code.into()); } - } else { - let (id, code) = prepare_error(&data); - send_error(id, &tx, code.into()); - } + }; + + method_executors.add(Box::pin(fut)); } _ => send_error(Id::Null, &tx, ErrorCode::ParseError.into()), } From ff3337b107bf29bef6067164c20c6a0b0b5bdc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C2=AF=5C=5F=28=E3=83=84=29=5F/=C2=AF?= <92186471+DefinitelyNotHilbert@users.noreply.github.com> Date: Wed, 3 Nov 2021 15:26:17 +0100 Subject: [PATCH 03/31] Proc mac support map param (#544) * feat(proc_macro): add support for map arguments * feat(proc_macro): formatting * feat(proc_macro): fix issues with Into trait * feat(proc_macro): param_format for methods * feat(proc_macro): improve param_format checking - Addressed @niklasad1's suggestion to use an Option instead of just defaulting to "array". * feat(proc_macro): apply suggestions, add test case - Use enum for param format. - Extract parsing logic into separate function. - Add ui test. * feat(proc_macro): run cargo fmt * feat(proc_macro): address suggestions * feat(proc_macro): document param_kind argument * feat(proc_macro): consistent spacing Apply @maciejhirsz formatting suggestion. Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * feat(proc_macro): apply suggestions - make parameter encoding DRY - remove strings from param_kind - return result from parse_param_kind * feat(proc_macro): formatting Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> --- proc-macros/src/attributes.rs | 17 ++++ proc-macros/src/lib.rs | 2 + proc-macros/src/render_client.rs | 84 +++++++++++++------ proc-macros/src/rpc_macro.rs | 18 ++-- proc-macros/tests/ui/correct/param_kind.rs | 63 ++++++++++++++ .../method/method_unexpected_field.stderr | 4 +- .../sub/sub_unsupported_field.stderr | 4 +- 7 files changed, 155 insertions(+), 37 deletions(-) create mode 100644 proc-macros/tests/ui/correct/param_kind.rs diff --git a/proc-macros/src/attributes.rs b/proc-macros/src/attributes.rs index c90ffd6141..ba28bf3718 100644 --- a/proc-macros/src/attributes.rs +++ b/proc-macros/src/attributes.rs @@ -40,6 +40,12 @@ pub(crate) struct Argument { pub tokens: TokenStream2, } +#[derive(Debug, Clone)] +pub enum ParamKind { + Array, + Map, +} + #[derive(Debug, Clone)] pub struct Resource { pub name: syn::LitStr, @@ -189,3 +195,14 @@ where { arg.ok().map(transform).transpose() } + +pub(crate) fn parse_param_kind(arg: Result) -> syn::Result { + let kind: Option = optional(arg, Argument::value)?; + + match kind { + None => Ok(ParamKind::Array), + Some(ident) if ident == "array" => Ok(ParamKind::Array), + Some(ident) if ident == "map" => Ok(ParamKind::Map), + ident => Err(Error::new(ident.span(), "param_kind must be either `map` or `array`")), + } +} diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index ad913d73db..b5f1346af5 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -164,6 +164,7 @@ pub(crate) mod visitor; /// - `name` (mandatory): name of the RPC method. Does not have to be the same as the Rust method name. /// - `aliases`: list of name aliases for the RPC method as a comma separated string. /// - `blocking`: when set method execution will always spawn on a dedicated thread. Only usable with non-`async` methods. +/// - `param_kind`: kind of structure to use for parameter passing. Can be "array" or "map", defaults to "array". /// /// **Method requirements:** /// @@ -180,6 +181,7 @@ pub(crate) mod visitor; /// - `name` (mandatory): name of the RPC method. Does not have to be the same as the Rust method name. /// - `unsub` (mandatory): name of the RPC method to unsubscribe from the subscription. Must not be the same as `name`. /// - `item` (mandatory): type of items yielded by the subscription. Note that it must be the type, not string. +/// - `param_kind`: kind of structure to use for parameter passing. Can be "array" or "map", defaults to "array". /// /// **Method requirements:** /// diff --git a/proc-macros/src/render_client.rs b/proc-macros/src/render_client.rs index 3cda7daa4f..11a12680bf 100644 --- a/proc-macros/src/render_client.rs +++ b/proc-macros/src/render_client.rs @@ -23,12 +23,12 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR // IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. - +use crate::attributes::ParamKind; use crate::helpers::generate_where_clause; use crate::rpc_macro::{RpcDescription, RpcMethod, RpcSubscription}; use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use syn::TypeParam; +use syn::{FnArg, Pat, PatIdent, PatType, TypeParam}; impl RpcDescription { pub(super) fn render_client(&self) -> Result { @@ -95,18 +95,7 @@ impl RpcDescription { }; // Encoded parameters for the request. - let parameters = if !method.params.is_empty() { - let serde_json = self.jrps_client_item(quote! { types::__reexports::serde_json }); - let params = method.params.iter().map(|(param, _param_type)| { - quote! { #serde_json::to_value(&#param)? } - }); - quote! { - Some(vec![ #(#params),* ].into()) - } - } else { - quote! { None } - }; - + let parameters = self.encode_params(&method.params, &method.param_kind, &method.signature); // Doc-comment to be associated with the method. let docs = &method.docs; @@ -138,18 +127,7 @@ impl RpcDescription { let returns = quote! { Result<#sub_type<#item>, #jrps_error> }; // Encoded parameters for the request. - let parameters = if !sub.params.is_empty() { - let serde_json = self.jrps_client_item(quote! { types::__reexports::serde_json }); - let params = sub.params.iter().map(|(param, _param_type)| { - quote! { #serde_json::to_value(&#param)? } - }); - quote! { - Some(vec![ #(#params),* ].into()) - } - } else { - quote! { None } - }; - + let parameters = self.encode_params(&sub.params, &sub.param_kind, &sub.signature); // Doc-comment to be associated with the method. let docs = &sub.docs; @@ -161,4 +139,58 @@ impl RpcDescription { }; Ok(method) } + + fn encode_params( + &self, + params: &Vec<(syn::PatIdent, syn::Type)>, + param_kind: &ParamKind, + signature: &syn::TraitItemMethod, + ) -> TokenStream2 { + if !params.is_empty() { + let serde_json = self.jrps_client_item(quote! { types::__reexports::serde_json }); + let params = params.iter().map(|(param, _param_type)| { + quote! { #serde_json::to_value(&#param)? } + }); + match param_kind { + ParamKind::Map => { + // Extract parameter names. + let param_names = extract_param_names(&signature.sig); + // Combine parameter names and values into tuples. + let params = param_names.iter().zip(params).map(|pair| { + let param = pair.0; + let value = pair.1; + quote! { (#param, #value) } + }); + quote! { + Some(types::v2::ParamsSer::Map( + std::collections::BTreeMap::<&str, #serde_json::Value>::from( + [#(#params),*] + ) + ) + ) + } + } + ParamKind::Array => { + quote! { + Some(vec![ #(#params),* ].into()) + } + } + } + } else { + quote! { None } + } + } +} + +fn extract_param_names(sig: &syn::Signature) -> Vec { + sig.inputs + .iter() + .filter_map(|param| match param { + FnArg::Typed(PatType { pat, .. }) => match &**pat { + Pat::Ident(PatIdent { ident, .. }) => Some(ident.to_string()), + _ => None, + }, + _ => None, + }) + .collect() } diff --git a/proc-macros/src/rpc_macro.rs b/proc-macros/src/rpc_macro.rs index 3f10f763cd..1ae2f5f05e 100644 --- a/proc-macros/src/rpc_macro.rs +++ b/proc-macros/src/rpc_macro.rs @@ -27,7 +27,7 @@ //! Declaration of the JSON RPC generator procedural macros. use crate::{ - attributes::{optional, Argument, AttributeMeta, MissingArgument, Resource}, + attributes::{optional, parse_param_kind, Argument, AttributeMeta, MissingArgument, ParamKind, Resource}, helpers::extract_doc_comments, }; @@ -42,6 +42,7 @@ pub struct RpcMethod { pub blocking: bool, pub docs: TokenStream2, pub params: Vec<(syn::PatIdent, syn::Type)>, + pub param_kind: ParamKind, pub returns: Option, pub signature: syn::TraitItemMethod, pub aliases: Vec, @@ -50,12 +51,13 @@ pub struct RpcMethod { impl RpcMethod { pub fn from_item(attr: Attribute, mut method: syn::TraitItemMethod) -> syn::Result { - let [aliases, blocking, name, resources] = - AttributeMeta::parse(attr)?.retain(["aliases", "blocking", "name", "resources"])?; + let [aliases, blocking, name, param_kind, resources] = + AttributeMeta::parse(attr)?.retain(["aliases", "blocking", "name", "param_kind", "resources"])?; let aliases = parse_aliases(aliases)?; let blocking = optional(blocking, Argument::flag)?.is_some(); let name = name?.string()?; + let param_kind = parse_param_kind(param_kind)?; let resources = optional(resources, Argument::group)?.unwrap_or_default(); let sig = method.sig.clone(); @@ -85,7 +87,7 @@ impl RpcMethod { // We've analyzed attributes and don't need them anymore. method.attrs.clear(); - Ok(Self { aliases, blocking, name, params, returns, signature: method, docs, resources }) + Ok(Self { aliases, blocking, name, params, param_kind, returns, signature: method, docs, resources }) } } @@ -95,6 +97,7 @@ pub struct RpcSubscription { pub docs: TokenStream2, pub unsubscribe: String, pub params: Vec<(syn::PatIdent, syn::Type)>, + pub param_kind: ParamKind, pub item: syn::Type, pub signature: syn::TraitItemMethod, pub aliases: Vec, @@ -103,12 +106,13 @@ pub struct RpcSubscription { impl RpcSubscription { pub fn from_item(attr: syn::Attribute, mut sub: syn::TraitItemMethod) -> syn::Result { - let [aliases, item, name, unsubscribe_aliases] = - AttributeMeta::parse(attr)?.retain(["aliases", "item", "name", "unsubscribe_aliases"])?; + let [aliases, item, name, param_kind, unsubscribe_aliases] = + AttributeMeta::parse(attr)?.retain(["aliases", "item", "name", "param_kind", "unsubscribe_aliases"])?; let aliases = parse_aliases(aliases)?; let name = name?.string()?; let item = item?.value()?; + let param_kind = parse_param_kind(param_kind)?; let unsubscribe_aliases = parse_aliases(unsubscribe_aliases)?; let sig = sub.sig.clone(); @@ -130,7 +134,7 @@ impl RpcSubscription { // We've analyzed attributes and don't need them anymore. sub.attrs.clear(); - Ok(Self { name, unsubscribe, unsubscribe_aliases, params, item, signature: sub, aliases, docs }) + Ok(Self { name, unsubscribe, unsubscribe_aliases, params, param_kind, item, signature: sub, aliases, docs }) } } diff --git a/proc-macros/tests/ui/correct/param_kind.rs b/proc-macros/tests/ui/correct/param_kind.rs new file mode 100644 index 0000000000..52c76ea7ac --- /dev/null +++ b/proc-macros/tests/ui/correct/param_kind.rs @@ -0,0 +1,63 @@ +use jsonrpsee::{ + proc_macros::rpc, + types::{async_trait, RpcResult}, + ws_client::*, + ws_server::WsServerBuilder, +}; + +use std::net::SocketAddr; + +#[rpc(client, server, namespace = "foo")] +pub trait Rpc { + #[method(name = "method_with_array_param", param_kind = array)] + async fn method_with_array_param(&self, param_a: u8, param_b: String) -> RpcResult; + + #[method(name="method_with_map_param", param_kind= map)] + async fn method_with_map_param(&self, param_a: u8, param_b: String) -> RpcResult; + + #[method(name="method_with_default_param")] + async fn method_with_default_param(&self, param_a: u8, param_b: String) -> RpcResult; +} + +pub struct RpcServerImpl; + +#[async_trait] +impl RpcServer for RpcServerImpl { + async fn method_with_array_param(&self, param_a: u8, param_b: String) -> RpcResult { + assert_eq!(param_a, 0); + assert_eq!(¶m_b, "a"); + Ok(42u16) + } + + async fn method_with_map_param(&self, param_a: u8, param_b: String) -> RpcResult { + assert_eq!(param_a, 0); + assert_eq!(¶m_b, "a"); + Ok(42u16) + } + + async fn method_with_default_param(&self, param_a: u8, param_b: String) -> RpcResult { + assert_eq!(param_a, 0); + assert_eq!(¶m_b, "a"); + Ok(42u16) + } +} + +pub async fn websocket_server() -> SocketAddr { + let server = WsServerBuilder::default().build("127.0.0.1:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + + server.start(RpcServerImpl.into_rpc()).unwrap(); + + addr +} + +#[tokio::main] +async fn main() { + let server_addr = websocket_server().await; + let server_url = format!("ws://{}", server_addr); + let client = WsClientBuilder::default().build(&server_url).await.unwrap(); + + assert_eq!(client.method_with_array_param(0, "a".into()).await.unwrap(), 42); + assert_eq!(client.method_with_map_param(0, "a".into()).await.unwrap(), 42); + assert_eq!(client.method_with_default_param(0, "a".into()).await.unwrap(), 42); +} diff --git a/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr b/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr index fccf3ba76c..81b031b034 100644 --- a/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr +++ b/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr @@ -1,5 +1,5 @@ -error: Unknown argument `magic`, expected one of: `aliases`, `blocking`, `name`, `resources` - --> $DIR/method_unexpected_field.rs:6:25 +error: Unknown argument `magic`, expected one of: `aliases`, `blocking`, `name`, `param_kind`, `resources` + --> tests/ui/incorrect/method/method_unexpected_field.rs:6:25 | 6 | #[method(name = "foo", magic = false)] | ^^^^^ diff --git a/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr b/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr index a6cced13f0..87e90136fe 100644 --- a/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr +++ b/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr @@ -1,5 +1,5 @@ -error: Unknown argument `magic`, expected one of: `aliases`, `item`, `name`, `unsubscribe_aliases` - --> $DIR/sub_unsupported_field.rs:6:42 +error: Unknown argument `magic`, expected one of: `aliases`, `item`, `name`, `param_kind`, `unsubscribe_aliases` + --> tests/ui/incorrect/sub/sub_unsupported_field.rs:6:42 | 6 | #[subscription(name = "sub", item = u8, magic = true)] | ^^^^^ From a8796c61940b03a352be957d8229f97c1d4e4cfb Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Fri, 5 Nov 2021 13:34:49 +0100 Subject: [PATCH 04/31] ci: run check on each feature individually (#552) * ci: test each feature individually * fix nit: --all-targets is an arg * fix rustdoc link * get rid of cargo hack test; too slow --- .github/workflows/ci.yml | 86 +++++----------------------------------- http-client/Cargo.toml | 2 +- utils/Cargo.toml | 2 +- utils/src/client.rs | 2 +- 4 files changed, 12 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 158ead18e6..fe6b1215f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: uses: actions-rs/cargo@v1.0.3 with: command: clippy + args: --all-targets - name: Check rustdoc links run: RUSTDOCFLAGS="--deny broken_intra_doc_links" cargo doc --verbose --workspace --no-deps --document-private-items @@ -59,68 +60,14 @@ jobs: toolchain: stable override: true + - name: Install cargo-hack + run: cargo install cargo-hack + - name: Rust Cache uses: Swatinem/rust-cache@v1.3.0 - - name: Cargo check all targets (use Cargo.toml in workspace) - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --all-targets - - - name: Cargo check HTTP client - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path http-client/Cargo.toml - - - name: Cargo check HTTP server - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path http-server/Cargo.toml - - - name: Cargo check WS client - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path ws-client/Cargo.toml - - - name: Cargo check WS server - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path ws-server/Cargo.toml - - - name: Cargo check types - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path types/Cargo.toml - - - name: Cargo check utils - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path utils/Cargo.toml - - - name: Cargo check proc macros - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path proc-macros/Cargo.toml - - - name: Cargo check test utils - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path test-utils/Cargo.toml - - - name: Cargo check examples - uses: actions-rs/cargo@v1.0.3 - with: - command: check - args: --manifest-path examples/Cargo.toml + - name: Cargo check all targets and features + run: cargo hack check --workspace --each-feature --all-targets tests: name: Run tests Ubuntu @@ -139,16 +86,11 @@ jobs: - name: Rust Cache uses: Swatinem/rust-cache@v1.3.0 - - name: Cargo build - uses: actions-rs/cargo@v1.0.3 - with: - command: build - args: --workspace - - name: Cargo test uses: actions-rs/cargo@v1.0.3 with: command: test + args: --workspace tests_macos: name: Run tests macos @@ -167,16 +109,11 @@ jobs: - name: Rust Cache uses: Swatinem/rust-cache@v1.3.0 - - name: Cargo build - uses: actions-rs/cargo@v1.0.3 - with: - command: build - args: --workspace - - name: Cargo test uses: actions-rs/cargo@v1.0.3 with: command: test + args: --workspace tests_windows: name: Run tests Windows @@ -195,13 +132,8 @@ jobs: - name: Rust Cache uses: Swatinem/rust-cache@v1.3.0 - - name: Cargo build - uses: actions-rs/cargo@v1.0.3 - with: - command: build - args: --workspace - - name: Cargo test uses: actions-rs/cargo@v1.0.3 with: command: test + args: --workspace diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index f2fbaf4420..44f0feee5a 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -15,7 +15,7 @@ fnv = "1" hyper = { version = "0.14.10", features = ["client", "http1", "http2", "tcp"] } hyper-rustls = { version = "0.22", features = ["webpki-tokio"] } jsonrpsee-types = { path = "../types", version = "0.4.1" } -jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["http-helpers"] } +jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["client", "http-helpers"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" thiserror = "1.0" diff --git a/utils/Cargo.toml b/utils/Cargo.toml index ebe250401a..25cebb8825 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -43,4 +43,4 @@ client = ["jsonrpsee-types"] [dev-dependencies] serde_json = "1.0" tokio = { version = "1", features = ["macros", "rt"] } -jsonrpsee = { path = "../jsonrpsee" } +jsonrpsee = { path = "../jsonrpsee", features = ["server", "macros"] } diff --git a/utils/src/client.rs b/utils/src/client.rs index e6d00150bd..e765472e6c 100644 --- a/utils/src/client.rs +++ b/utils/src/client.rs @@ -32,7 +32,7 @@ pub mod __reexports { } #[macro_export] -/// Convert the given values to a [`ParamsSer`] as expected by a jsonrpsee Client (http or websocket). +/// Convert the given values to a [`jsonrpsee_types::v2::ParamsSer`] as expected by a jsonrpsee Client (http or websocket). macro_rules! rpc_params { ($($param:expr),*) => { { From 32d29259acb644591592aaba0ded18649342d078 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Fri, 5 Nov 2021 17:15:22 +0100 Subject: [PATCH 05/31] clients: request ID as RAII guard (#543) * request ID as RAII guard * clippify * fmt * address grumbles: naming `RequestIdGuard` -> `RequestIdManager` `RequestId` -> `RequestIdGuard` --- http-client/src/client.rs | 51 ++++++++--------------- types/src/client.rs | 87 ++++++++++++++++++++++++--------------- ws-client/src/client.rs | 56 ++++++++----------------- 3 files changed, 89 insertions(+), 105 deletions(-) diff --git a/http-client/src/client.rs b/http-client/src/client.rs index d1bff8213b..dca85ce2e1 100644 --- a/http-client/src/client.rs +++ b/http-client/src/client.rs @@ -28,7 +28,7 @@ use crate::transport::HttpTransportClient; use crate::types::{ traits::Client, v2::{Id, NotificationSer, ParamsSer, RequestSer, Response, RpcError}, - CertificateStore, Error, RequestIdGuard, TEN_MB_SIZE_BYTES, + CertificateStore, Error, RequestIdManager, TEN_MB_SIZE_BYTES, }; use async_trait::async_trait; use fnv::FnvHashMap; @@ -75,7 +75,7 @@ impl HttpClientBuilder { .map_err(|e| Error::Transport(e.into()))?; Ok(HttpClient { transport, - id_guard: RequestIdGuard::new(self.max_concurrent_requests), + id_manager: RequestIdManager::new(self.max_concurrent_requests), request_timeout: self.request_timeout, }) } @@ -100,7 +100,7 @@ pub struct HttpClient { /// Request timeout. Defaults to 60sec. request_timeout: Duration, /// Request ID manager. - id_guard: RequestIdGuard, + id_manager: RequestIdManager, } #[async_trait] @@ -120,27 +120,20 @@ impl Client for HttpClient { where R: DeserializeOwned, { - // NOTE: the IDs wrap on overflow which is intended. - let id = self.id_guard.next_request_id()?; - let request = RequestSer::new(Id::Number(id), method, params); - - let fut = self.transport.send_and_read_body(serde_json::to_string(&request).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?); + let id = self.id_manager.next_request_id()?; + let request = RequestSer::new(Id::Number(*id.inner()), method, params); + + let fut = self.transport.send_and_read_body(serde_json::to_string(&request).map_err(Error::ParseError)?); let body = match tokio::time::timeout(self.request_timeout, fut).await { Ok(Ok(body)) => body, Err(_e) => { - self.id_guard.reclaim_request_id(); return Err(Error::RequestTimeout); } Ok(Err(e)) => { - self.id_guard.reclaim_request_id(); return Err(Error::Transport(e.into())); } }; - self.id_guard.reclaim_request_id(); let response: Response<_> = match serde_json::from_slice(&body) { Ok(response) => response, Err(_) => { @@ -151,7 +144,7 @@ impl Client for HttpClient { let response_id = response.id.as_number().copied().ok_or(Error::InvalidRequestId)?; - if response_id == id { + if response_id == *id.inner() { Ok(response.result) } else { Err(Error::InvalidRequestId) @@ -167,17 +160,14 @@ impl Client for HttpClient { let mut ordered_requests = Vec::with_capacity(batch.len()); let mut request_set = FnvHashMap::with_capacity_and_hasher(batch.len(), Default::default()); - let ids = self.id_guard.next_request_ids(batch.len())?; + let ids = self.id_manager.next_request_ids(batch.len())?; for (pos, (method, params)) in batch.into_iter().enumerate() { - batch_request.push(RequestSer::new(Id::Number(ids[pos]), method, params)); - ordered_requests.push(ids[pos]); - request_set.insert(ids[pos], pos); + batch_request.push(RequestSer::new(Id::Number(ids.inner()[pos]), method, params)); + ordered_requests.push(ids.inner()[pos]); + request_set.insert(ids.inner()[pos], pos); } - let fut = self.transport.send_and_read_body(serde_json::to_string(&batch_request).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?); + let fut = self.transport.send_and_read_body(serde_json::to_string(&batch_request).map_err(Error::ParseError)?); let body = match tokio::time::timeout(self.request_timeout, fut).await { Ok(Ok(body)) => body, @@ -185,16 +175,11 @@ impl Client for HttpClient { Ok(Err(e)) => return Err(Error::Transport(e.into())), }; - let rps: Vec> = match serde_json::from_slice(&body) { - Ok(response) => response, - Err(_) => { - let err: RpcError = serde_json::from_slice(&body).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?; - return Err(Error::Request(err.to_string())); - } - }; + let rps: Vec> = + serde_json::from_slice(&body).map_err(|_| match serde_json::from_slice::(&body) { + Ok(e) => Error::Request(e.to_string()), + Err(e) => Error::ParseError(e), + })?; // NOTE: `R::default` is placeholder and will be replaced in loop below. let mut responses = vec![R::default(); ordered_requests.len()]; diff --git a/types/src/client.rs b/types/src/client.rs index 3f45387417..c0a0cf0685 100644 --- a/types/src/client.rs +++ b/types/src/client.rs @@ -30,7 +30,8 @@ use futures_channel::{mpsc, oneshot}; use futures_util::{future::FutureExt, sink::SinkExt, stream::StreamExt}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::Value as JsonValue; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; /// Subscription kind #[derive(Debug)] @@ -188,65 +189,83 @@ impl Drop for Subscription { #[derive(Debug)] /// Keep track of request IDs. -pub struct RequestIdGuard { +pub struct RequestIdManager { // Current pending requests. - current_pending: AtomicUsize, + current_pending: Arc<()>, /// Max concurrent pending requests allowed. max_concurrent_requests: usize, /// Get the next request ID. current_id: AtomicU64, } -impl RequestIdGuard { +impl RequestIdManager { /// Create a new `RequestIdGuard` with the provided concurrency limit. pub fn new(limit: usize) -> Self { - Self { current_pending: AtomicUsize::new(0), max_concurrent_requests: limit, current_id: AtomicU64::new(0) } + Self { current_pending: Arc::new(()), max_concurrent_requests: limit, current_id: AtomicU64::new(0) } } - fn get_slot(&self) -> Result<(), Error> { - self.current_pending - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |val| { - if val >= self.max_concurrent_requests { - None - } else { - Some(val + 1) - } - }) - .map(|_| ()) - .map_err(|_| Error::MaxSlotsExceeded) + fn get_slot(&self) -> Result, Error> { + // Strong count is 1 at start, so that's why we use `>` and not `>=`. + if Arc::strong_count(&self.current_pending) > self.max_concurrent_requests { + Err(Error::MaxSlotsExceeded) + } else { + Ok(self.current_pending.clone()) + } } /// Attempts to get the next request ID. /// /// Fails if request limit has been exceeded. - pub fn next_request_id(&self) -> Result { - self.get_slot()?; + pub fn next_request_id(&self) -> Result, Error> { + let rc = self.get_slot()?; let id = self.current_id.fetch_add(1, Ordering::SeqCst); - Ok(id) + Ok(RequestIdGuard { _rc: rc, id }) } /// Attempts to get the `n` number next IDs that only counts as one request. /// /// Fails if request limit has been exceeded. - pub fn next_request_ids(&self, len: usize) -> Result, Error> { - self.get_slot()?; - let mut batch = Vec::with_capacity(len); + pub fn next_request_ids(&self, len: usize) -> Result>, Error> { + let rc = self.get_slot()?; + let mut ids = Vec::with_capacity(len); for _ in 0..len { - batch.push(self.current_id.fetch_add(1, Ordering::SeqCst)); + ids.push(self.current_id.fetch_add(1, Ordering::SeqCst)); } - Ok(batch) + Ok(RequestIdGuard { _rc: rc, id: ids }) + } +} + +/// Reference counted request ID. +#[derive(Debug)] +pub struct RequestIdGuard { + id: T, + /// Reference count decreased when dropped. + _rc: Arc<()>, +} + +impl RequestIdGuard { + /// Get the actual ID. + pub fn inner(&self) -> &T { + &self.id } +} + +#[cfg(test)] +mod tests { + use super::RequestIdManager; + + #[test] + fn request_id_guard_works() { + let manager = RequestIdManager::new(2); + let _first = manager.next_request_id().unwrap(); + + { + let _second = manager.next_request_ids(13).unwrap(); + assert!(manager.next_request_id().is_err()); + // second dropped here. + } - /// Decrease the currently pending counter by one (saturated at 0). - pub fn reclaim_request_id(&self) { - // NOTE we ignore the error here, since we are simply saturating at 0 - let _ = self.current_pending.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |val| { - if val > 0 { - Some(val - 1) - } else { - None - } - }); + assert!(manager.next_request_id().is_ok()); } } diff --git a/ws-client/src/client.rs b/ws-client/src/client.rs index 962e54d643..aa7e90d0db 100644 --- a/ws-client/src/client.rs +++ b/ws-client/src/client.rs @@ -28,7 +28,7 @@ use crate::transport::{Receiver as WsReceiver, Sender as WsSender, WsHandshakeEr use crate::types::{ traits::{Client, SubscriptionClient}, v2::{Id, Notification, NotificationSer, ParamsSer, RequestSer, Response, RpcError, SubscriptionResponse}, - BatchMessage, CertificateStore, Error, FrontToBack, RegisterNotificationMessage, RequestIdGuard, RequestMessage, + BatchMessage, CertificateStore, Error, FrontToBack, RegisterNotificationMessage, RequestIdManager, RequestMessage, Subscription, SubscriptionKind, SubscriptionMessage, TEN_MB_SIZE_BYTES, }; use crate::{ @@ -98,7 +98,7 @@ pub struct WsClient { /// Request timeout. Defaults to 60sec. request_timeout: Duration, /// Request ID manager. - id_guard: RequestIdGuard, + id_manager: RequestIdManager, } /// Builder for [`WsClient`]. @@ -242,7 +242,7 @@ impl<'a> WsClientBuilder<'a> { to_back, request_timeout, error: Mutex::new(ErrorFromBack::Unread(err_rx)), - id_guard: RequestIdGuard::new(max_concurrent_requests), + id_manager: RequestIdManager::new(max_concurrent_requests), }) } } @@ -273,12 +273,9 @@ impl Drop for WsClient { impl Client for WsClient { async fn notification<'a>(&self, method: &'a str, params: Option>) -> Result<(), Error> { // NOTE: we use this to guard against max number of concurrent requests. - let _req_id = self.id_guard.next_request_id()?; + let _req_id = self.id_manager.next_request_id()?; let notif = NotificationSer::new(method, params); - let raw = serde_json::to_string(¬if).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?; + let raw = serde_json::to_string(¬if).map_err(Error::ParseError)?; tracing::trace!("[frontend]: send notification: {:?}", raw); let mut sender = self.to_back.clone(); @@ -291,7 +288,6 @@ impl Client for WsClient { _ = timeout => return Err(Error::RequestTimeout) }; - self.id_guard.reclaim_request_id(); match res { Ok(()) => Ok(()), Err(_) => Err(self.read_error_from_backend().await), @@ -303,27 +299,22 @@ impl Client for WsClient { R: DeserializeOwned, { let (send_back_tx, send_back_rx) = oneshot::channel(); - let req_id = self.id_guard.next_request_id()?; - let raw = serde_json::to_string(&RequestSer::new(Id::Number(req_id), method, params)).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?; + let req_id = self.id_manager.next_request_id()?; + let id = *req_id.inner(); + let raw = serde_json::to_string(&RequestSer::new(Id::Number(id), method, params)).map_err(Error::ParseError)?; tracing::trace!("[frontend]: send request: {:?}", raw); if self .to_back .clone() - .send(FrontToBack::Request(RequestMessage { raw, id: req_id, send_back: Some(send_back_tx) })) + .send(FrontToBack::Request(RequestMessage { raw, id, send_back: Some(send_back_tx) })) .await .is_err() { - self.id_guard.reclaim_request_id(); return Err(self.read_error_from_backend().await); } let res = call_with_timeout(self.request_timeout, send_back_rx).await; - - self.id_guard.reclaim_request_id(); let json_value = match res { Ok(Ok(v)) => v, Ok(Err(err)) => return Err(err), @@ -336,34 +327,28 @@ impl Client for WsClient { where R: DeserializeOwned + Default + Clone, { - let batch_ids = self.id_guard.next_request_ids(batch.len())?; + let batch_ids = self.id_manager.next_request_ids(batch.len())?; let mut batches = Vec::with_capacity(batch.len()); for (idx, (method, params)) in batch.into_iter().enumerate() { - batches.push(RequestSer::new(Id::Number(batch_ids[idx]), method, params)); + batches.push(RequestSer::new(Id::Number(batch_ids.inner()[idx]), method, params)); } let (send_back_tx, send_back_rx) = oneshot::channel(); - let raw = serde_json::to_string(&batches).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?; + let raw = serde_json::to_string(&batches).map_err(Error::ParseError)?; tracing::trace!("[frontend]: send batch request: {:?}", raw); if self .to_back .clone() - .send(FrontToBack::Batch(BatchMessage { raw, ids: batch_ids, send_back: send_back_tx })) + .send(FrontToBack::Batch(BatchMessage { raw, ids: batch_ids.inner().clone(), send_back: send_back_tx })) .await .is_err() { - self.id_guard.reclaim_request_id(); return Err(self.read_error_from_backend().await); } let res = call_with_timeout(self.request_timeout, send_back_rx).await; - - self.id_guard.reclaim_request_id(); let json_values = match res { Ok(Ok(v)) => v, Ok(Err(err)) => return Err(err), @@ -397,12 +382,9 @@ impl SubscriptionClient for WsClient { return Err(Error::SubscriptionNameConflict(unsubscribe_method.to_owned())); } - let ids = self.id_guard.next_request_ids(2)?; - let raw = - serde_json::to_string(&RequestSer::new(Id::Number(ids[0]), subscribe_method, params)).map_err(|e| { - self.id_guard.reclaim_request_id(); - Error::ParseError(e) - })?; + let ids = self.id_manager.next_request_ids(2)?; + let raw = serde_json::to_string(&RequestSer::new(Id::Number(ids.inner()[0]), subscribe_method, params)) + .map_err(Error::ParseError)?; let (send_back_tx, send_back_rx) = oneshot::channel(); if self @@ -410,21 +392,19 @@ impl SubscriptionClient for WsClient { .clone() .send(FrontToBack::Subscribe(SubscriptionMessage { raw, - subscribe_id: ids[0], - unsubscribe_id: ids[1], + subscribe_id: ids.inner()[0], + unsubscribe_id: ids.inner()[1], unsubscribe_method: unsubscribe_method.to_owned(), send_back: send_back_tx, })) .await .is_err() { - self.id_guard.reclaim_request_id(); return Err(self.read_error_from_backend().await); } let res = call_with_timeout(self.request_timeout, send_back_rx).await; - self.id_guard.reclaim_request_id(); let (notifs_rx, id) = match res { Ok(Ok(val)) => val, Ok(Err(err)) => return Err(err), From afcf411d9bbf1fce95caacab8b5e66857880064b Mon Sep 17 00:00:00 2001 From: Chris Sosnin <48099298+slumber@users.noreply.github.com> Date: Mon, 8 Nov 2021 18:57:06 +0300 Subject: [PATCH 06/31] Allow awaiting on server handles (#550) * Implement Future for server handles * Explicitly assert timeout errors in tests --- benches/helpers.rs | 4 ++-- examples/http.rs | 8 ++++---- examples/proc_macro.rs | 4 ++-- http-server/src/lib.rs | 2 +- http-server/src/server.rs | 33 ++++++++++++++++++++++--------- http-server/src/tests.rs | 34 ++++++++++++++++++++++++++------ tests/tests/helpers.rs | 12 +++++------ tests/tests/resource_limiting.rs | 32 +++++++++++++++--------------- ws-server/src/future.rs | 21 +++++++++++++++----- ws-server/src/lib.rs | 2 +- ws-server/src/server.rs | 8 ++++---- ws-server/src/tests.rs | 31 ++++++++++++++++++++++------- 12 files changed, 128 insertions(+), 63 deletions(-) diff --git a/benches/helpers.rs b/benches/helpers.rs index 8161c33b63..a8d64a4b12 100644 --- a/benches/helpers.rs +++ b/benches/helpers.rs @@ -70,7 +70,7 @@ pub async fn ws_server(handle: tokio::runtime::Handle) -> (String, jsonrpc_ws_se /// Run jsonrpsee HTTP server for benchmarks. #[cfg(not(feature = "jsonrpc-crate"))] -pub async fn http_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee::http_server::HttpStopHandle) { +pub async fn http_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee::http_server::HttpServerHandle) { use jsonrpsee::http_server::{HttpServerBuilder, RpcModule}; let server = HttpServerBuilder::default() @@ -88,7 +88,7 @@ pub async fn http_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee:: /// Run jsonrpsee WebSocket server for benchmarks. #[cfg(not(feature = "jsonrpc-crate"))] -pub async fn ws_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee::ws_server::WsStopHandle) { +pub async fn ws_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee::ws_server::WsServerHandle) { use jsonrpsee::ws_server::{RpcModule, WsServerBuilder}; let server = WsServerBuilder::default() diff --git a/examples/http.rs b/examples/http.rs index 7bc735cf49..80bdab93f9 100644 --- a/examples/http.rs +++ b/examples/http.rs @@ -26,7 +26,7 @@ use jsonrpsee::{ http_client::HttpClientBuilder, - http_server::{HttpServerBuilder, HttpStopHandle, RpcModule}, + http_server::{HttpServerBuilder, HttpServerHandle, RpcModule}, rpc_params, types::traits::Client, }; @@ -49,12 +49,12 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -async fn run_server() -> anyhow::Result<(SocketAddr, HttpStopHandle)> { +async fn run_server() -> anyhow::Result<(SocketAddr, HttpServerHandle)> { let server = HttpServerBuilder::default().build("127.0.0.1:0".parse()?)?; let mut module = RpcModule::new(()); module.register_method("say_hello", |_, _| Ok("lo"))?; let addr = server.local_addr()?; - let stop_handle = server.start(module)?; - Ok((addr, stop_handle)) + let server_handle = server.start(module)?; + Ok((addr, server_handle)) } diff --git a/examples/proc_macro.rs b/examples/proc_macro.rs index ee12c89016..c5449b9c24 100644 --- a/examples/proc_macro.rs +++ b/examples/proc_macro.rs @@ -28,7 +28,7 @@ use jsonrpsee::{ proc_macros::rpc, types::{async_trait, error::Error, Subscription}, ws_client::WsClientBuilder, - ws_server::{SubscriptionSink, WsServerBuilder, WsStopHandle}, + ws_server::{SubscriptionSink, WsServerBuilder, WsServerHandle}, }; use std::net::SocketAddr; @@ -89,7 +89,7 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -async fn run_server() -> anyhow::Result<(SocketAddr, WsStopHandle)> { +async fn run_server() -> anyhow::Result<(SocketAddr, WsServerHandle)> { let server = WsServerBuilder::default().build("127.0.0.1:0").await?; let addr = server.local_addr()?; diff --git a/http-server/src/lib.rs b/http-server/src/lib.rs index c164ff5231..9418c1b44c 100644 --- a/http-server/src/lib.rs +++ b/http-server/src/lib.rs @@ -43,7 +43,7 @@ pub use access_control::{ }; pub use jsonrpsee_types as types; pub use jsonrpsee_utils::server::rpc_module::RpcModule; -pub use server::{Builder as HttpServerBuilder, Server as HttpServer, StopHandle as HttpStopHandle}; +pub use server::{Builder as HttpServerBuilder, Server as HttpServer, ServerHandle as HttpServerHandle}; #[cfg(test)] mod tests; diff --git a/http-server/src/server.rs b/http-server/src/server.rs index 276d075ee9..19ed870d98 100644 --- a/http-server/src/server.rs +++ b/http-server/src/server.rs @@ -26,8 +26,7 @@ use crate::{response, AccessControl}; use futures_channel::mpsc; -use futures_util::future::join_all; -use futures_util::stream::StreamExt; +use futures_util::{future::join_all, stream::StreamExt, FutureExt}; use hyper::{ server::{conn::AddrIncoming, Builder as HyperBuilder}, service::{make_service_fn, service_fn}, @@ -49,7 +48,10 @@ use serde_json::value::RawValue; use socket2::{Domain, Socket, Type}; use std::{ cmp, + future::Future, net::{SocketAddr, TcpListener}, + pin::Pin, + task::{Context, Poll}, }; /// Builder to create JSON-RPC HTTP server. @@ -143,17 +145,17 @@ impl Default for Builder { } } -/// Handle used to stop the running server. +/// Handle used to run or stop the server. #[derive(Debug)] -pub struct StopHandle { +pub struct ServerHandle { stop_sender: mpsc::Sender<()>, - stop_handle: Option>, + pub(crate) handle: Option>, } -impl StopHandle { +impl ServerHandle { /// Requests server to stop. Returns an error if server was already stopped. pub fn stop(mut self) -> Result, Error> { - let stop = self.stop_sender.try_send(()).map(|_| self.stop_handle.take()); + let stop = self.stop_sender.try_send(()).map(|_| self.handle.take()); match stop { Ok(Some(handle)) => Ok(handle), _ => Err(Error::AlreadyStopped), @@ -161,6 +163,19 @@ impl StopHandle { } } +impl Future for ServerHandle { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let handle = match &mut self.handle { + Some(handle) => handle, + None => return Poll::Ready(()), + }; + + handle.poll_unpin(cx).map(|_| ()) + } +} + /// An HTTP JSON RPC server. #[derive(Debug)] pub struct Server { @@ -185,7 +200,7 @@ impl Server { } /// Start the server. - pub fn start(mut self, methods: impl Into) -> Result { + pub fn start(mut self, methods: impl Into) -> Result { let max_request_body_size = self.max_request_body_size; let access_control = self.access_control; let (tx, mut rx) = mpsc::channel(1); @@ -298,7 +313,7 @@ impl Server { let _ = server.with_graceful_shutdown(async move { rx.next().await.map_or((), |_| ()) }).await; }); - Ok(StopHandle { stop_handle: Some(handle), stop_sender: tx }) + Ok(ServerHandle { handle: Some(handle), stop_sender: tx }) } } diff --git a/http-server/src/tests.rs b/http-server/src/tests.rs index ee79cfb931..4e3bf7cf44 100644 --- a/http-server/src/tests.rs +++ b/http-server/src/tests.rs @@ -27,16 +27,17 @@ #![cfg(test)] use std::net::SocketAddr; +use std::time::Duration; use crate::types::error::{CallError, Error}; -use crate::{server::StopHandle, HttpServerBuilder, RpcModule}; +use crate::{server::ServerHandle, HttpServerBuilder, RpcModule}; use jsonrpsee_test_utils::helpers::*; use jsonrpsee_test_utils::mocks::{Id, StatusCode, TestContext}; use jsonrpsee_test_utils::TimeoutFutureExt; use serde_json::Value as JsonValue; -async fn server() -> (SocketAddr, StopHandle) { +async fn server() -> (SocketAddr, ServerHandle) { let server = HttpServerBuilder::default().build("127.0.0.1:0".parse().unwrap()).unwrap(); let ctx = TestContext; let mut module = RpcModule::new(ctx); @@ -78,8 +79,8 @@ async fn server() -> (SocketAddr, StopHandle) { }) .unwrap(); - let stop_handle = server.start(module).unwrap(); - (addr, stop_handle) + let server_handle = server.start(module).unwrap(); + (addr, server_handle) } #[tokio::test] @@ -380,6 +381,27 @@ async fn can_register_modules() { #[tokio::test] async fn stop_works() { let _ = env_logger::try_init(); - let (_addr, stop_handle) = server().with_default_timeout().await.unwrap(); - assert!(matches!(stop_handle.stop().unwrap().await, Ok(_))); + let (_addr, server_handle) = server().with_default_timeout().await.unwrap(); + assert!(matches!(server_handle.stop().unwrap().await, Ok(_))); +} + +#[tokio::test] +async fn run_forever() { + const TIMEOUT: Duration = Duration::from_millis(200); + + let _ = env_logger::try_init(); + let (_addr, server_handle) = server().with_default_timeout().await.unwrap(); + + assert!(matches!(server_handle.with_timeout(TIMEOUT).await, Err(_timeout_err))); + + let (_addr, server_handle) = server().await; + server_handle.handle.as_ref().unwrap().abort(); + + // Cancelled task is still considered to be finished without errors. + // A subject to change. + server_handle.with_timeout(TIMEOUT).await.unwrap(); + + let (_addr, mut server_handle) = server().with_default_timeout().await.unwrap(); + server_handle.handle.take(); + server_handle.with_timeout(TIMEOUT).await.unwrap(); } diff --git a/tests/tests/helpers.rs b/tests/tests/helpers.rs index 2ee4db9011..f33086e445 100644 --- a/tests/tests/helpers.rs +++ b/tests/tests/helpers.rs @@ -25,15 +25,15 @@ // DEALINGS IN THE SOFTWARE. use jsonrpsee::{ - http_server::{HttpServerBuilder, HttpStopHandle}, + http_server::{HttpServerBuilder, HttpServerHandle}, types::Error, - ws_server::{WsServerBuilder, WsStopHandle}, + ws_server::{WsServerBuilder, WsServerHandle}, RpcModule, }; use std::net::SocketAddr; use std::time::Duration; -pub async fn websocket_server_with_subscription() -> (SocketAddr, WsStopHandle) { +pub async fn websocket_server_with_subscription() -> (SocketAddr, WsServerHandle) { let server = WsServerBuilder::default().build("127.0.0.1:0").await.unwrap(); let mut module = RpcModule::new(()); @@ -88,9 +88,9 @@ pub async fn websocket_server_with_subscription() -> (SocketAddr, WsStopHandle) .unwrap(); let addr = server.local_addr().unwrap(); - let stop_handle = server.start(module).unwrap(); + let server_handle = server.start(module).unwrap(); - (addr, stop_handle) + (addr, server_handle) } pub async fn websocket_server() -> SocketAddr { @@ -112,7 +112,7 @@ pub async fn websocket_server() -> SocketAddr { addr } -pub async fn http_server() -> (SocketAddr, HttpStopHandle) { +pub async fn http_server() -> (SocketAddr, HttpServerHandle) { let server = HttpServerBuilder::default().build("127.0.0.1:0".parse().unwrap()).unwrap(); let mut module = RpcModule::new(()); let addr = server.local_addr().unwrap(); diff --git a/tests/tests/resource_limiting.rs b/tests/tests/resource_limiting.rs index 332d14233e..736de5123d 100644 --- a/tests/tests/resource_limiting.rs +++ b/tests/tests/resource_limiting.rs @@ -26,11 +26,11 @@ use jsonrpsee::{ http_client::HttpClientBuilder, - http_server::{HttpServerBuilder, HttpStopHandle}, + http_server::{HttpServerBuilder, HttpServerHandle}, proc_macros::rpc, types::{traits::Client, Error}, ws_client::WsClientBuilder, - ws_server::{WsServerBuilder, WsStopHandle}, + ws_server::{WsServerBuilder, WsServerHandle}, RpcModule, }; use tokio::time::sleep; @@ -91,7 +91,7 @@ fn module_macro() -> RpcModule<()> { ().into_rpc() } -async fn websocket_server(module: RpcModule<()>) -> Result<(SocketAddr, WsStopHandle), Error> { +async fn websocket_server(module: RpcModule<()>) -> Result<(SocketAddr, WsServerHandle), Error> { let server = WsServerBuilder::default() .register_resource("CPU", 6, 2)? .register_resource("MEM", 10, 1)? @@ -104,7 +104,7 @@ async fn websocket_server(module: RpcModule<()>) -> Result<(SocketAddr, WsStopHa Ok((addr, handle)) } -async fn http_server(module: RpcModule<()>) -> Result<(SocketAddr, HttpStopHandle), Error> { +async fn http_server(module: RpcModule<()>) -> Result<(SocketAddr, HttpServerHandle), Error> { let server = HttpServerBuilder::default() .register_resource("CPU", 6, 2)? .register_resource("MEM", 10, 1)? @@ -128,7 +128,7 @@ fn assert_server_busy(fail: Result) { } } -async fn run_tests_on_ws_server(server_addr: SocketAddr, stop_handle: WsStopHandle) { +async fn run_tests_on_ws_server(server_addr: SocketAddr, server_handle: WsServerHandle) { let server_url = format!("ws://{}", server_addr); let client = WsClientBuilder::default().build(&server_url).await.unwrap(); @@ -162,10 +162,10 @@ async fn run_tests_on_ws_server(server_addr: SocketAddr, stop_handle: WsStopHand // Client being active prevents the server from shutting down?! drop(client); - stop_handle.stop().unwrap().await; + server_handle.stop().unwrap().await; } -async fn run_tests_on_http_server(server_addr: SocketAddr, stop_handle: HttpStopHandle) { +async fn run_tests_on_http_server(server_addr: SocketAddr, server_handle: HttpServerHandle) { let server_url = format!("http://{}", server_addr); let client = HttpClientBuilder::default().build(&server_url).unwrap(); @@ -190,33 +190,33 @@ async fn run_tests_on_http_server(server_addr: SocketAddr, stop_handle: HttpStop assert_eq!(passes, 3); - stop_handle.stop().unwrap().await.unwrap(); + server_handle.stop().unwrap().await.unwrap(); } #[tokio::test] async fn ws_server_with_manual_module() { - let (server_addr, stop_handle) = websocket_server(module_manual().unwrap()).await.unwrap(); + let (server_addr, server_handle) = websocket_server(module_manual().unwrap()).await.unwrap(); - run_tests_on_ws_server(server_addr, stop_handle).await; + run_tests_on_ws_server(server_addr, server_handle).await; } #[tokio::test] async fn ws_server_with_macro_module() { - let (server_addr, stop_handle) = websocket_server(module_macro()).await.unwrap(); + let (server_addr, server_handle) = websocket_server(module_macro()).await.unwrap(); - run_tests_on_ws_server(server_addr, stop_handle).await; + run_tests_on_ws_server(server_addr, server_handle).await; } #[tokio::test] async fn http_server_with_manual_module() { - let (server_addr, stop_handle) = http_server(module_manual().unwrap()).await.unwrap(); + let (server_addr, server_handle) = http_server(module_manual().unwrap()).await.unwrap(); - run_tests_on_http_server(server_addr, stop_handle).await; + run_tests_on_http_server(server_addr, server_handle).await; } #[tokio::test] async fn http_server_with_macro_module() { - let (server_addr, stop_handle) = http_server(module_macro()).await.unwrap(); + let (server_addr, server_handle) = http_server(module_macro()).await.unwrap(); - run_tests_on_http_server(server_addr, stop_handle).await; + run_tests_on_http_server(server_addr, server_handle).await; } diff --git a/ws-server/src/future.rs b/ws-server/src/future.rs index be90bb23cc..69b35b6977 100644 --- a/ws-server/src/future.rs +++ b/ws-server/src/future.rs @@ -166,16 +166,17 @@ impl StopMonitor { self.0.shutdown_requested.load(Ordering::Relaxed) } - pub(crate) fn handle(&self) -> StopHandle { - StopHandle(Arc::downgrade(&self.0)) + pub(crate) fn handle(&self) -> ServerHandle { + ServerHandle(Arc::downgrade(&self.0)) } } -/// Handle that is able to stop the running server. +/// Handle that is able to stop the running server or wait for it to finish +/// its execution. #[derive(Debug, Clone)] -pub struct StopHandle(Weak); +pub struct ServerHandle(Weak); -impl StopHandle { +impl ServerHandle { /// Requests server to stop. Returns an error if server was already stopped. /// /// Returns a future that can be awaited for when the server shuts down. @@ -190,6 +191,16 @@ impl StopHandle { } } +impl Future for ServerHandle { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut shutdown_waiter = ShutdownWaiter(self.0.clone()); + + shutdown_waiter.poll_unpin(cx) + } +} + /// A `Future` that resolves once the server has stopped. #[derive(Debug)] pub struct ShutdownWaiter(Weak); diff --git a/ws-server/src/lib.rs b/ws-server/src/lib.rs index c734a6fc94..27e46129c7 100644 --- a/ws-server/src/lib.rs +++ b/ws-server/src/lib.rs @@ -38,7 +38,7 @@ mod server; #[cfg(test)] mod tests; -pub use future::{ShutdownWaiter as WsShutdownWaiter, StopHandle as WsStopHandle}; +pub use future::{ServerHandle as WsServerHandle, ShutdownWaiter as WsShutdownWaiter}; pub use jsonrpsee_types as types; pub use jsonrpsee_utils::server::rpc_module::{RpcModule, SubscriptionSink}; pub use server::{Builder as WsServerBuilder, Server as WsServer}; diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index 6fccc7b8a3..3eb1a0cfe4 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -29,7 +29,7 @@ use std::net::SocketAddr; use std::pin::Pin; use std::task::{Context, Poll}; -use crate::future::{FutureDriver, StopHandle, StopMonitor}; +use crate::future::{FutureDriver, ServerHandle, StopMonitor}; use crate::types::{ error::Error, v2::{ErrorCode, Id, Request}, @@ -66,14 +66,14 @@ impl Server { } /// Returns the handle to stop the running server. - pub fn stop_handle(&self) -> StopHandle { + pub fn server_handle(&self) -> ServerHandle { self.stop_monitor.handle() } /// Start responding to connections requests. This will run on the tokio runtime until the server is stopped. - pub fn start(mut self, methods: impl Into) -> Result { + pub fn start(mut self, methods: impl Into) -> Result { let methods = methods.into().initialize_resources(&self.resources)?; - let handle = self.stop_handle(); + let handle = self.server_handle(); match self.cfg.tokio_runtime.take() { Some(rt) => rt.spawn(self.start_inner(methods)), diff --git a/ws-server/src/tests.rs b/ws-server/src/tests.rs index 020ae7b61a..e0c1236412 100644 --- a/ws-server/src/tests.rs +++ b/ws-server/src/tests.rs @@ -27,14 +27,16 @@ #![cfg(test)] use crate::types::error::{CallError, Error}; -use crate::{future::StopHandle, RpcModule, WsServerBuilder}; +use crate::{future::ServerHandle, RpcModule, WsServerBuilder}; use anyhow::anyhow; +use futures_util::future::join; use jsonrpsee_test_utils::helpers::*; use jsonrpsee_test_utils::mocks::{Id, TestContext, WebSocketTestClient, WebSocketTestError}; use jsonrpsee_test_utils::TimeoutFutureExt; use serde_json::Value as JsonValue; use std::fmt; use std::net::SocketAddr; +use std::time::Duration; /// Applications can/should provide their own error. #[derive(Debug)] @@ -58,7 +60,7 @@ async fn server() -> SocketAddr { /// other: `invalid_params` (always returns `CallError::InvalidParams`), `call_fail` (always returns /// `CallError::Failed`), `sleep_for` Returns the address together with handles for server future /// and server stop. -async fn server_with_handles() -> (SocketAddr, StopHandle) { +async fn server_with_handles() -> (SocketAddr, ServerHandle) { let server = WsServerBuilder::default().build("127.0.0.1:0").with_default_timeout().await.unwrap().unwrap(); let mut module = RpcModule::new(()); module @@ -105,8 +107,8 @@ async fn server_with_handles() -> (SocketAddr, StopHandle) { let addr = server.local_addr().unwrap(); - let stop_handle = server.start(module).unwrap(); - (addr, stop_handle) + let server_handle = server.start(module).unwrap(); + (addr, server_handle) } /// Run server with user provided context. @@ -538,12 +540,27 @@ async fn can_register_modules() { #[tokio::test] async fn stop_works() { let _ = env_logger::try_init(); - let (_addr, stop_handle) = server_with_handles().with_default_timeout().await.unwrap(); - stop_handle.clone().stop().unwrap().with_default_timeout().await.unwrap(); + let (_addr, server_handle) = server_with_handles().with_default_timeout().await.unwrap(); + server_handle.clone().stop().unwrap().with_default_timeout().await.unwrap(); // After that we should be able to wait for task handle to finish. // First `unwrap` is timeout, second is `JoinHandle`'s one. // After server was stopped, attempt to stop it again should result in an error. - assert!(matches!(stop_handle.stop(), Err(Error::AlreadyStopped))); + assert!(matches!(server_handle.stop(), Err(Error::AlreadyStopped))); +} + +#[tokio::test] +async fn run_forever() { + const TIMEOUT: Duration = Duration::from_millis(200); + + let _ = env_logger::try_init(); + let (_addr, server_handle) = server_with_handles().with_default_timeout().await.unwrap(); + + assert!(matches!(server_handle.with_timeout(TIMEOUT).await, Err(_timeout_err))); + + let (_addr, server_handle) = server_with_handles().with_default_timeout().await.unwrap(); + + // Send the shutdown request from one handle and await the server on the second one. + join(server_handle.clone().stop().unwrap(), server_handle).with_timeout(TIMEOUT).await.unwrap(); } From 13b2a0bf46114ad12e283373ef0a08bfdfc5dbce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Nov 2021 13:13:37 +0000 Subject: [PATCH 07/31] Bump actions/checkout from 2.3.5 to 2.4.0 (#548) Bumps [actions/checkout](https://github.com/actions/checkout) from 2.3.5 to 2.4.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.3.5...v2.4.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/benchmarks.yml | 2 +- .github/workflows/ci.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f2f6faa9b1..8778b0ca5a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout Sources - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2.4.0 - name: Install Rust nightly toolchain uses: actions-rs/toolchain@v1.0.7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe6b1215f1..1348c40111 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1.0.7 @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1.0.7 @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2.4.0 - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1.0.7 @@ -97,7 +97,7 @@ jobs: runs-on: macos-latest steps: - name: Checkout sources - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2.4.0 - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1.0.7 @@ -120,7 +120,7 @@ jobs: runs-on: windows-latest steps: - name: Checkout sources - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2.4.0 - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1.0.7 From 945d62e6f16abc480d66247c660c72e0f7d7e9a2 Mon Sep 17 00:00:00 2001 From: Tarik Gul <47201679+TarikGul@users.noreply.github.com> Date: Tue, 9 Nov 2021 08:18:29 -0500 Subject: [PATCH 08/31] docs(release): improve release checklist (#514) * docs(release): improve release checklist * Update RELEASE-CHECKLIST.md Co-authored-by: David * Update RELEASE-CHECKLIST.md Co-authored-by: David * Update RELEASE-CHECKLIST.md Co-authored-by: Niklas Adolfsson Co-authored-by: David Co-authored-by: Niklas Adolfsson --- RELEASE-CHECKLIST.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASE-CHECKLIST.md b/RELEASE-CHECKLIST.md index 4704253108..2644b0160e 100644 --- a/RELEASE-CHECKLIST.md +++ b/RELEASE-CHECKLIST.md @@ -5,4 +5,6 @@ 1. In the `CHANGELOG.md` file, move everything under "Unreleased" to a new section named `## [vx.y.z] – YYYY-MM-DD` 1. Make a dryrun like so: 1. Ensure you're in the project root dir - 1. Run: `./scripts/publish.sh --dry-run --allow-dirty` + Note: the script will publish the crates in the correct order and pause after each crate to ensure it's available at the crates registry before proceeding. This means the dry run isn't as useful and will end up in an infinite loop. If you're really unsure about the changes and want to do a dry run you should do a `cargo publish --dry-run` for each individual crate. +1. Publish: `./scripts/publish.sh` +1. Once published, make sure to "create a release" for the pushed tag on github. From 6dac20da11305f59707280fa5acf8b46de014316 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Tue, 9 Nov 2021 15:57:30 +0100 Subject: [PATCH 09/31] ws server: respect max limit for received messages (#537) * ws server: don't kill connection max limit exceeds * Update ws-server/src/server.rs * actually use max size in soketto * rewrite me * improve logs * use soketto fix * rewrite me * fix nit * revert unintentional change * use soketto 0.7.1 * fix logger * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * fix build Co-authored-by: David --- examples/http.rs | 7 +++-- examples/proc_macro.rs | 7 +++-- examples/ws.rs | 7 +++-- examples/ws_sub_with_params.rs | 7 +++-- examples/ws_subscription.rs | 7 +++-- test-utils/Cargo.toml | 2 +- types/Cargo.toml | 2 +- ws-client/Cargo.toml | 2 +- ws-server/Cargo.toml | 4 +-- ws-server/src/server.rs | 52 +++++++++++++++++++++++++--------- ws-server/src/tests.rs | 17 +++++++---- 11 files changed, 74 insertions(+), 40 deletions(-) diff --git a/examples/http.rs b/examples/http.rs index 80bdab93f9..30d12393e7 100644 --- a/examples/http.rs +++ b/examples/http.rs @@ -34,9 +34,10 @@ use std::net::SocketAddr; #[tokio::main] async fn main() -> anyhow::Result<()> { - // init tracing `FmtSubscriber`. - let subscriber = tracing_subscriber::FmtSubscriber::new(); - tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); let (server_addr, _handle) = run_server().await?; let url = format!("http://{}", server_addr); diff --git a/examples/proc_macro.rs b/examples/proc_macro.rs index c5449b9c24..11825834b3 100644 --- a/examples/proc_macro.rs +++ b/examples/proc_macro.rs @@ -72,9 +72,10 @@ impl RpcServer for RpcServerImpl { #[tokio::main] async fn main() -> anyhow::Result<()> { - // init tracing `FmtSubscriber`. - let subscriber = tracing_subscriber::FmtSubscriber::builder().finish(); - tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); let (server_addr, _handle) = run_server().await?; let url = format!("ws://{}", server_addr); diff --git a/examples/ws.rs b/examples/ws.rs index 6768c52b83..641a5cff5d 100644 --- a/examples/ws.rs +++ b/examples/ws.rs @@ -33,9 +33,10 @@ use std::net::SocketAddr; #[tokio::main] async fn main() -> anyhow::Result<()> { - // init tracing `FmtSubscriber`. - let subscriber = tracing_subscriber::FmtSubscriber::builder().finish(); - tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); let addr = run_server().await?; let url = format!("ws://{}", addr); diff --git a/examples/ws_sub_with_params.rs b/examples/ws_sub_with_params.rs index d6720204ed..3c3c61c3d1 100644 --- a/examples/ws_sub_with_params.rs +++ b/examples/ws_sub_with_params.rs @@ -34,9 +34,10 @@ use std::net::SocketAddr; #[tokio::main] async fn main() -> anyhow::Result<()> { - // init tracing `FmtSubscriber`. - let subscriber = tracing_subscriber::FmtSubscriber::builder().finish(); - tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); let addr = run_server().await?; let url = format!("ws://{}", addr); diff --git a/examples/ws_subscription.rs b/examples/ws_subscription.rs index 02308a7004..f9521992dc 100644 --- a/examples/ws_subscription.rs +++ b/examples/ws_subscription.rs @@ -36,9 +36,10 @@ const NUM_SUBSCRIPTION_RESPONSES: usize = 5; #[tokio::main] async fn main() -> anyhow::Result<()> { - // init tracing `FmtSubscriber`. - let subscriber = tracing_subscriber::FmtSubscriber::builder().finish(); - tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); let addr = run_server().await?; let url = format!("ws://{}", addr); diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 74ceae0d51..bf9ff16001 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -15,6 +15,6 @@ hyper = { version = "0.14.10", features = ["full"] } tracing = "0.1" serde = { version = "1", default-features = false, features = ["derive"] } serde_json = "1" -soketto = { version = "0.7", features = ["http"] } +soketto = { version = "0.7.1", features = ["http"] } tokio = { version = "1", features = ["net", "rt-multi-thread", "macros", "time"] } tokio-util = { version = "0.6", features = ["compat"] } diff --git a/types/Cargo.toml b/types/Cargo.toml index cc488e43e9..9b4fffcae6 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -19,5 +19,5 @@ tracing = { version = "0.1", default-features = false } serde = { version = "1", default-features = false, features = ["derive"] } serde_json = { version = "1", default-features = false, features = ["alloc", "raw_value", "std"] } thiserror = "1.0" -soketto = "0.7" +soketto = "0.7.1" hyper = "0.14.10" diff --git a/ws-client/Cargo.toml b/ws-client/Cargo.toml index b6943e5a47..b528b140b0 100644 --- a/ws-client/Cargo.toml +++ b/ws-client/Cargo.toml @@ -19,7 +19,7 @@ pin-project = "1" rustls-native-certs = "0.6.0" serde = "1" serde_json = "1" -soketto = "0.7" +soketto = "0.7.1" thiserror = "1" tokio = { version = "1", features = ["net", "time", "rt-multi-thread", "macros"] } tokio-rustls = "0.23" diff --git a/ws-server/Cargo.toml b/ws-server/Cargo.toml index 68652d1437..377eaf1d01 100644 --- a/ws-server/Cargo.toml +++ b/ws-server/Cargo.toml @@ -16,12 +16,12 @@ jsonrpsee-types = { path = "../types", version = "0.4.1" } jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["server"] } tracing = "0.1" serde_json = { version = "1", features = ["raw_value"] } -soketto = "0.7" +soketto = "0.7.1" tokio = { version = "1", features = ["net", "rt-multi-thread", "macros"] } tokio-util = { version = "0.6", features = ["compat"] } [dev-dependencies] anyhow = "1" -env_logger = "0.9" jsonrpsee-test-utils = { path = "../test-utils" } jsonrpsee = { path = "../jsonrpsee", features = ["full"] } +tracing-subscriber = "0.2.25" diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index 3eb1a0cfe4..1356ccad06 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -39,6 +39,7 @@ use futures_channel::mpsc; use futures_util::future::FutureExt; use futures_util::io::{BufReader, BufWriter}; use futures_util::stream::{self, StreamExt}; +use soketto::connection::Error as SokettoError; use soketto::handshake::{server::Response, Server as SokettoServer}; use tokio::net::{TcpListener, TcpStream, ToSocketAddrs}; use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; @@ -195,6 +196,7 @@ async fn handshake(socket: tokio::net::TcpStream, mode: HandshakeResponse<'_>) - Ok(()) } HandshakeResponse::Accept { conn_id, methods, resources, cfg, stop_monitor } => { + tracing::debug!("Accepting new connection: {}", conn_id); let key = { let req = server.receive_request().await?; let host_check = cfg.allowed_hosts.verify("Host", Some(req.headers().host)); @@ -243,7 +245,9 @@ async fn background_task( stop_server: StopMonitor, ) -> Result<(), Error> { // And we can finally transition to a websocket background_task. - let (mut sender, mut receiver) = server.into_builder().finish(); + let mut builder = server.into_builder(); + builder.set_max_message_size(max_request_body_size as usize); + let (mut sender, mut receiver) = builder.finish(); let (tx, mut rx) = mpsc::unbounded::(); let stop_server2 = stop_server.clone(); @@ -252,8 +256,10 @@ async fn background_task( while !stop_server2.shutdown_requested() { match rx.next().await { Some(response) => { - tracing::debug!("send: {}", response); - let _ = sender.send_text(response).await; + // TODO: check length of response https://github.com/paritytech/jsonrpsee/issues/536 + tracing::debug!("send {} bytes", response.len()); + tracing::trace!("send: {}", response); + let _ = sender.send_text_owned(response).await; let _ = sender.flush().await; } None => break, @@ -272,22 +278,38 @@ async fn background_task( while !stop_server.shutdown_requested() { data.clear(); - if let Err(e) = method_executors.select_with(receiver.receive_data(&mut data)).await { - tracing::error!("Could not receive WS data: {:?}; closing connection", e); - tx.close_channel(); - return Err(e.into()); - } + if let Err(err) = method_executors.select_with(receiver.receive_data(&mut data)).await { + match err { + SokettoError::Closed => { + tracing::debug!("Remote peer terminated the connection: {}", conn_id); + tx.close_channel(); + return Ok(()); + } + SokettoError::MessageTooLarge { current, maximum } => { + tracing::warn!( + "WS transport error: message is too big error ({} bytes, max is {})", + current, + maximum + ); + send_error(Id::Null, &tx, ErrorCode::OversizedRequest.into()); + continue; + } + // These errors can not be gracefully handled, so just log them and terminate the connection. + err => { + tracing::error!("WS transport error: {:?} => terminating connection {}", err, conn_id); + tx.close_channel(); + return Err(err.into()); + } + }; + }; - if data.len() > max_request_body_size as usize { - tracing::warn!("Request is too big ({} bytes, max is {})", data.len(), max_request_body_size); - send_error(Id::Null, &tx, ErrorCode::OversizedRequest.into()); - continue; - } + tracing::debug!("recv {} bytes", data.len()); match data.get(0) { Some(b'{') => { if let Ok(req) = serde_json::from_slice::(&data) { - tracing::debug!("recv: {:?}", req); + tracing::debug!("recv method call={}", req.method); + tracing::trace!("recv: req={:?}", req); if let Some(fut) = methods.execute_with_resources(&tx, req, conn_id, &resources) { method_executors.add(fut); } @@ -309,6 +331,8 @@ async fn background_task( // complete batch response back to the client over `tx`. let (tx_batch, mut rx_batch) = mpsc::unbounded(); if let Ok(batch) = serde_json::from_slice::>(&d) { + tracing::debug!("recv batch len={}", batch.len()); + tracing::trace!("recv: batch={:?}", batch); if !batch.is_empty() { let methods_stream = stream::iter(batch.into_iter().filter_map(|req| { diff --git a/ws-server/src/tests.rs b/ws-server/src/tests.rs index e0c1236412..20bd65e67c 100644 --- a/ws-server/src/tests.rs +++ b/ws-server/src/tests.rs @@ -34,9 +34,12 @@ use jsonrpsee_test_utils::helpers::*; use jsonrpsee_test_utils::mocks::{Id, TestContext, WebSocketTestClient, WebSocketTestError}; use jsonrpsee_test_utils::TimeoutFutureExt; use serde_json::Value as JsonValue; -use std::fmt; -use std::net::SocketAddr; -use std::time::Duration; +use std::{fmt, net::SocketAddr, time::Duration}; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; + +fn init_logger() { + let _ = FmtSubscriber::builder().with_env_filter(EnvFilter::from_default_env()).try_init(); +} /// Applications can/should provide their own error. #[derive(Debug)] @@ -156,6 +159,8 @@ async fn server_with_context() -> SocketAddr { #[tokio::test] async fn can_set_the_max_request_body_size() { + init_logger(); + let addr = "127.0.0.1:0"; // Rejects all requests larger than 10 bytes let server = WsServerBuilder::default().max_request_body_size(10).build(addr).await.unwrap(); @@ -225,6 +230,7 @@ async fn single_method_calls_works() { #[tokio::test] async fn async_method_calls_works() { + init_logger(); let addr = server().await; let mut client = WebSocketTestClient::new(addr).await.unwrap(); @@ -342,7 +348,6 @@ async fn single_method_call_with_params_works() { #[tokio::test] async fn single_method_call_with_faulty_params_returns_err() { - let _ = env_logger::try_init(); let addr = server().await; let mut client = WebSocketTestClient::new(addr).with_default_timeout().await.unwrap().unwrap(); let expected = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"invalid type: string \"should be a number\", expected u64 at line 1 column 21"},"id":1}"#; @@ -539,7 +544,7 @@ async fn can_register_modules() { #[tokio::test] async fn stop_works() { - let _ = env_logger::try_init(); + init_logger(); let (_addr, server_handle) = server_with_handles().with_default_timeout().await.unwrap(); server_handle.clone().stop().unwrap().with_default_timeout().await.unwrap(); @@ -554,7 +559,7 @@ async fn stop_works() { async fn run_forever() { const TIMEOUT: Duration = Duration::from_millis(200); - let _ = env_logger::try_init(); + init_logger(); let (_addr, server_handle) = server_with_handles().with_default_timeout().await.unwrap(); assert!(matches!(server_handle.with_timeout(TIMEOUT).await, Err(_timeout_err))); From f9b99ad6f29d9ed3e4e7cbd96db0ade3a50e135f Mon Sep 17 00:00:00 2001 From: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Date: Wed, 10 Nov 2021 14:55:28 +0100 Subject: [PATCH 10/31] Re-export tracing for macros (#555) --- http-server/src/lib.rs | 1 + jsonrpsee/src/lib.rs | 6 ++++++ proc-macros/Cargo.toml | 1 - proc-macros/src/render_server.rs | 5 +++-- ws-server/src/lib.rs | 1 + 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/http-server/src/lib.rs b/http-server/src/lib.rs index 9418c1b44c..81530cd814 100644 --- a/http-server/src/lib.rs +++ b/http-server/src/lib.rs @@ -44,6 +44,7 @@ pub use access_control::{ pub use jsonrpsee_types as types; pub use jsonrpsee_utils::server::rpc_module::RpcModule; pub use server::{Builder as HttpServerBuilder, Server as HttpServer, ServerHandle as HttpServerHandle}; +pub use tracing; #[cfg(test)] mod tests; diff --git a/jsonrpsee/src/lib.rs b/jsonrpsee/src/lib.rs index 413457c7a6..4fc8cbddc5 100644 --- a/jsonrpsee/src/lib.rs +++ b/jsonrpsee/src/lib.rs @@ -75,3 +75,9 @@ pub use jsonrpsee_types as types; /// Set of RPC methods that can be mounted to the server. #[cfg(any(feature = "http-server", feature = "ws-server"))] pub use jsonrpsee_utils::server::rpc_module::{RpcModule, SubscriptionSink}; + +#[cfg(feature = "http-server")] +pub use http_server::tracing; + +#[cfg(all(feature = "ws-server", not(feature = "http-server")))] +pub use ws_server::tracing; diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml index 6e209c7108..05bf1e5787 100644 --- a/proc-macros/Cargo.toml +++ b/proc-macros/Cargo.toml @@ -17,7 +17,6 @@ proc-macro2 = "1.0" quote = "1.0" syn = { version = "1.0", default-features = false, features = ["extra-traits", "full", "visit", "parsing"] } proc-macro-crate = "1" -tracing = "0.1" [dev-dependencies] jsonrpsee = { path = "../jsonrpsee", features = ["full"] } diff --git a/proc-macros/src/render_server.rs b/proc-macros/src/render_server.rs index 0e1bf71cbe..962aa4bb00 100644 --- a/proc-macros/src/render_server.rs +++ b/proc-macros/src/render_server.rs @@ -285,6 +285,7 @@ impl RpcDescription { let params_fields_seq = params.iter().map(|(name, _)| name); let params_fields = quote! { #(#params_fields_seq),* }; + let tracing = self.jrps_server_item(quote! { tracing }); // Code to decode sequence of parameters from a JSON array. let decode_array = { @@ -294,7 +295,7 @@ impl RpcDescription { let #name: #ty = match seq.optional_next() { Ok(v) => v, Err(e) => { - tracing::error!(concat!("Error parsing optional \"", stringify!(#name), "\" as \"", stringify!(#ty), "\": {:?}"), e); + #tracing::error!(concat!("Error parsing optional \"", stringify!(#name), "\" as \"", stringify!(#ty), "\": {:?}"), e); return Err(e.into()) } }; @@ -304,7 +305,7 @@ impl RpcDescription { let #name: #ty = match seq.next() { Ok(v) => v, Err(e) => { - tracing::error!(concat!("Error parsing \"", stringify!(#name), "\" as \"", stringify!(#ty), "\": {:?}"), e); + #tracing::error!(concat!("Error parsing \"", stringify!(#name), "\" as \"", stringify!(#ty), "\": {:?}"), e); return Err(e.into()) } }; diff --git a/ws-server/src/lib.rs b/ws-server/src/lib.rs index 27e46129c7..d2c02a9870 100644 --- a/ws-server/src/lib.rs +++ b/ws-server/src/lib.rs @@ -42,3 +42,4 @@ pub use future::{ServerHandle as WsServerHandle, ShutdownWaiter as WsShutdownWai pub use jsonrpsee_types as types; pub use jsonrpsee_utils::server::rpc_module::{RpcModule, SubscriptionSink}; pub use server::{Builder as WsServerBuilder, Server as WsServer}; +pub use tracing; From ac33d20629fa369d0989dfe1d7c88021646c2770 Mon Sep 17 00:00:00 2001 From: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Date: Thu, 11 Nov 2021 15:10:55 +0100 Subject: [PATCH 11/31] Array syntax aliases (#557) * Array syntax aliases * fmt --- proc-macros/src/attributes.rs | 16 +++++++++++++ proc-macros/src/render_server.rs | 9 +++---- proc-macros/src/rpc_macro.rs | 24 +++++++------------ .../ui/correct/alias_doesnt_use_namespace.rs | 4 ++-- proc-macros/tests/ui/correct/basic.rs | 4 ++-- .../tests/ui/correct/parse_angle_brackets.rs | 6 ++--- .../ui/incorrect/rpc/rpc_conflicting_alias.rs | 2 +- .../ui/incorrect/sub/sub_conflicting_alias.rs | 2 +- tests/tests/proc_macros.rs | 2 +- 9 files changed, 38 insertions(+), 31 deletions(-) diff --git a/proc-macros/src/attributes.rs b/proc-macros/src/attributes.rs index ba28bf3718..02da827efa 100644 --- a/proc-macros/src/attributes.rs +++ b/proc-macros/src/attributes.rs @@ -53,6 +53,10 @@ pub struct Resource { pub value: syn::LitInt, } +pub struct Aliases { + pub list: Punctuated, +} + impl Parse for Argument { fn parse(input: ParseStream) -> syn::Result { let label = input.parse()?; @@ -87,6 +91,18 @@ impl Parse for Resource { } } +impl Parse for Aliases { + fn parse(input: ParseStream) -> syn::Result { + let content; + + syn::bracketed!(content in input); + + let list = content.parse_terminated(Parse::parse)?; + + Ok(Aliases { list }) + } +} + fn parenthesized(input: ParseStream) -> syn::Result> { let content; diff --git a/proc-macros/src/render_server.rs b/proc-macros/src/render_server.rs index 962aa4bb00..86b96cb757 100644 --- a/proc-macros/src/render_server.rs +++ b/proc-macros/src/render_server.rs @@ -205,8 +205,7 @@ impl RpcDescription { .aliases .iter() .map(|alias| { - let alias = alias.trim().to_string(); - check_name(&alias, rust_method_name.span()); + check_name(alias, rust_method_name.span()); handle_register_result(quote! { rpc.register_alias(#alias, #rpc_name) }) @@ -229,8 +228,7 @@ impl RpcDescription { .aliases .iter() .map(|alias| { - let alias = alias.trim().to_string(); - check_name(&alias, rust_method_name.span()); + check_name(alias, rust_method_name.span()); handle_register_result(quote! { rpc.register_alias(#alias, #sub_name) }) @@ -240,8 +238,7 @@ impl RpcDescription { .unsubscribe_aliases .iter() .map(|alias| { - let alias = alias.trim().to_string(); - check_name(&alias, rust_method_name.span()); + check_name(alias, rust_method_name.span()); handle_register_result(quote! { rpc.register_alias(#alias, #unsub_name) }) diff --git a/proc-macros/src/rpc_macro.rs b/proc-macros/src/rpc_macro.rs index 1ae2f5f05e..be251cae00 100644 --- a/proc-macros/src/rpc_macro.rs +++ b/proc-macros/src/rpc_macro.rs @@ -27,12 +27,13 @@ //! Declaration of the JSON RPC generator procedural macros. use crate::{ - attributes::{optional, parse_param_kind, Argument, AttributeMeta, MissingArgument, ParamKind, Resource}, + attributes::{optional, parse_param_kind, Aliases, Argument, AttributeMeta, MissingArgument, ParamKind, Resource}, helpers::extract_doc_comments, }; use proc_macro2::TokenStream as TokenStream2; use quote::quote; +use std::borrow::Cow; use syn::spanned::Spanned; use syn::{punctuated::Punctuated, Attribute, Token}; @@ -276,32 +277,25 @@ impl RpcDescription { /// Examples: /// For namespace `foo` and method `makeSpam`, result will be `foo_makeSpam`. /// For no namespace and method `makeSpam` it will be just `makeSpam. - pub(crate) fn rpc_identifier(&self, method: &str) -> String { + pub(crate) fn rpc_identifier<'a>(&self, method: &'a str) -> Cow<'a, str> { if let Some(ns) = &self.namespace { - format!("{}_{}", ns, method.trim()) + format!("{}_{}", ns, method).into() } else { - method.to_string() + Cow::Borrowed(method) } } } fn parse_aliases(arg: Result) -> syn::Result> { - let aliases = optional(arg, Argument::string)?; + let aliases = optional(arg, Argument::value::)?; - Ok(aliases.map(|a| a.split(',').map(Into::into).collect()).unwrap_or_default()) + Ok(aliases.map(|a| a.list.into_iter().map(|lit| lit.value()).collect()).unwrap_or_default()) } fn find_attr<'a>(attrs: &'a [Attribute], ident: &str) -> Option<&'a Attribute> { attrs.iter().find(|a| a.path.is_ident(ident)) } -fn build_unsubscribe_method(existing_method: &str) -> String { - let method = existing_method.trim(); - let mut new_method = String::from("unsubscribe"); - if method.starts_with("subscribe") { - new_method.extend(method.chars().skip(9)); - } else { - new_method.push_str(method); - } - new_method +fn build_unsubscribe_method(method: &str) -> String { + format!("unsubscribe{}", method.strip_prefix("subscribe").unwrap_or(method)) } diff --git a/proc-macros/tests/ui/correct/alias_doesnt_use_namespace.rs b/proc-macros/tests/ui/correct/alias_doesnt_use_namespace.rs index 5e4c203001..5ba32df63e 100644 --- a/proc-macros/tests/ui/correct/alias_doesnt_use_namespace.rs +++ b/proc-macros/tests/ui/correct/alias_doesnt_use_namespace.rs @@ -3,10 +3,10 @@ use jsonrpsee::{proc_macros::rpc, types::RpcResult}; #[rpc(client, server, namespace = "myapi")] pub trait Rpc { /// Alias doesn't use the namespace so not duplicated. - #[method(name = "getTemp", aliases = "getTemp")] + #[method(name = "getTemp", aliases = ["getTemp"])] async fn async_method(&self, param_a: u8, param_b: String) -> RpcResult; - #[subscription(name = "getFood", item = String, aliases = "getFood", unsubscribe_aliases = "unsubscribegetFood")] + #[subscription(name = "getFood", item = String, aliases = ["getFood"], unsubscribe_aliases = ["unsubscribegetFood"])] fn sub(&self) -> RpcResult<()>; } diff --git a/proc-macros/tests/ui/correct/basic.rs b/proc-macros/tests/ui/correct/basic.rs index 07a6e99a40..c8052ce288 100644 --- a/proc-macros/tests/ui/correct/basic.rs +++ b/proc-macros/tests/ui/correct/basic.rs @@ -11,7 +11,7 @@ use std::net::SocketAddr; #[rpc(client, server, namespace = "foo")] pub trait Rpc { - #[method(name = "foo", aliases = "fooAlias, Other")] + #[method(name = "foo", aliases = ["fooAlias", "Other"])] async fn async_method(&self, param_a: u8, param_b: String) -> RpcResult; #[method(name = "optional_params")] @@ -29,7 +29,7 @@ pub trait Rpc { #[subscription(name = "sub", item = String)] fn sub(&self) -> RpcResult<()>; - #[subscription(name = "echo", aliases = "ECHO", item = u32, unsubscribe_aliases = "NotInterested, listenNoMore")] + #[subscription(name = "echo", aliases = ["ECHO"], item = u32, unsubscribe_aliases = ["NotInterested", "listenNoMore"])] fn sub_with_params(&self, val: u32) -> RpcResult<()>; } diff --git a/proc-macros/tests/ui/correct/parse_angle_brackets.rs b/proc-macros/tests/ui/correct/parse_angle_brackets.rs index 783f8df756..037e86c155 100644 --- a/proc-macros/tests/ui/correct/parse_angle_brackets.rs +++ b/proc-macros/tests/ui/correct/parse_angle_brackets.rs @@ -5,12 +5,12 @@ fn main() { pub trait Rpc { #[subscription( name = "submitAndWatchExtrinsic", - aliases = "author_extrinsicUpdate", - unsubscribe_aliases = "author_unwatchExtrinsic", + aliases = ["author_extrinsicUpdate"], + unsubscribe_aliases = ["author_unwatchExtrinsic"], // Arguments are being parsed the nearest comma, // angle braces need to be accounted for manually. item = TransactionStatus, )] fn dummy_subscription(&self) -> RpcResult<()>; } -} \ No newline at end of file +} diff --git a/proc-macros/tests/ui/incorrect/rpc/rpc_conflicting_alias.rs b/proc-macros/tests/ui/incorrect/rpc/rpc_conflicting_alias.rs index 00ed6b9b9e..3ba041a144 100644 --- a/proc-macros/tests/ui/incorrect/rpc/rpc_conflicting_alias.rs +++ b/proc-macros/tests/ui/incorrect/rpc/rpc_conflicting_alias.rs @@ -2,7 +2,7 @@ use jsonrpsee::proc_macros::rpc; use jsonrpsee::types::RpcResult; #[rpc(client, server)] pub trait DuplicatedAlias { - #[method(name = "foo", aliases = "foo_dup, foo_dup")] + #[method(name = "foo", aliases = ["foo_dup", "foo_dup"])] async fn async_method(&self) -> RpcResult; } diff --git a/proc-macros/tests/ui/incorrect/sub/sub_conflicting_alias.rs b/proc-macros/tests/ui/incorrect/sub/sub_conflicting_alias.rs index 595c792bff..993063c51a 100644 --- a/proc-macros/tests/ui/incorrect/sub/sub_conflicting_alias.rs +++ b/proc-macros/tests/ui/incorrect/sub/sub_conflicting_alias.rs @@ -2,7 +2,7 @@ use jsonrpsee::{proc_macros::rpc, types::RpcResult}; #[rpc(client, server)] pub trait DuplicatedSubAlias { - #[subscription(name = "alias", item = String, aliases = "hello_is_goodbye", unsubscribe_aliases = "hello_is_goodbye")] + #[subscription(name = "alias", item = String, aliases = ["hello_is_goodbye"], unsubscribe_aliases = ["hello_is_goodbye"])] fn async_method(&self) -> RpcResult<()>; } diff --git a/tests/tests/proc_macros.rs b/tests/tests/proc_macros.rs index 1fdbb9d4ab..84f9a313ec 100644 --- a/tests/tests/proc_macros.rs +++ b/tests/tests/proc_macros.rs @@ -49,7 +49,7 @@ mod rpc_impl { #[subscription(name = "sub", item = String)] fn sub(&self) -> RpcResult<()>; - #[subscription(name = "echo", aliases = "alias_echo", item = u32)] + #[subscription(name = "echo", aliases = ["alias_echo"], item = u32)] fn sub_with_params(&self, val: u32) -> RpcResult<()>; #[method(name = "params")] From 682ecbe39edd7e9c94ff2506329727a91846be24 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Thu, 11 Nov 2021 15:41:07 +0100 Subject: [PATCH 12/31] ws server: reject too big response (#553) * ws server: don't kill connection max limit exceeds * Update ws-server/src/server.rs * actually use max size in soketto * rewrite me * improve logs * use soketto fix * rewrite me * fix nit * revert unintentional change * use soketto 0.7.1 * fix logger * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * Update ws-server/src/server.rs Co-authored-by: David * fix build * reject too large response * fix some DRY code * feat: bounded serializer for RpcModule * Update utils/src/server/helpers.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * Update utils/src/server/helpers.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * address grumbles: safety and other nits * address grumbles: MaxCallSize on closures instead * Update utils/src/server/helpers.rs Co-authored-by: David * use max response size on errors too * Revert "use max response size on errors too" This reverts commit 3b07e42d257b2eebae311b92b7f72594d94d5f87. * include max limit in error response Co-authored-by: David Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> --- http-server/src/server.rs | 12 ++--- test-utils/src/helpers.rs | 8 +++ types/src/v2/error.rs | 4 ++ utils/src/server/helpers.rs | 94 ++++++++++++++++++++++++++++++++-- utils/src/server/rpc_module.rs | 52 +++++++++++-------- ws-server/src/server.rs | 38 ++++++++++---- ws-server/src/tests.rs | 12 ++--- 7 files changed, 173 insertions(+), 47 deletions(-) diff --git a/http-server/src/server.rs b/http-server/src/server.rs index 19ed870d98..44e2bfcd45 100644 --- a/http-server/src/server.rs +++ b/http-server/src/server.rs @@ -251,7 +251,9 @@ impl Server { if is_single { if let Ok(req) = serde_json::from_slice::(&body) { // NOTE: we don't need to track connection id on HTTP, so using hardcoded 0 here. - if let Some(fut) = methods.execute_with_resources(&tx, req, 0, &resources) { + if let Some(fut) = + methods.execute_with_resources(&tx, req, 0, &resources, max_request_body_size) + { fut.await; } } else if let Ok(_req) = serde_json::from_slice::(&body) { @@ -264,11 +266,9 @@ impl Server { // Batch of requests or notifications } else if let Ok(batch) = serde_json::from_slice::>(&body) { if !batch.is_empty() { - join_all( - batch - .into_iter() - .filter_map(|req| methods.execute_with_resources(&tx, req, 0, &resources)), - ) + join_all(batch.into_iter().filter_map(|req| { + methods.execute_with_resources(&tx, req, 0, &resources, max_request_body_size) + })) .await; } else { // "If the batch rpc call itself fails to be recognized as an valid JSON or as an diff --git a/test-utils/src/helpers.rs b/test-utils/src/helpers.rs index 7cb3e7521f..4f6e2be210 100644 --- a/test-utils/src/helpers.rs +++ b/test-utils/src/helpers.rs @@ -73,6 +73,14 @@ pub fn oversized_request() -> String { r#"{"jsonrpc":"2.0","error":{"code":-32701,"message":"Request is too big"},"id":null}"#.into() } +pub fn oversized_response(id: Id, max_limit: u32) -> String { + format!( + r#"{{"jsonrpc":"2.0","error":{{"code":-32702,"message":"Response is too big","data":"Exceeded max limit {}"}},"id":{}}}"#, + max_limit, + serde_json::to_string(&id).unwrap(), + ) +} + pub fn invalid_request(id: Id) -> String { format!( r#"{{"jsonrpc":"2.0","error":{{"code":-32600,"message":"Invalid request"}},"id":{}}}"#, diff --git a/types/src/v2/error.rs b/types/src/v2/error.rs index b1ddb91bbf..3cdbdfa4ca 100644 --- a/types/src/v2/error.rs +++ b/types/src/v2/error.rs @@ -82,6 +82,8 @@ impl<'a> PartialEq for ErrorObject<'a> { pub const PARSE_ERROR_CODE: i32 = -32700; /// Oversized request error code. pub const OVERSIZED_REQUEST_CODE: i32 = -32701; +/// Oversized response error code. +pub const OVERSIZED_RESPONSE_CODE: i32 = -32702; /// Internal error code. pub const INTERNAL_ERROR_CODE: i32 = -32603; /// Invalid params error code. @@ -101,6 +103,8 @@ pub const UNKNOWN_ERROR_CODE: i32 = -32001; pub const PARSE_ERROR_MSG: &str = "Parse error"; /// Oversized request message pub const OVERSIZED_REQUEST_MSG: &str = "Request is too big"; +/// Oversized response message +pub const OVERSIZED_RESPONSE_MSG: &str = "Response is too big"; /// Internal error message. pub const INTERNAL_ERROR_MSG: &str = "Internal error"; /// Invalid params error message. diff --git a/utils/src/server/helpers.rs b/utils/src/server/helpers.rs index 82924245dc..99d2256f62 100644 --- a/utils/src/server/helpers.rs +++ b/utils/src/server/helpers.rs @@ -28,20 +28,83 @@ use crate::server::rpc_module::MethodSink; use futures_channel::mpsc; use futures_util::stream::StreamExt; use jsonrpsee_types::error::{CallError, Error}; +use jsonrpsee_types::to_json_raw_value; +use jsonrpsee_types::v2::error::{OVERSIZED_RESPONSE_CODE, OVERSIZED_RESPONSE_MSG}; use jsonrpsee_types::v2::{ error::{CALL_EXECUTION_FAILED_CODE, UNKNOWN_ERROR_CODE}, ErrorCode, ErrorObject, Id, InvalidRequest, Response, RpcError, TwoPointZero, }; use serde::Serialize; +use std::io; + +/// Bounded writer that allows writing at most `max_len` bytes. +/// +/// ``` +/// use jsonrpsee_utils::server::helpers::BoundedWriter; +/// use std::io::Write; +/// +/// let mut writer = BoundedWriter::new(10); +/// (&mut writer).write("hello".as_bytes()).unwrap(); +/// assert_eq!(std::str::from_utf8(&writer.into_bytes()).unwrap(), "hello"); +/// ``` +#[derive(Debug)] +pub struct BoundedWriter { + max_len: usize, + buf: Vec, +} + +impl BoundedWriter { + /// Create a new bounded writer. + pub fn new(max_len: usize) -> Self { + Self { max_len, buf: Vec::with_capacity(128) } + } + + /// Consume the writer and extract the written bytes. + pub fn into_bytes(self) -> Vec { + self.buf + } +} + +impl<'a> io::Write for &'a mut BoundedWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let len = self.buf.len() + buf.len(); + if self.max_len >= len { + self.buf.extend_from_slice(buf); + Ok(buf.len()) + } else { + Err(io::Error::new(io::ErrorKind::OutOfMemory, "Memory capacity exceeded")) + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + /// Helper for sending JSON-RPC responses to the client -pub fn send_response(id: Id, tx: &MethodSink, result: impl Serialize) { - let json = match serde_json::to_string(&Response { jsonrpc: TwoPointZero, id: id.clone(), result }) { - Ok(json) => json, +pub fn send_response(id: Id, tx: &MethodSink, result: impl Serialize, max_response_size: u32) { + let mut writer = BoundedWriter::new(max_response_size as usize); + + let json = match serde_json::to_writer(&mut writer, &Response { jsonrpc: TwoPointZero, id: id.clone(), result }) { + Ok(_) => { + // Safety - serde_json does not emit invalid UTF-8. + unsafe { String::from_utf8_unchecked(writer.into_bytes()) } + } Err(err) => { tracing::error!("Error serializing response: {:?}", err); - return send_error(id, tx, ErrorCode::InternalError.into()); + if err.is_io() { + let data = to_json_raw_value(&format!("Exceeded max limit {}", max_response_size)).ok(); + let err = ErrorObject { + code: ErrorCode::ServerError(OVERSIZED_RESPONSE_CODE), + message: OVERSIZED_RESPONSE_MSG, + data: data.as_deref(), + }; + return send_error(id, tx, err); + } else { + return send_error(id, tx, ErrorCode::InternalError.into()); + } } }; @@ -108,3 +171,26 @@ pub async fn collect_batch_response(rx: mpsc::UnboundedReceiver) -> Stri buf.push(']'); buf } + +#[cfg(test)] +mod tests { + use super::{BoundedWriter, Id, Response, TwoPointZero}; + + #[test] + fn bounded_serializer_work() { + let mut writer = BoundedWriter::new(100); + let result = "success"; + + assert!( + serde_json::to_writer(&mut writer, &Response { jsonrpc: TwoPointZero, id: Id::Number(1), result }).is_ok() + ); + assert_eq!(String::from_utf8(writer.into_bytes()).unwrap(), r#"{"jsonrpc":"2.0","result":"success","id":1}"#); + } + + #[test] + fn bounded_serializer_cap_works() { + let mut writer = BoundedWriter::new(100); + // NOTE: `"` is part of the serialization so 101 characters. + assert!(serde_json::to_writer(&mut writer, &"x".repeat(99)).is_err()); + } +} diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index 6275607094..fe2aadfb64 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -53,10 +53,11 @@ use std::sync::Arc; /// implemented as a function pointer to a `Fn` function taking four arguments: /// the `id`, `params`, a channel the function uses to communicate the result (or error) /// back to `jsonrpsee`, and the connection ID (useful for the websocket transport). -pub type SyncMethod = Arc; +pub type SyncMethod = Arc; /// Similar to [`SyncMethod`], but represents an asynchronous handler and takes an additional argument containing a [`ResourceGuard`] if configured. -pub type AsyncMethod<'a> = - Arc, Params<'a>, MethodSink, Option) -> BoxFuture<'a, ()>>; +pub type AsyncMethod<'a> = Arc< + dyn Send + Sync + Fn(Id<'a>, Params<'a>, MethodSink, Option, MaxResponseSize) -> BoxFuture<'a, ()>, +>; /// Connection ID, used for stateful protocol such as WebSockets. /// For stateless protocols such as http it's unused, so feel free to set it some hardcoded value. pub type ConnectionId = usize; @@ -64,6 +65,8 @@ pub type ConnectionId = usize; pub type SubscriptionId = u64; /// Sink that is used to send back the result to the server for a specific method. pub type MethodSink = mpsc::UnboundedSender; +/// Max response size in bytes for a executed call. +pub type MaxResponseSize = u32; type Subscribers = Arc)>>>; @@ -146,6 +149,7 @@ impl MethodCallback { req: Request<'_>, conn_id: ConnectionId, claimed: Option, + max_response_size: MaxResponseSize, ) -> Option> { let id = req.id.clone(); let params = Params::new(req.params.map(|params| params.get())); @@ -158,7 +162,7 @@ impl MethodCallback { id, conn_id ); - (callback)(id, params, tx, conn_id); + (callback)(id, params, tx, conn_id, max_response_size); // Release claimed resources drop(claimed); @@ -176,7 +180,7 @@ impl MethodCallback { conn_id ); - Some((callback)(id, params, tx, claimed)) + Some((callback)(id, params, tx, claimed, max_response_size)) } } } @@ -283,10 +287,16 @@ impl Methods { } /// Attempt to execute a callback, sending the resulting JSON (success or error) to the specified sink. - pub fn execute(&self, tx: &MethodSink, req: Request, conn_id: ConnectionId) -> Option> { + pub fn execute( + &self, + tx: &MethodSink, + req: Request, + conn_id: ConnectionId, + max_response_size: MaxResponseSize, + ) -> Option> { tracing::trace!("[Methods::execute] Executing request: {:?}", req); match self.callbacks.get(&*req.method) { - Some(callback) => callback.execute(tx, req, conn_id, None), + Some(callback) => callback.execute(tx, req, conn_id, None, max_response_size), None => { send_error(req.id, tx, ErrorCode::MethodNotFound.into()); None @@ -301,11 +311,12 @@ impl Methods { req: Request, conn_id: ConnectionId, resources: &Resources, + max_response_size: MaxResponseSize, ) -> Option> { tracing::trace!("[Methods::execute_with_resources] Executing request: {:?}", req); match self.callbacks.get(&*req.method) { Some(callback) => match callback.claim(&req.method, resources) { - Ok(guard) => callback.execute(tx, req, conn_id, Some(guard)), + Ok(guard) => callback.execute(tx, req, conn_id, Some(guard), max_response_size), Err(err) => { tracing::error!("[Methods::execute_with_resources] failed to lock resources: {:?}", err); send_error(req.id, tx, ErrorCode::ServerIsBusy.into()); @@ -339,7 +350,7 @@ impl Methods { let (tx, mut rx) = mpsc::unbounded(); - if let Some(fut) = self.execute(&tx, req, 0) { + if let Some(fut) = self.execute(&tx, req, 0, MaxResponseSize::MAX) { fut.await; } @@ -356,7 +367,7 @@ impl Methods { let (tx, mut rx) = mpsc::unbounded(); - if let Some(fut) = self.execute(&tx, req, 0) { + if let Some(fut) = self.execute(&tx, req, 0, MaxResponseSize::MAX) { fut.await; } let response = rx.next().await.expect("Could not establish subscription."); @@ -423,9 +434,9 @@ impl RpcModule { let ctx = self.ctx.clone(); let callback = self.methods.verify_and_insert( method_name, - MethodCallback::new_sync(Arc::new(move |id, params, tx, _| { + MethodCallback::new_sync(Arc::new(move |id, params, tx, _, max_response_size| { match callback(params, &*ctx) { - Ok(res) => send_response(id, tx, res), + Ok(res) => send_response(id, tx, res, max_response_size), Err(err) => send_call_error(id, tx, err), }; })), @@ -448,11 +459,11 @@ impl RpcModule { let ctx = self.ctx.clone(); let callback = self.methods.verify_and_insert( method_name, - MethodCallback::new_async(Arc::new(move |id, params, tx, claimed| { + MethodCallback::new_async(Arc::new(move |id, params, tx, claimed, max_response_size| { let ctx = ctx.clone(); let future = async move { match callback(params, ctx).await { - Ok(res) => send_response(id, &tx, res), + Ok(res) => send_response(id, &tx, res, max_response_size), Err(err) => send_call_error(id, &tx, err), }; @@ -481,12 +492,12 @@ impl RpcModule { let ctx = self.ctx.clone(); let callback = self.methods.verify_and_insert( method_name, - MethodCallback::new_async(Arc::new(move |id, params, tx, claimed| { + MethodCallback::new_async(Arc::new(move |id, params, tx, claimed, max_response_size| { let ctx = ctx.clone(); tokio::task::spawn_blocking(move || { match callback(params, ctx) { - Ok(res) => send_response(id, &tx, res), + Ok(res) => send_response(id, &tx, res, max_response_size), Err(err) => send_call_error(id, &tx, err), }; @@ -542,14 +553,13 @@ impl RpcModule { self.methods.verify_method_name(subscribe_method_name)?; self.methods.verify_method_name(unsubscribe_method_name)?; let ctx = self.ctx.clone(); - let subscribers = Subscribers::default(); { let subscribers = subscribers.clone(); self.methods.mut_callbacks().insert( subscribe_method_name, - MethodCallback::new_sync(Arc::new(move |id, params, method_sink, conn_id| { + MethodCallback::new_sync(Arc::new(move |id, params, method_sink, conn_id, max_response_size| { let (conn_tx, conn_rx) = oneshot::channel::<()>(); let sub_id = { const JS_NUM_MASK: SubscriptionId = !0 >> 11; @@ -561,7 +571,7 @@ impl RpcModule { sub_id }; - send_response(id.clone(), method_sink, sub_id); + send_response(id.clone(), method_sink, sub_id, max_response_size); let sink = SubscriptionSink { inner: method_sink.clone(), @@ -586,7 +596,7 @@ impl RpcModule { { self.methods.mut_callbacks().insert( unsubscribe_method_name, - MethodCallback::new_sync(Arc::new(move |id, params, tx, conn_id| { + MethodCallback::new_sync(Arc::new(move |id, params, tx, conn_id, max_response_size| { let sub_id = match params.one() { Ok(sub_id) => sub_id, Err(_) => { @@ -600,7 +610,7 @@ impl RpcModule { } }; subscribers.lock().remove(&SubscriptionKey { conn_id, sub_id }); - send_response(id, tx, "Unsubscribed"); + send_response(id, tx, "Unsubscribed", max_response_size); })), ); } diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index 1356ccad06..9b2f6ec459 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -41,6 +41,7 @@ use futures_util::io::{BufReader, BufWriter}; use futures_util::stream::{self, StreamExt}; use soketto::connection::Error as SokettoError; use soketto::handshake::{server::Response, Server as SokettoServer}; +use soketto::Sender; use tokio::net::{TcpListener, TcpStream, ToSocketAddrs}; use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; @@ -256,11 +257,11 @@ async fn background_task( while !stop_server2.shutdown_requested() { match rx.next().await { Some(response) => { - // TODO: check length of response https://github.com/paritytech/jsonrpsee/issues/536 - tracing::debug!("send {} bytes", response.len()); - tracing::trace!("send: {}", response); - let _ = sender.send_text_owned(response).await; - let _ = sender.flush().await; + // If websocket message send fail then terminate the connection. + if let Err(err) = send_ws_message(&mut sender, response).await { + tracing::error!("WS transport error: {:?}; terminate connection", err); + break; + } } None => break, }; @@ -310,7 +311,9 @@ async fn background_task( if let Ok(req) = serde_json::from_slice::(&data) { tracing::debug!("recv method call={}", req.method); tracing::trace!("recv: req={:?}", req); - if let Some(fut) = methods.execute_with_resources(&tx, req, conn_id, &resources) { + if let Some(fut) = + methods.execute_with_resources(&tx, req, conn_id, &resources, max_request_body_size) + { method_executors.add(fut); } } else { @@ -334,10 +337,15 @@ async fn background_task( tracing::debug!("recv batch len={}", batch.len()); tracing::trace!("recv: batch={:?}", batch); if !batch.is_empty() { - let methods_stream = - stream::iter(batch.into_iter().filter_map(|req| { - methods.execute_with_resources(&tx_batch, req, conn_id, resources) - })); + let methods_stream = stream::iter(batch.into_iter().filter_map(|req| { + methods.execute_with_resources( + &tx_batch, + req, + conn_id, + resources, + max_request_body_size, + ) + })); let results = methods_stream .for_each_concurrent(None, |item| item) @@ -536,3 +544,13 @@ impl Builder { Ok(Server { listener, cfg: self.settings, stop_monitor, resources }) } } + +async fn send_ws_message( + sender: &mut Sender>>>, + response: String, +) -> Result<(), Error> { + tracing::debug!("send {} bytes", response.len()); + tracing::trace!("send: {}", response); + sender.send_text_owned(response).await?; + sender.flush().await.map_err(Into::into) +} diff --git a/ws-server/src/tests.rs b/ws-server/src/tests.rs index 20bd65e67c..1a81b62f8b 100644 --- a/ws-server/src/tests.rs +++ b/ws-server/src/tests.rs @@ -163,23 +163,23 @@ async fn can_set_the_max_request_body_size() { let addr = "127.0.0.1:0"; // Rejects all requests larger than 10 bytes - let server = WsServerBuilder::default().max_request_body_size(10).build(addr).await.unwrap(); + let server = WsServerBuilder::default().max_request_body_size(100).build(addr).await.unwrap(); let mut module = RpcModule::new(()); - module.register_method("anything", |_p, _cx| Ok(())).unwrap(); + module.register_method("anything", |_p, _cx| Ok("a".repeat(100))).unwrap(); let addr = server.local_addr().unwrap(); let handle = server.start(module).unwrap(); let mut client = WebSocketTestClient::new(addr).await.unwrap(); // Invalid: too long - let req = "any string longer than 10 bytes"; + let req = format!(r#"{{"jsonrpc":"2.0", "method":{}, "id":1}}"#, "a".repeat(100)); let response = client.send_request_text(req).await.unwrap(); assert_eq!(response, oversized_request()); - // Still invalid, but not oversized - let req = "shorty"; + // Oversized response. + let req = r#"{"jsonrpc":"2.0", "method":"anything", "id":1}"#; let response = client.send_request_text(req).await.unwrap(); - assert_eq!(response, parse_error(Id::Null)); + assert_eq!(response, oversized_response(Id::Num(1), 100)); handle.stop().unwrap(); } From aacf7c0ecdb71da345e7c5cb0283f5cb5a040bd7 Mon Sep 17 00:00:00 2001 From: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:53:52 +0100 Subject: [PATCH 13/31] Periodically wake `DriverSelect` so we can poll whether or not `stop` had been called. (#556) * Fix some clippy issues * Add an interval to periodically wake the SelectDriver Waker * Apply suggestions from code review Co-authored-by: David * Naming grumbles Co-authored-by: David --- proc-macros/src/render_client.rs | 2 +- tests/tests/helpers.rs | 2 +- tests/tests/resource_limiting.rs | 2 - ws-server/Cargo.toml | 2 +- ws-server/src/future.rs | 18 +++++- ws-server/src/server.rs | 104 +++++++++++++++++++------------ 6 files changed, 85 insertions(+), 45 deletions(-) diff --git a/proc-macros/src/render_client.rs b/proc-macros/src/render_client.rs index 11a12680bf..c655a261ab 100644 --- a/proc-macros/src/render_client.rs +++ b/proc-macros/src/render_client.rs @@ -142,7 +142,7 @@ impl RpcDescription { fn encode_params( &self, - params: &Vec<(syn::PatIdent, syn::Type)>, + params: &[(syn::PatIdent, syn::Type)], param_kind: &ParamKind, signature: &syn::TraitItemMethod, ) -> TokenStream2 { diff --git a/tests/tests/helpers.rs b/tests/tests/helpers.rs index f33086e445..baeb76afbf 100644 --- a/tests/tests/helpers.rs +++ b/tests/tests/helpers.rs @@ -81,7 +81,7 @@ pub async fn websocket_server_with_subscription() -> (SocketAddr, WsServerHandle .register_subscription("subscribe_noop", "unsubscribe_noop", |_, mut sink, _| { std::thread::spawn(move || { std::thread::sleep(Duration::from_secs(1)); - sink.close("Server closed the stream because it was lazy".into()) + sink.close("Server closed the stream because it was lazy") }); Ok(()) }) diff --git a/tests/tests/resource_limiting.rs b/tests/tests/resource_limiting.rs index 736de5123d..033cec0abe 100644 --- a/tests/tests/resource_limiting.rs +++ b/tests/tests/resource_limiting.rs @@ -160,8 +160,6 @@ async fn run_tests_on_ws_server(server_addr: SocketAddr, server_handle: WsServer assert!(pass_mem.is_ok()); assert_server_busy(fail_mem); - // Client being active prevents the server from shutting down?! - drop(client); server_handle.stop().unwrap().await; } diff --git a/ws-server/Cargo.toml b/ws-server/Cargo.toml index 377eaf1d01..a0f21f5010 100644 --- a/ws-server/Cargo.toml +++ b/ws-server/Cargo.toml @@ -17,7 +17,7 @@ jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["server"] tracing = "0.1" serde_json = { version = "1", features = ["raw_value"] } soketto = "0.7.1" -tokio = { version = "1", features = ["net", "rt-multi-thread", "macros"] } +tokio = { version = "1", features = ["net", "rt-multi-thread", "macros", "time"] } tokio-util = { version = "0.6", features = ["compat"] } [dev-dependencies] diff --git a/ws-server/src/future.rs b/ws-server/src/future.rs index 69b35b6977..fd2964d194 100644 --- a/ws-server/src/future.rs +++ b/ws-server/src/future.rs @@ -36,6 +36,10 @@ use std::sync::{ Arc, Weak, }; use std::task::{Context, Poll}; +use tokio::time::{self, Duration, Interval}; + +/// Polling for server stop monitor interval in milliseconds. +const STOP_MONITOR_POLLING_INTERVAL: u64 = 1000; /// This is a flexible collection of futures that need to be driven to completion /// alongside some other future, such as connection handlers that need to be @@ -45,11 +49,16 @@ use std::task::{Context, Poll}; /// `select_with` providing some other future, the result of which you need. pub(crate) struct FutureDriver { futures: Vec, + stop_monitor_heartbeat: Interval, } impl Default for FutureDriver { fn default() -> Self { - FutureDriver { futures: Vec::new() } + let mut heartbeat = time::interval(Duration::from_millis(STOP_MONITOR_POLLING_INTERVAL)); + + heartbeat.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + + FutureDriver { futures: Vec::new(), stop_monitor_heartbeat: heartbeat } } } @@ -92,6 +101,12 @@ where } } } + + fn poll_stop_monitor_heartbeat(&mut self, cx: &mut Context) { + // We don't care about the ticks of the heartbeat, it's here only + // to periodically wake the `Waker` on `cx`. + let _ = self.stop_monitor_heartbeat.poll_tick(cx); + } } impl Future for FutureDriver @@ -132,6 +147,7 @@ where let this = Pin::into_inner(self); this.driver.drive(cx); + this.driver.poll_stop_monitor_heartbeat(cx); this.selector.poll_unpin(cx) } diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index 9b2f6ec459..bb950871aa 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -91,7 +91,7 @@ impl Server { let mut id = 0; let mut connections = FutureDriver::default(); - let mut incoming = Incoming::new(self.listener, &stop_monitor); + let mut incoming = Monitored::new(Incoming(self.listener), &stop_monitor); loop { match connections.select_with(&mut incoming).await { @@ -123,10 +123,10 @@ impl Server { id = id.wrapping_add(1); } - Err(IncomingError::Io(err)) => { + Err(MonitoredError::Selector(err)) => { tracing::error!("Error while awaiting a new connection: {:?}", err); } - Err(IncomingError::Shutdown) => break, + Err(MonitoredError::Shutdown) => break, } } @@ -134,35 +134,53 @@ impl Server { } } -/// This is a glorified select listening to new connections, while also checking -/// for `stop_receiver` signal. -struct Incoming<'a> { - listener: TcpListener, +/// This is a glorified select listening for new messages, while also checking the `stop_receiver` signal. +struct Monitored<'a, F> { + future: F, stop_monitor: &'a StopMonitor, } -impl<'a> Incoming<'a> { - fn new(listener: TcpListener, stop_monitor: &'a StopMonitor) -> Self { - Incoming { listener, stop_monitor } +impl<'a, F> Monitored<'a, F> { + fn new(future: F, stop_monitor: &'a StopMonitor) -> Self { + Monitored { future, stop_monitor } } } -enum IncomingError { +enum MonitoredError { Shutdown, - Io(std::io::Error), + Selector(E), +} + +struct Incoming(TcpListener); + +impl<'a> Future for Monitored<'a, Incoming> { + type Output = Result<(TcpStream, SocketAddr), MonitoredError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let this = Pin::into_inner(self); + + if this.stop_monitor.shutdown_requested() { + return Poll::Ready(Err(MonitoredError::Shutdown)); + } + + this.future.0.poll_accept(cx).map_err(MonitoredError::Selector) + } } -impl<'a> Future for Incoming<'a> { - type Output = Result<(TcpStream, SocketAddr), IncomingError>; +impl<'a, 'f, F, T, E> Future for Monitored<'a, Pin<&'f mut F>> +where + F: Future>, +{ + type Output = Result>; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { let this = Pin::into_inner(self); if this.stop_monitor.shutdown_requested() { - return Poll::Ready(Err(IncomingError::Shutdown)); + return Poll::Ready(Err(MonitoredError::Shutdown)); } - this.listener.poll_accept(cx).map_err(IncomingError::Io) + this.future.poll_unpin(cx).map_err(MonitoredError::Selector) } } @@ -276,31 +294,39 @@ async fn background_task( let mut data = Vec::with_capacity(100); let mut method_executors = FutureDriver::default(); - while !stop_server.shutdown_requested() { + loop { data.clear(); - if let Err(err) = method_executors.select_with(receiver.receive_data(&mut data)).await { - match err { - SokettoError::Closed => { - tracing::debug!("Remote peer terminated the connection: {}", conn_id); - tx.close_channel(); - return Ok(()); - } - SokettoError::MessageTooLarge { current, maximum } => { - tracing::warn!( - "WS transport error: message is too big error ({} bytes, max is {})", - current, - maximum - ); - send_error(Id::Null, &tx, ErrorCode::OversizedRequest.into()); - continue; - } - // These errors can not be gracefully handled, so just log them and terminate the connection. - err => { - tracing::error!("WS transport error: {:?} => terminating connection {}", err, conn_id); - tx.close_channel(); - return Err(err.into()); - } + { + // Need the extra scope to drop this pinned future and reclaim access to `data` + let receive = receiver.receive_data(&mut data); + + tokio::pin!(receive); + + if let Err(err) = method_executors.select_with(Monitored::new(receive, &stop_server)).await { + match err { + MonitoredError::Selector(SokettoError::Closed) => { + tracing::debug!("WS transport error: remote peer terminated the connection: {}", conn_id); + tx.close_channel(); + return Ok(()); + } + MonitoredError::Selector(SokettoError::MessageTooLarge { current, maximum }) => { + tracing::warn!( + "WS transport error: outgoing message is too big error ({} bytes, max is {})", + current, + maximum + ); + send_error(Id::Null, &tx, ErrorCode::OversizedRequest.into()); + continue; + } + // These errors can not be gracefully handled, so just log them and terminate the connection. + MonitoredError::Selector(err) => { + tracing::error!("WS transport error: {:?} => terminating connection {}", err, conn_id); + tx.close_channel(); + return Err(err.into()); + } + MonitoredError::Shutdown => break, + }; }; }; From e2d4722c25bec808e24d53605e4b87a7323afac2 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Wed, 17 Nov 2021 11:29:48 +0100 Subject: [PATCH 14/31] feat: make it possible to try several sockaddrs when starting server (#567) * fix: enable several sockaddress when starting servers * nits * more verbose asserts in rustdoc tests * fix tests * fix tests again --- benches/helpers.rs | 2 +- examples/http.rs | 2 +- http-server/src/server.rs | 58 +++++++++++++++++++++++++------- http-server/src/tests.rs | 2 +- tests/tests/helpers.rs | 2 +- tests/tests/resource_limiting.rs | 2 +- ws-server/src/server.rs | 19 +++++++++-- 7 files changed, 68 insertions(+), 19 deletions(-) diff --git a/benches/helpers.rs b/benches/helpers.rs index a8d64a4b12..f60fc122b6 100644 --- a/benches/helpers.rs +++ b/benches/helpers.rs @@ -76,7 +76,7 @@ pub async fn http_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee:: let server = HttpServerBuilder::default() .max_request_body_size(u32::MAX) .custom_tokio_runtime(handle) - .build("127.0.0.1:0".parse().unwrap()) + .build("127.0.0.1:0") .unwrap(); let mut module = RpcModule::new(()); module.register_method(SYNC_METHOD_NAME, |_, _| Ok("lo")).unwrap(); diff --git a/examples/http.rs b/examples/http.rs index 30d12393e7..8c4d2b485d 100644 --- a/examples/http.rs +++ b/examples/http.rs @@ -51,7 +51,7 @@ async fn main() -> anyhow::Result<()> { } async fn run_server() -> anyhow::Result<(SocketAddr, HttpServerHandle)> { - let server = HttpServerBuilder::default().build("127.0.0.1:0".parse()?)?; + let server = HttpServerBuilder::default().build("127.0.0.1:0".parse::()?)?; let mut module = RpcModule::new(()); module.register_method("say_hello", |_, _| Ok("lo"))?; diff --git a/http-server/src/server.rs b/http-server/src/server.rs index 44e2bfcd45..43269e9926 100644 --- a/http-server/src/server.rs +++ b/http-server/src/server.rs @@ -49,7 +49,7 @@ use socket2::{Domain, Socket, Type}; use std::{ cmp, future::Future, - net::{SocketAddr, TcpListener}, + net::{SocketAddr, TcpListener, ToSocketAddrs}, pin::Pin, task::{Context, Poll}, }; @@ -106,7 +106,50 @@ impl Builder { } /// Finalizes the configuration of the server. - pub fn build(self, addr: SocketAddr) -> Result { + /// + /// ```rust + /// #[tokio::main] + /// async fn main() { + /// let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + /// let occupied_addr = listener.local_addr().unwrap(); + /// let addrs: &[std::net::SocketAddr] = &[ + /// occupied_addr, + /// "127.0.0.1:0".parse().unwrap(), + /// ]; + /// assert!(jsonrpsee_http_server::HttpServerBuilder::default().build(occupied_addr).is_err()); + /// assert!(jsonrpsee_http_server::HttpServerBuilder::default().build(addrs).is_ok()); + /// } + /// ``` + pub fn build(self, addrs: impl ToSocketAddrs) -> Result { + let mut err: Option = None; + + for addr in addrs.to_socket_addrs()? { + let (listener, local_addr) = match self.inner_builder(addr) { + Ok(res) => res, + Err(e) => { + err = Some(e); + continue; + } + }; + + return Ok(Server { + listener, + local_addr, + access_control: self.access_control, + max_request_body_size: self.max_request_body_size, + resources: self.resources, + tokio_runtime: self.tokio_runtime, + }); + } + + let err = err.unwrap_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "No address found").into()); + Err(err) + } + + fn inner_builder( + &self, + addr: SocketAddr, + ) -> Result<(hyper::server::Builder, Option), Error> { let domain = Domain::for_address(addr); let socket = Socket::new(domain, Type::STREAM, None)?; socket.set_nodelay(true)?; @@ -119,17 +162,8 @@ impl Builder { socket.listen(128)?; let listener: TcpListener = socket.into(); let local_addr = listener.local_addr().ok(); - let listener = hyper::Server::from_tcp(listener)?; - - Ok(Server { - listener, - local_addr, - access_control: self.access_control, - max_request_body_size: self.max_request_body_size, - resources: self.resources, - tokio_runtime: self.tokio_runtime, - }) + Ok((listener, local_addr)) } } diff --git a/http-server/src/tests.rs b/http-server/src/tests.rs index 4e3bf7cf44..6fa15b26e7 100644 --- a/http-server/src/tests.rs +++ b/http-server/src/tests.rs @@ -38,7 +38,7 @@ use jsonrpsee_test_utils::TimeoutFutureExt; use serde_json::Value as JsonValue; async fn server() -> (SocketAddr, ServerHandle) { - let server = HttpServerBuilder::default().build("127.0.0.1:0".parse().unwrap()).unwrap(); + let server = HttpServerBuilder::default().build("127.0.0.1:0").unwrap(); let ctx = TestContext; let mut module = RpcModule::new(ctx); let addr = server.local_addr().unwrap(); diff --git a/tests/tests/helpers.rs b/tests/tests/helpers.rs index baeb76afbf..a9773522f4 100644 --- a/tests/tests/helpers.rs +++ b/tests/tests/helpers.rs @@ -113,7 +113,7 @@ pub async fn websocket_server() -> SocketAddr { } pub async fn http_server() -> (SocketAddr, HttpServerHandle) { - let server = HttpServerBuilder::default().build("127.0.0.1:0".parse().unwrap()).unwrap(); + let server = HttpServerBuilder::default().build("127.0.0.1:0").unwrap(); let mut module = RpcModule::new(()); let addr = server.local_addr().unwrap(); module.register_method("say_hello", |_, _| Ok("hello")).unwrap(); diff --git a/tests/tests/resource_limiting.rs b/tests/tests/resource_limiting.rs index 033cec0abe..f36f9cf9cf 100644 --- a/tests/tests/resource_limiting.rs +++ b/tests/tests/resource_limiting.rs @@ -108,7 +108,7 @@ async fn http_server(module: RpcModule<()>) -> Result<(SocketAddr, HttpServerHan let server = HttpServerBuilder::default() .register_resource("CPU", 6, 2)? .register_resource("MEM", 10, 1)? - .build("127.0.0.1:0".parse().unwrap())?; + .build("127.0.0.1:0")?; let addr = server.local_addr()?; let handle = server.start(module)?; diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index bb950871aa..6594f0d144 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -563,8 +563,23 @@ impl Builder { } /// Finalize the configuration of the server. Consumes the [`Builder`]. - pub async fn build(self, addr: impl ToSocketAddrs) -> Result { - let listener = TcpListener::bind(addr).await?; + /// + /// ```rust + /// #[tokio::main] + /// async fn main() { + /// let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + /// let occupied_addr = listener.local_addr().unwrap(); + /// let addrs: &[std::net::SocketAddr] = &[ + /// occupied_addr, + /// "127.0.0.1:0".parse().unwrap(), + /// ]; + /// assert!(jsonrpsee_ws_server::WsServerBuilder::default().build(occupied_addr).await.is_err()); + /// assert!(jsonrpsee_ws_server::WsServerBuilder::default().build(addrs).await.is_ok()); + /// } + /// ``` + /// + pub async fn build(self, addrs: impl ToSocketAddrs) -> Result { + let listener = TcpListener::bind(addrs).await?; let stop_monitor = StopMonitor::new(); let resources = self.resources; Ok(Server { listener, cfg: self.settings, stop_monitor, resources }) From 6af6db24b9f92e7f9ce1232d4f667f7d36db583a Mon Sep 17 00:00:00 2001 From: David Date: Wed, 17 Nov 2021 14:53:27 +0100 Subject: [PATCH 15/31] Implement SubscriptionClient for HttpClient (#563) Closes https://github.com/paritytech/jsonrpsee/issues/448 This PR adds an implementation for `SubscriptionClient` to the `HttpClient` struct, which makes it possible for http clients to use macro-generated RPC servers. If an http client tries to set up a subscription it will fail with a `HttpNotImplemented` error. --- http-client/src/client.rs | 28 ++++++++++++++++++++++++++-- tests/tests/proc_macros.rs | 21 ++++++++++++++++++++- types/src/error.rs | 3 +++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/http-client/src/client.rs b/http-client/src/client.rs index dca85ce2e1..300c248d5e 100644 --- a/http-client/src/client.rs +++ b/http-client/src/client.rs @@ -26,9 +26,9 @@ use crate::transport::HttpTransportClient; use crate::types::{ - traits::Client, + traits::{Client, SubscriptionClient}, v2::{Id, NotificationSer, ParamsSer, RequestSer, Response, RpcError}, - CertificateStore, Error, RequestIdManager, TEN_MB_SIZE_BYTES, + CertificateStore, Error, RequestIdManager, Subscription, TEN_MB_SIZE_BYTES, }; use async_trait::async_trait; use fnv::FnvHashMap; @@ -194,3 +194,27 @@ impl Client for HttpClient { Ok(responses) } } + +#[async_trait] +impl SubscriptionClient for HttpClient { + /// Send a subscription request to the server. Not implemented for HTTP; will always return [`Error::HttpNotImplemented`]. + async fn subscribe<'a, N>( + &self, + _subscribe_method: &'a str, + _params: Option>, + _unsubscribe_method: &'a str, + ) -> Result, Error> + where + N: DeserializeOwned, + { + Err(Error::HttpNotImplemented) + } + + /// Subscribe to a specific method. Not implemented for HTTP; will always return [`Error::HttpNotImplemented`]. + async fn subscribe_to_method<'a, N>(&self, _method: &'a str) -> Result, Error> + where + N: DeserializeOwned, + { + Err(Error::HttpNotImplemented) + } +} diff --git a/tests/tests/proc_macros.rs b/tests/tests/proc_macros.rs index 84f9a313ec..2597e8caf1 100644 --- a/tests/tests/proc_macros.rs +++ b/tests/tests/proc_macros.rs @@ -28,7 +28,11 @@ use std::net::SocketAddr; -use jsonrpsee::{ws_client::*, ws_server::WsServerBuilder}; +use jsonrpsee::{ + http_client::HttpClientBuilder, http_server::HttpServerBuilder, types::Error, ws_client::*, + ws_server::WsServerBuilder, +}; + use serde_json::value::RawValue; mod rpc_impl { @@ -305,3 +309,18 @@ async fn multiple_blocking_calls_overlap() { // Each request takes 50ms, added 10ms margin for scheduling assert!(elapsed < Duration::from_millis(60), "Expected less than 60ms, got {:?}", elapsed); } + +#[tokio::test] +async fn subscriptions_do_not_work_for_http_servers() { + let htserver = HttpServerBuilder::default().build("127.0.0.1:0".parse().unwrap()).unwrap(); + let addr = htserver.local_addr().unwrap(); + let htserver_url = format!("http://{}", addr); + let _handle = htserver.start(RpcServerImpl.into_rpc()).unwrap(); + + let htclient = HttpClientBuilder::default().build(&htserver_url).unwrap(); + + assert_eq!(htclient.sync_method().await.unwrap(), 10); + assert!(htclient.sub().await.is_err()); + assert!(matches!(htclient.sub().await, Err(Error::HttpNotImplemented))); + assert_eq!(htclient.sync_method().await.unwrap(), 10); +} diff --git a/types/src/error.rs b/types/src/error.rs index c0fc7e865b..16ea73b452 100644 --- a/types/src/error.rs +++ b/types/src/error.rs @@ -160,6 +160,9 @@ pub enum Error { /// Custom error. #[error("Custom error: {0}")] Custom(String), + /// Not implemented for HTTP clients. + #[error("Not implemented")] + HttpNotImplemented, } impl Error { From fff8460c13c6976976e1f5a872a603c5b6d2a6e9 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Thu, 18 Nov 2021 10:19:44 +0100 Subject: [PATCH 16/31] rpc module: report error on invalid subscription (#561) * rpc module: report error on invalid subscription * fix tests * remove some boiler plate * remove unused code --- test-utils/src/helpers.rs | 10 +++++++ types/src/v2/error.rs | 23 ++++++++++++++- utils/src/server/rpc_module.rs | 17 ++++++++--- ws-server/src/tests.rs | 53 ++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/test-utils/src/helpers.rs b/test-utils/src/helpers.rs index 4f6e2be210..3db1050a07 100644 --- a/test-utils/src/helpers.rs +++ b/test-utils/src/helpers.rs @@ -27,6 +27,7 @@ use crate::mocks::{Body, HttpResponse, Id, Uri}; use hyper::service::{make_service_fn, service_fn}; use hyper::{Request, Response, Server}; +use serde::Serialize; use serde_json::Value; use std::convert::Infallible; use std::net::SocketAddr; @@ -95,6 +96,15 @@ pub fn invalid_params(id: Id) -> String { ) } +pub fn call(method: &str, params: Vec, id: Id) -> String { + format!( + r#"{{"jsonrpc":"2.0","method":{},"params":{},"id":{}}}"#, + serde_json::to_string(method).unwrap(), + serde_json::to_string(¶ms).unwrap(), + serde_json::to_string(&id).unwrap() + ) +} + pub fn call_execution_failed(msg: &str, id: Id) -> String { format!( r#"{{"jsonrpc":"2.0","error":{{"code":-32000,"message":"{}"}},"id":{}}}"#, diff --git a/types/src/v2/error.rs b/types/src/v2/error.rs index 3cdbdfa4ca..27380e2891 100644 --- a/types/src/v2/error.rs +++ b/types/src/v2/error.rs @@ -44,6 +44,13 @@ pub struct RpcError<'a> { pub id: Id<'a>, } +impl<'a> RpcError<'a> { + /// Create a new `RpcError`. + pub fn new(error: ErrorObject<'a>, id: Id<'a>) -> Self { + Self { jsonrpc: TwoPointZero, error, id } + } +} + impl<'a> fmt::Display for RpcError<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", serde_json::to_string(&self).expect("infallible; qed")) @@ -64,6 +71,13 @@ pub struct ErrorObject<'a> { pub data: Option<&'a RawValue>, } +impl<'a> ErrorObject<'a> { + /// Create a new `ErrorObject` with optional data. + pub fn new(code: ErrorCode, data: Option<&'a RawValue>) -> ErrorObject<'a> { + Self { code, message: code.message(), data } + } +} + impl<'a> From for ErrorObject<'a> { fn from(code: ErrorCode) -> Self { Self { code, message: code.message(), data: None } @@ -73,7 +87,7 @@ impl<'a> From for ErrorObject<'a> { impl<'a> PartialEq for ErrorObject<'a> { fn eq(&self, other: &Self) -> bool { let this_raw = self.data.map(|r| r.get()); - let other_raw = self.data.map(|r| r.get()); + let other_raw = other.data.map(|r| r.get()); self.code == other.code && self.message == other.message && this_raw == other_raw } } @@ -98,6 +112,8 @@ pub const SERVER_IS_BUSY_CODE: i32 = -32604; pub const CALL_EXECUTION_FAILED_CODE: i32 = -32000; /// Unknown error. pub const UNKNOWN_ERROR_CODE: i32 = -32001; +/// Invalid subscription error code. +pub const INVALID_SUBSCRIPTION_CODE: i32 = -32002; /// Parse error message pub const PARSE_ERROR_MSG: &str = "Parse error"; @@ -212,6 +228,11 @@ impl serde::Serialize for ErrorCode { } } +/// Create a invalid subscription ID error. +pub fn invalid_subscription_err(data: Option<&RawValue>) -> ErrorObject { + ErrorObject::new(ErrorCode::ServerError(INVALID_SUBSCRIPTION_CODE), data) +} + #[cfg(test)] mod tests { use super::{ErrorCode, ErrorObject, Id, RpcError, TwoPointZero}; diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index fe2aadfb64..2d91cebf88 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -29,6 +29,8 @@ use crate::server::resource_limiting::{ResourceGuard, ResourceTable, ResourceVec use beef::Cow; use futures_channel::{mpsc, oneshot}; use futures_util::{future::BoxFuture, FutureExt, StreamExt}; +use jsonrpsee_types::to_json_raw_value; +use jsonrpsee_types::v2::error::{invalid_subscription_err, CALL_EXECUTION_FAILED_CODE}; use jsonrpsee_types::{ error::{Error, SubscriptionClosedError}, traits::ToRpcParams, @@ -587,7 +589,7 @@ impl RpcModule { err, id ); - send_error(id, method_sink, ErrorCode::ServerError(-1).into()); + send_error(id, method_sink, ErrorCode::ServerError(CALL_EXECUTION_FAILED_CODE).into()); } })), ); @@ -605,12 +607,18 @@ impl RpcModule { unsubscribe_method_name, id ); - send_error(id, tx, ErrorCode::ServerError(-1).into()); + let err = to_json_raw_value(&"Invalid subscription ID type, must be integer").ok(); + send_error(id, tx, invalid_subscription_err(err.as_deref())); return; } }; - subscribers.lock().remove(&SubscriptionKey { conn_id, sub_id }); - send_response(id, tx, "Unsubscribed", max_response_size); + + if subscribers.lock().remove(&SubscriptionKey { conn_id, sub_id }).is_some() { + send_response(id, tx, "Unsubscribed", max_response_size); + } else { + let err = to_json_raw_value(&format!("Invalid subscription ID={}", sub_id)).ok(); + send_error(id, tx, invalid_subscription_err(err.as_deref())) + } })), ); } @@ -698,6 +706,7 @@ impl SubscriptionSink { fn inner_close(&mut self, err: &SubscriptionClosedError) { self.is_connected.take(); if let Some((sink, _)) = self.subscribers.lock().remove(&self.uniq_sub) { + tracing::debug!("Closing subscription: {:?}", self.uniq_sub.sub_id); let msg = self.build_message(err).expect("valid json infallible; qed"); let _ = sink.unbounded_send(msg); } diff --git a/ws-server/src/tests.rs b/ws-server/src/tests.rs index 1a81b62f8b..121cbdf7fb 100644 --- a/ws-server/src/tests.rs +++ b/ws-server/src/tests.rs @@ -27,12 +27,16 @@ #![cfg(test)] use crate::types::error::{CallError, Error}; +use crate::types::v2::{self, Response, RpcError}; +use crate::types::DeserializeOwned; use crate::{future::ServerHandle, RpcModule, WsServerBuilder}; use anyhow::anyhow; use futures_util::future::join; use jsonrpsee_test_utils::helpers::*; use jsonrpsee_test_utils::mocks::{Id, TestContext, WebSocketTestClient, WebSocketTestError}; use jsonrpsee_test_utils::TimeoutFutureExt; +use jsonrpsee_types::to_json_raw_value; +use jsonrpsee_types::v2::error::invalid_subscription_err; use serde_json::Value as JsonValue; use std::{fmt, net::SocketAddr, time::Duration}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -41,6 +45,11 @@ fn init_logger() { let _ = FmtSubscriber::builder().with_env_filter(EnvFilter::from_default_env()).try_init(); } +fn deser_call(raw: String) -> T { + let out: Response = serde_json::from_str(&raw).unwrap(); + out.result +} + /// Applications can/should provide their own error. #[derive(Debug)] struct MyAppError; @@ -107,6 +116,15 @@ async fn server_with_handles() -> (SocketAddr, ServerHandle) { Ok("Yawn!") }) .unwrap(); + module + .register_subscription("subscribe_hello", "unsubscribe_hello", |_, sink, _| { + std::thread::spawn(move || loop { + let _ = sink; + std::thread::sleep(std::time::Duration::from_secs(30)); + }); + Ok(()) + }) + .unwrap(); let addr = server.local_addr().unwrap(); @@ -569,3 +587,38 @@ async fn run_forever() { // Send the shutdown request from one handle and await the server on the second one. join(server_handle.clone().stop().unwrap(), server_handle).with_timeout(TIMEOUT).await.unwrap(); } + +#[tokio::test] +async fn unsubscribe_twice_should_indicate_error() { + init_logger(); + let addr = server().await; + let mut client = WebSocketTestClient::new(addr).with_default_timeout().await.unwrap().unwrap(); + + let sub_call = call("subscribe_hello", Vec::<()>::new(), Id::Num(0)); + let sub_id: u64 = deser_call(client.send_request_text(sub_call).await.unwrap()); + + let unsub_call = call("unsubscribe_hello", vec![sub_id], Id::Num(1)); + let unsub_1: String = deser_call(client.send_request_text(unsub_call).await.unwrap()); + assert_eq!(&unsub_1, "Unsubscribed"); + + let unsub_call = call("unsubscribe_hello", vec![sub_id], Id::Num(2)); + let unsub_2 = client.send_request_text(unsub_call).await.unwrap(); + let unsub_2_err: RpcError = serde_json::from_str(&unsub_2).unwrap(); + let sub_id = to_json_raw_value(&sub_id).unwrap(); + + let err = Some(to_json_raw_value(&format!("Invalid subscription ID={}", sub_id)).unwrap()); + assert_eq!(unsub_2_err, RpcError::new(invalid_subscription_err(err.as_deref()), v2::Id::Number(2))); +} + +#[tokio::test] +async fn unsubscribe_wrong_sub_id_type() { + init_logger(); + let addr = server().await; + let mut client = WebSocketTestClient::new(addr).with_default_timeout().await.unwrap().unwrap(); + + let unsub = + client.send_request_text(call("unsubscribe_hello", vec!["string_is_not_supported"], Id::Num(0))).await.unwrap(); + let unsub_2_err: RpcError = serde_json::from_str(&unsub).unwrap(); + let err = Some(to_json_raw_value(&"Invalid subscription ID type, must be integer").unwrap()); + assert_eq!(unsub_2_err, RpcError::new(invalid_subscription_err(err.as_deref()), v2::Id::Number(0))); +} From 0e46b5cea9cd632dc438a005c77bbaa5c2af562f Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Thu, 18 Nov 2021 12:03:57 +0100 Subject: [PATCH 17/31] [rpc module]: improve `TestSubscription` to return `None` when closed (#566) * fix(TestSubscription): use None for closed. * add test for subscription close --- utils/src/server/rpc_module.rs | 47 +++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index 2d91cebf88..4652962c60 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -739,13 +739,17 @@ impl TestSubscription { self.sub_id } - /// Get the next element of type T from the underlying stream. + /// Returns `Some((val, sub_id))` for the next element of type T from the underlying stream, + /// otherwise `None` if the subscruption was closed. /// - /// Panics if the stream was closed or if the decoding the value as `T`. - pub async fn next(&mut self) -> (T, jsonrpsee_types::v2::SubscriptionId) { - let raw = self.rx.next().await.expect("subscription not closed"); - let val: SubscriptionResponse = serde_json::from_str(&raw).expect("valid response"); - (val.params.result, val.params.subscription) + /// # Panics + /// + /// If the decoding the value as `T` fails. + pub async fn next(&mut self) -> Option<(T, jsonrpsee_types::v2::SubscriptionId)> { + let raw = self.rx.next().await?; + let val: SubscriptionResponse = + serde_json::from_str(&raw).expect("valid response in TestSubscription::next()"); + Some((val.params.result, val.params.subscription)) } } @@ -937,14 +941,39 @@ mod tests { let mut my_sub: TestSubscription = module.test_subscription("my_sub", Vec::<()>::new()).await; for i in (0..=2).rev() { - let (val, id) = my_sub.next::().await; + let (val, id) = my_sub.next::().await.unwrap(); assert_eq!(val, std::char::from_digit(i, 10).unwrap()); assert_eq!(id, v2::params::SubscriptionId::Num(my_sub.subscription_id())); } - // The subscription is now closed - let (sub_closed_err, _) = my_sub.next::().await; + // The subscription is now closed by the server. + let (sub_closed_err, _) = my_sub.next::().await.unwrap(); assert_eq!(sub_closed_err.subscription_id(), my_sub.subscription_id()); assert_eq!(sub_closed_err.close_reason(), "Closed by the server"); } + + #[tokio::test] + async fn close_test_subscribing_without_server() { + let mut module = RpcModule::new(()); + module + .register_subscription("my_sub", "my_unsub", |_, mut sink, _| { + std::thread::spawn(move || loop { + if let Err(Error::SubscriptionClosed(_)) = sink.send(&"lo") { + return; + } + std::thread::sleep(std::time::Duration::from_millis(500)); + }); + Ok(()) + }) + .unwrap(); + + let mut my_sub: TestSubscription = module.test_subscription("my_sub", Vec::<()>::new()).await; + let (val, id) = my_sub.next::().await.unwrap(); + assert_eq!(&val, "lo"); + assert_eq!(id, v2::params::SubscriptionId::Num(my_sub.subscription_id())); + + // close the subscription to ensure it doesn't return any items. + my_sub.close(); + assert_eq!(None, my_sub.next::().await); + } } From 9c6fd4bfee44aec6ebb10dae0fb2779562ecf125 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Fri, 19 Nov 2021 19:30:47 +0100 Subject: [PATCH 18/31] feat: make it possible to override `method name` in subscriptions (#568) * feat: override `method` subscription notif * Arrow syntax for overwrites (#569) * check that unique notifs are used * check that custom sub name is unique * cargo fmt * address grumbles * Update proc-macros/src/rpc_macro.rs * commit added tests * Update proc-macros/src/render_server.rs Co-authored-by: David * Update proc-macros/src/render_server.rs Co-authored-by: David * Update proc-macros/src/rpc_macro.rs Co-authored-by: David * Update proc-macros/src/rpc_macro.rs Co-authored-by: David * Update utils/src/server/rpc_module.rs Co-authored-by: David * grumbles * fix long lines * Update utils/src/server/rpc_module.rs Co-authored-by: David * Update utils/src/server/rpc_module.rs Co-authored-by: David * Update proc-macros/src/rpc_macro.rs Co-authored-by: David * Update proc-macros/src/render_server.rs Co-authored-by: David * Update proc-macros/src/render_server.rs Co-authored-by: David * more grumbles Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Co-authored-by: David --- benches/helpers.rs | 2 +- examples/proc_macro.rs | 2 +- examples/ws_sub_with_params.rs | 4 +- examples/ws_subscription.rs | 2 +- proc-macros/src/attributes.rs | 39 +++++++++++++++---- proc-macros/src/render_server.rs | 12 +++++- proc-macros/src/rpc_macro.rs | 28 +++++++++++-- proc-macros/tests/ui/correct/basic.rs | 13 +++++++ .../ui/incorrect/sub/sub_dup_name_override.rs | 12 ++++++ .../sub/sub_dup_name_override.stderr | 5 +++ .../ui/incorrect/sub/sub_name_override.rs | 10 +++++ .../ui/incorrect/sub/sub_name_override.stderr | 5 +++ tests/tests/helpers.rs | 33 +++++++++------- tests/tests/integration_tests.rs | 2 +- tests/tests/proc_macros.rs | 2 +- utils/src/server/rpc_module.rs | 29 ++++++++++---- ws-server/src/tests.rs | 12 ++++-- 17 files changed, 167 insertions(+), 45 deletions(-) create mode 100644 proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.rs create mode 100644 proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr create mode 100644 proc-macros/tests/ui/incorrect/sub/sub_name_override.rs create mode 100644 proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr diff --git a/benches/helpers.rs b/benches/helpers.rs index f60fc122b6..dcd3de7394 100644 --- a/benches/helpers.rs +++ b/benches/helpers.rs @@ -101,7 +101,7 @@ pub async fn ws_server(handle: tokio::runtime::Handle) -> (String, jsonrpsee::ws module.register_method(SYNC_METHOD_NAME, |_, _| Ok("lo")).unwrap(); module.register_async_method(ASYNC_METHOD_NAME, |_, _| async { Ok("lo") }).unwrap(); module - .register_subscription(SUB_METHOD_NAME, UNSUB_METHOD_NAME, |_params, mut sink, _ctx| { + .register_subscription(SUB_METHOD_NAME, SUB_METHOD_NAME, UNSUB_METHOD_NAME, |_params, mut sink, _ctx| { let x = "Hello"; tokio::spawn(async move { sink.send(&x) }); Ok(()) diff --git a/examples/proc_macro.rs b/examples/proc_macro.rs index 11825834b3..f13a8b29b4 100644 --- a/examples/proc_macro.rs +++ b/examples/proc_macro.rs @@ -45,7 +45,7 @@ where async fn storage_keys(&self, storage_key: StorageKey, hash: Option) -> Result, Error>; /// Subscription that takes a `StorageKey` as input and produces a `Vec`. - #[subscription(name = "subscribeStorage", item = Vec)] + #[subscription(name = "subscribeStorage" => "override", item = Vec)] fn subscribe_storage(&self, keys: Option>) -> Result<(), Error>; } diff --git a/examples/ws_sub_with_params.rs b/examples/ws_sub_with_params.rs index 3c3c61c3d1..1275168d64 100644 --- a/examples/ws_sub_with_params.rs +++ b/examples/ws_sub_with_params.rs @@ -62,7 +62,7 @@ async fn run_server() -> anyhow::Result { let server = WsServerBuilder::default().build("127.0.0.1:0").await?; let mut module = RpcModule::new(()); module - .register_subscription("sub_one_param", "unsub_one_param", |params, mut sink, _| { + .register_subscription("sub_one_param", "sub_one_param", "unsub_one_param", |params, mut sink, _| { let idx: usize = params.one()?; std::thread::spawn(move || loop { let _ = sink.send(&LETTERS.chars().nth(idx)); @@ -72,7 +72,7 @@ async fn run_server() -> anyhow::Result { }) .unwrap(); module - .register_subscription("sub_params_two", "unsub_params_two", |params, mut sink, _| { + .register_subscription("sub_params_two", "params_two", "unsub_params_two", |params, mut sink, _| { let (one, two): (usize, usize) = params.parse()?; std::thread::spawn(move || loop { let _ = sink.send(&LETTERS[one..two].to_string()); diff --git a/examples/ws_subscription.rs b/examples/ws_subscription.rs index f9521992dc..4af06c9fa0 100644 --- a/examples/ws_subscription.rs +++ b/examples/ws_subscription.rs @@ -61,7 +61,7 @@ async fn main() -> anyhow::Result<()> { async fn run_server() -> anyhow::Result { let server = WsServerBuilder::default().build("127.0.0.1:0").await?; let mut module = RpcModule::new(()); - module.register_subscription("subscribe_hello", "unsubscribe_hello", |_, mut sink, _| { + module.register_subscription("subscribe_hello", "s_hello", "unsubscribe_hello", |_, mut sink, _| { std::thread::spawn(move || loop { if let Err(Error::SubscriptionClosed(_)) = sink.send(&"hello my friend") { return; diff --git a/proc-macros/src/attributes.rs b/proc-macros/src/attributes.rs index 02da827efa..a143a7a873 100644 --- a/proc-macros/src/attributes.rs +++ b/proc-macros/src/attributes.rs @@ -28,7 +28,7 @@ use proc_macro2::{Span, TokenStream as TokenStream2, TokenTree}; use std::{fmt, iter}; use syn::parse::{Parse, ParseStream, Parser}; use syn::punctuated::Punctuated; -use syn::{spanned::Spanned, Attribute, Error, Token}; +use syn::{spanned::Spanned, Attribute, Error, LitInt, LitStr, Token}; pub(crate) struct AttributeMeta { pub path: syn::Path, @@ -48,15 +48,22 @@ pub enum ParamKind { #[derive(Debug, Clone)] pub struct Resource { - pub name: syn::LitStr, + pub name: LitStr, pub assign: Token![=], - pub value: syn::LitInt, + pub value: LitInt, } -pub struct Aliases { - pub list: Punctuated, +pub struct NameMapping { + pub name: String, + pub mapped: Option, } +pub struct Bracketed { + pub list: Punctuated, +} + +pub type Aliases = Bracketed; + impl Parse for Argument { fn parse(input: ParseStream) -> syn::Result { let label = input.parse()?; @@ -91,7 +98,23 @@ impl Parse for Resource { } } -impl Parse for Aliases { +impl Parse for NameMapping { + fn parse(input: ParseStream) -> syn::Result { + let name = input.parse::()?.value(); + + let mapped = if input.peek(Token![=>]) { + input.parse::]>()?; + + Some(input.parse::()?.value()) + } else { + None + }; + + Ok(NameMapping { name, mapped }) + } +} + +impl Parse for Bracketed { fn parse(input: ParseStream) -> syn::Result { let content; @@ -99,7 +122,7 @@ impl Parse for Aliases { let list = content.parse_terminated(Parse::parse)?; - Ok(Aliases { list }) + Ok(Bracketed { list }) } } @@ -201,7 +224,7 @@ impl Argument { /// Asserts that the argument is `key = "string"` and gets the value of the string pub fn string(self) -> syn::Result { - self.value::().map(|lit| lit.value()) + self.value::().map(|lit| lit.value()) } } diff --git a/proc-macros/src/render_server.rs b/proc-macros/src/render_server.rs index 86b96cb757..68d4b7e816 100644 --- a/proc-macros/src/render_server.rs +++ b/proc-macros/src/render_server.rs @@ -174,6 +174,8 @@ impl RpcDescription { let rust_method_name = &sub.signature.sig.ident; // Name of the RPC method to subscribe to (e.g. `foo_sub`). let rpc_sub_name = self.rpc_identifier(&sub.name); + // Name of `method` in the subscription response. + let rpc_notif_name_override = sub.notif_name_override.as_ref().map(|m| self.rpc_identifier(m)); // Name of the RPC method to unsubscribe (e.g. `foo_sub`). let rpc_unsub_name = self.rpc_identifier(&sub.unsubscribe); // `parsing` is the code associated with parsing structure from the @@ -184,8 +186,16 @@ impl RpcDescription { check_name(&rpc_sub_name, rust_method_name.span()); check_name(&rpc_unsub_name, rust_method_name.span()); + let rpc_notif_name = match rpc_notif_name_override { + Some(notif) => { + check_name(¬if, rust_method_name.span()); + notif + } + None => rpc_sub_name.clone(), + }; + handle_register_result(quote! { - rpc.register_subscription(#rpc_sub_name, #rpc_unsub_name, |params, sink, context| { + rpc.register_subscription(#rpc_sub_name, #rpc_notif_name, #rpc_unsub_name, |params, sink, context| { #parsing context.as_ref().#rust_method_name(sink, #params_seq) }) diff --git a/proc-macros/src/rpc_macro.rs b/proc-macros/src/rpc_macro.rs index be251cae00..156f2f051b 100644 --- a/proc-macros/src/rpc_macro.rs +++ b/proc-macros/src/rpc_macro.rs @@ -27,7 +27,9 @@ //! Declaration of the JSON RPC generator procedural macros. use crate::{ - attributes::{optional, parse_param_kind, Aliases, Argument, AttributeMeta, MissingArgument, ParamKind, Resource}, + attributes::{ + optional, parse_param_kind, Aliases, Argument, AttributeMeta, MissingArgument, NameMapping, ParamKind, Resource, + }, helpers::extract_doc_comments, }; @@ -95,6 +97,13 @@ impl RpcMethod { #[derive(Debug, Clone)] pub struct RpcSubscription { pub name: String, + /// When subscribing to an RPC, users can override the content of the `method` field + /// in the JSON data sent to subscribers. + /// Each subscription thus has one method name to set up the subscription, + /// one to unsubscribe and, optionally, a third method name used to describe the + /// payload (aka "notification") sent back from the server to subscribers. + /// If no override is provided, the subscription method name is used. + pub notif_name_override: Option, pub docs: TokenStream2, pub unsubscribe: String, pub params: Vec<(syn::PatIdent, syn::Type)>, @@ -111,7 +120,9 @@ impl RpcSubscription { AttributeMeta::parse(attr)?.retain(["aliases", "item", "name", "param_kind", "unsubscribe_aliases"])?; let aliases = parse_aliases(aliases)?; - let name = name?.string()?; + let map = name?.value::()?; + let name = map.name; + let notif_name_override = map.mapped; let item = item?.value()?; let param_kind = parse_param_kind(param_kind)?; let unsubscribe_aliases = parse_aliases(unsubscribe_aliases)?; @@ -135,7 +146,18 @@ impl RpcSubscription { // We've analyzed attributes and don't need them anymore. sub.attrs.clear(); - Ok(Self { name, unsubscribe, unsubscribe_aliases, params, param_kind, item, signature: sub, aliases, docs }) + Ok(Self { + name, + notif_name_override, + unsubscribe, + unsubscribe_aliases, + params, + param_kind, + item, + signature: sub, + aliases, + docs, + }) } } diff --git a/proc-macros/tests/ui/correct/basic.rs b/proc-macros/tests/ui/correct/basic.rs index c8052ce288..05b062b88a 100644 --- a/proc-macros/tests/ui/correct/basic.rs +++ b/proc-macros/tests/ui/correct/basic.rs @@ -31,6 +31,11 @@ pub trait Rpc { #[subscription(name = "echo", aliases = ["ECHO"], item = u32, unsubscribe_aliases = ["NotInterested", "listenNoMore"])] fn sub_with_params(&self, val: u32) -> RpcResult<()>; + + // This will send data to subscribers with the `method` field in the JSON payload set to `foo_subscribe_override` + // because it's in the `foo` namespace. + #[subscription(name = "subscribe_method" => "subscribe_override", item = u32)] + fn sub_with_override_notif_method(&self) -> RpcResult<()>; } pub struct RpcServerImpl; @@ -68,6 +73,10 @@ impl RpcServer for RpcServerImpl { sink.send(&val)?; sink.send(&val) } + + fn sub_with_override_notif_method(&self, mut sink: SubscriptionSink) -> RpcResult<()> { + sink.send(&1) + } } pub async fn websocket_server() -> SocketAddr { @@ -102,4 +111,8 @@ async fn main() { assert_eq!(first_recv, Some("Response_A".to_string())); let second_recv = sub.next().await.unwrap(); assert_eq!(second_recv, Some("Response_B".to_string())); + + let mut sub = client.sub_with_override_notif_method().await.unwrap(); + let recv = sub.next().await.unwrap(); + assert_eq!(recv, Some(1)); } diff --git a/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.rs b/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.rs new file mode 100644 index 0000000000..53adf3fe2e --- /dev/null +++ b/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.rs @@ -0,0 +1,12 @@ +use jsonrpsee::{proc_macros::rpc, types::RpcResult}; + +// Subscription method must not use the same override name. +#[rpc(client, server)] +pub trait DupOverride { + #[subscription(name = "one" => "override", item = u8)] + fn one(&self) -> RpcResult<()>; + #[subscription(name = "two" => "override", item = u8)] + fn two(&self) -> RpcResult<()>; +} + +fn main() {} diff --git a/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr b/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr new file mode 100644 index 0000000000..a34210fe70 --- /dev/null +++ b/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr @@ -0,0 +1,5 @@ +error: "override" is already defined + --> tests/ui/incorrect/sub/sub_dup_name_override.rs:9:5 + | +9 | fn two(&self) -> RpcResult<()>; + | ^^^ diff --git a/proc-macros/tests/ui/incorrect/sub/sub_name_override.rs b/proc-macros/tests/ui/incorrect/sub/sub_name_override.rs new file mode 100644 index 0000000000..740b30699f --- /dev/null +++ b/proc-macros/tests/ui/incorrect/sub/sub_name_override.rs @@ -0,0 +1,10 @@ +use jsonrpsee::{proc_macros::rpc, types::RpcResult}; + +// Subscription method name conflict with notif override. +#[rpc(client, server)] +pub trait DupName { + #[subscription(name = "one" => "one", item = u8)] + fn one(&self) -> RpcResult<()>; +} + +fn main() {} diff --git a/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr b/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr new file mode 100644 index 0000000000..719b2e88cf --- /dev/null +++ b/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr @@ -0,0 +1,5 @@ +error: "one" is already defined + --> tests/ui/incorrect/sub/sub_name_override.rs:7:5 + | +7 | fn one(&self) -> RpcResult<()>; + | ^^^ diff --git a/tests/tests/helpers.rs b/tests/tests/helpers.rs index a9773522f4..7df1d923e6 100644 --- a/tests/tests/helpers.rs +++ b/tests/tests/helpers.rs @@ -40,7 +40,7 @@ pub async fn websocket_server_with_subscription() -> (SocketAddr, WsServerHandle module.register_method("say_hello", |_, _| Ok("hello")).unwrap(); module - .register_subscription("subscribe_hello", "unsubscribe_hello", |_, mut sink, _| { + .register_subscription("subscribe_hello", "subscribe_hello", "unsubscribe_hello", |_, mut sink, _| { std::thread::spawn(move || loop { if let Err(Error::SubscriptionClosed(_)) = sink.send(&"hello from subscription") { break; @@ -52,7 +52,7 @@ pub async fn websocket_server_with_subscription() -> (SocketAddr, WsServerHandle .unwrap(); module - .register_subscription("subscribe_foo", "unsubscribe_foo", |_, mut sink, _| { + .register_subscription("subscribe_foo", "subscribe_foo", "unsubscribe_foo", |_, mut sink, _| { std::thread::spawn(move || loop { if let Err(Error::SubscriptionClosed(_)) = sink.send(&1337) { break; @@ -64,21 +64,26 @@ pub async fn websocket_server_with_subscription() -> (SocketAddr, WsServerHandle .unwrap(); module - .register_subscription("subscribe_add_one", "unsubscribe_add_one", |params, mut sink, _| { - let mut count: usize = params.one()?; - std::thread::spawn(move || loop { - count = count.wrapping_add(1); - if let Err(Error::SubscriptionClosed(_)) = sink.send(&count) { - break; - } - std::thread::sleep(Duration::from_millis(100)); - }); - Ok(()) - }) + .register_subscription( + "subscribe_add_one", + "subscribe_add_one", + "unsubscribe_add_one", + |params, mut sink, _| { + let mut count: usize = params.one()?; + std::thread::spawn(move || loop { + count = count.wrapping_add(1); + if let Err(Error::SubscriptionClosed(_)) = sink.send(&count) { + break; + } + std::thread::sleep(Duration::from_millis(100)); + }); + Ok(()) + }, + ) .unwrap(); module - .register_subscription("subscribe_noop", "unsubscribe_noop", |_, mut sink, _| { + .register_subscription("subscribe_noop", "subscribe_noop", "unsubscribe_noop", |_, mut sink, _| { std::thread::spawn(move || { std::thread::sleep(Duration::from_secs(1)); sink.close("Server closed the stream because it was lazy") diff --git a/tests/tests/integration_tests.rs b/tests/tests/integration_tests.rs index 118c270a81..267a8205ca 100644 --- a/tests/tests/integration_tests.rs +++ b/tests/tests/integration_tests.rs @@ -332,7 +332,7 @@ async fn ws_server_should_stop_subscription_after_client_drop() { let mut module = RpcModule::new(tx); module - .register_subscription("subscribe_hello", "unsubscribe_hello", |_, mut sink, mut tx| { + .register_subscription("subscribe_hello", "subscribe_hello", "unsubscribe_hello", |_, mut sink, mut tx| { tokio::spawn(async move { let close_err = loop { if let Err(Error::SubscriptionClosed(err)) = sink.send(&1) { diff --git a/tests/tests/proc_macros.rs b/tests/tests/proc_macros.rs index 2597e8caf1..08d2ce9a64 100644 --- a/tests/tests/proc_macros.rs +++ b/tests/tests/proc_macros.rs @@ -312,7 +312,7 @@ async fn multiple_blocking_calls_overlap() { #[tokio::test] async fn subscriptions_do_not_work_for_http_servers() { - let htserver = HttpServerBuilder::default().build("127.0.0.1:0".parse().unwrap()).unwrap(); + let htserver = HttpServerBuilder::default().build("127.0.0.1:0").unwrap(); let addr = htserver.local_addr().unwrap(); let htserver_url = format!("http://{}", addr); let _handle = htserver.start(RpcServerImpl.into_rpc()).unwrap(); diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index 4652962c60..294b7f9a72 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -516,9 +516,20 @@ impl RpcModule { Ok(MethodResourcesBuilder { build: ResourceVec::new(), callback }) } - /// Register a new RPC subscription that invokes callback on every subscription request. - /// The callback itself takes three parameters: - /// - [`Params`]: JSONRPC parameters in the subscription request. + /// Register a new RPC subscription that invokes s callback on every subscription call. + /// + /// This method ensures that the `subscription_method_name` and `unsubscription_method_name` are unique. + /// The `notif_method_name` argument sets the content of the `method` field in the JSON document that + /// the server sends back to the client. The uniqueness of this value is not machine checked and it's up to + /// the user to ensure it is not used in any other [`RpcModule`] used in the server. + /// + /// # Arguments + /// + /// * `subscription_method_name` - name of the method to call to initiate a subscription + /// * `notif_method_name` - name of method to be used in the subscription payload (technically a JSON-RPC notification) + /// * `unsubscription_method` - name of the method to call to terminate a subscription + /// * `callback` - A callback to invoke on each subscription; it takes three parameters: + /// - [`Params`]: JSON-RPC parameters in the subscription call. /// - [`SubscriptionSink`]: A sink to send messages to the subscriber. /// - Context: Any type that can be embedded into the [`RpcModule`]. /// @@ -529,7 +540,7 @@ impl RpcModule { /// use jsonrpsee_utils::server::rpc_module::RpcModule; /// /// let mut ctx = RpcModule::new(99_usize); - /// ctx.register_subscription("sub", "unsub", |params, mut sink, ctx| { + /// ctx.register_subscription("sub", "notif_name", "unsub", |params, mut sink, ctx| { /// let x: usize = params.one()?; /// std::thread::spawn(move || { /// let sum = x + (*ctx); @@ -541,6 +552,7 @@ impl RpcModule { pub fn register_subscription( &mut self, subscribe_method_name: &'static str, + notif_method_name: &'static str, unsubscribe_method_name: &'static str, callback: F, ) -> Result<(), Error> @@ -554,6 +566,7 @@ impl RpcModule { self.methods.verify_method_name(subscribe_method_name)?; self.methods.verify_method_name(unsubscribe_method_name)?; + let ctx = self.ctx.clone(); let subscribers = Subscribers::default(); @@ -577,7 +590,7 @@ impl RpcModule { let sink = SubscriptionSink { inner: method_sink.clone(), - method: subscribe_method_name, + method: notif_method_name, subscribers: subscribers.clone(), uniq_sub: SubscriptionKey { conn_id, sub_id }, is_connected: Some(conn_tx), @@ -784,7 +797,7 @@ mod tests { fn rpc_context_modules_can_register_subscriptions() { let cx = (); let mut cxmodule = RpcModule::new(cx); - let _subscription = cxmodule.register_subscription("hi", "goodbye", |_, _, _| Ok(())); + let _subscription = cxmodule.register_subscription("hi", "hi", "goodbye", |_, _, _| Ok(())); assert!(cxmodule.method("hi").is_some()); assert!(cxmodule.method("goodbye").is_some()); @@ -922,7 +935,7 @@ mod tests { async fn subscribing_without_server() { let mut module = RpcModule::new(()); module - .register_subscription("my_sub", "my_unsub", |_, mut sink, _| { + .register_subscription("my_sub", "my_sub", "my_unsub", |_, mut sink, _| { let mut stream_data = vec!['0', '1', '2']; std::thread::spawn(move || loop { tracing::debug!("This is your friendly subscription sending data."); @@ -956,7 +969,7 @@ mod tests { async fn close_test_subscribing_without_server() { let mut module = RpcModule::new(()); module - .register_subscription("my_sub", "my_unsub", |_, mut sink, _| { + .register_subscription("my_sub", "my_sub", "my_unsub", |_, mut sink, _| { std::thread::spawn(move || loop { if let Err(Error::SubscriptionClosed(_)) = sink.send(&"lo") { return; diff --git a/ws-server/src/tests.rs b/ws-server/src/tests.rs index 121cbdf7fb..67e5f36f66 100644 --- a/ws-server/src/tests.rs +++ b/ws-server/src/tests.rs @@ -117,7 +117,7 @@ async fn server_with_handles() -> (SocketAddr, ServerHandle) { }) .unwrap(); module - .register_subscription("subscribe_hello", "unsubscribe_hello", |_, sink, _| { + .register_subscription("subscribe_hello", "subscribe_hello", "unsubscribe_hello", |_, sink, _| { std::thread::spawn(move || loop { let _ = sink; std::thread::sleep(std::time::Duration::from_secs(30)); @@ -472,8 +472,12 @@ async fn register_methods_works() { let mut module = RpcModule::new(()); assert!(module.register_method("say_hello", |_, _| Ok("lo")).is_ok()); assert!(module.register_method("say_hello", |_, _| Ok("lo")).is_err()); - assert!(module.register_subscription("subscribe_hello", "unsubscribe_hello", |_, _, _| Ok(())).is_ok()); - assert!(module.register_subscription("subscribe_hello_again", "unsubscribe_hello", |_, _, _| Ok(())).is_err()); + assert!(module + .register_subscription("subscribe_hello", "subscribe_hello", "unsubscribe_hello", |_, _, _| Ok(())) + .is_ok()); + assert!(module + .register_subscription("subscribe_hello_again", "subscribe_hello_again", "unsubscribe_hello", |_, _, _| Ok(())) + .is_err()); assert!( module.register_method("subscribe_hello_again", |_, _| Ok("lo")).is_ok(), "Failed register_subscription should not have side-effects" @@ -484,7 +488,7 @@ async fn register_methods_works() { async fn register_same_subscribe_unsubscribe_is_err() { let mut module = RpcModule::new(()); assert!(matches!( - module.register_subscription("subscribe_hello", "subscribe_hello", |_, _, _| Ok(())), + module.register_subscription("subscribe_hello", "subscribe_hello", "subscribe_hello", |_, _, _| Ok(())), Err(Error::SubscriptionNameConflict(_)) )); } From 9a3c1e981bcdbbb558b1457bbd78277a14dca2da Mon Sep 17 00:00:00 2001 From: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> Date: Sun, 21 Nov 2021 16:20:50 +0200 Subject: [PATCH 19/31] proc-macros: Support deprecated methods for rpc client (#570) * proc-macros: Fix documentation typo of `rpc_identifier` * proc-macros: Support deprecated methods for rpc client (#564) Calling a deprecated method of the RPC client should warn the user at compile-time. Extract the `#[deprecated]` macro as is while parsing the RpcMethod, and pass through the macro to the RPC client rendering. * tests/ui: Check deprecated method for rpc client (#564) To ensure that the test will fail during compilation, warnings are denied. Check that the deprecate macro will generate warnings just for the methods that are utilized. --- proc-macros/src/render_client.rs | 3 + proc-macros/src/rpc_macro.rs | 20 +++++- .../ui/incorrect/rpc/rpc_deprecated_method.rs | 67 +++++++++++++++++++ .../rpc/rpc_deprecated_method.stderr | 12 ++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.rs create mode 100644 proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.stderr diff --git a/proc-macros/src/render_client.rs b/proc-macros/src/render_client.rs index c655a261ab..8ec3020f29 100644 --- a/proc-macros/src/render_client.rs +++ b/proc-macros/src/render_client.rs @@ -98,9 +98,12 @@ impl RpcDescription { let parameters = self.encode_params(&method.params, &method.param_kind, &method.signature); // Doc-comment to be associated with the method. let docs = &method.docs; + // Mark the method as deprecated, if previously declared as so. + let deprecated = &method.deprecated; let method = quote! { #docs + #deprecated async fn #rust_method_name(#rust_method_params) -> #returns { self.#called_method(#rpc_method_name, #parameters).await } diff --git a/proc-macros/src/rpc_macro.rs b/proc-macros/src/rpc_macro.rs index 156f2f051b..cd63753a1e 100644 --- a/proc-macros/src/rpc_macro.rs +++ b/proc-macros/src/rpc_macro.rs @@ -44,6 +44,7 @@ pub struct RpcMethod { pub name: String, pub blocking: bool, pub docs: TokenStream2, + pub deprecated: TokenStream2, pub params: Vec<(syn::PatIdent, syn::Type)>, pub param_kind: ParamKind, pub returns: Option, @@ -65,6 +66,10 @@ impl RpcMethod { let sig = method.sig.clone(); let docs = extract_doc_comments(&method.attrs); + let deprecated = match find_attr(&method.attrs, "deprecated") { + Some(attr) => quote!(#attr), + None => quote!(), + }; if blocking && sig.asyncness.is_some() { return Err(syn::Error::new(sig.span(), "Blocking method must be synchronous")); @@ -90,7 +95,18 @@ impl RpcMethod { // We've analyzed attributes and don't need them anymore. method.attrs.clear(); - Ok(Self { aliases, blocking, name, params, param_kind, returns, signature: method, docs, resources }) + Ok(Self { + aliases, + blocking, + name, + params, + param_kind, + returns, + signature: method, + docs, + resources, + deprecated, + }) } } @@ -298,7 +314,7 @@ impl RpcDescription { /// Based on the namespace, renders the full name of the RPC method/subscription. /// Examples: /// For namespace `foo` and method `makeSpam`, result will be `foo_makeSpam`. - /// For no namespace and method `makeSpam` it will be just `makeSpam. + /// For no namespace and method `makeSpam` it will be just `makeSpam`. pub(crate) fn rpc_identifier<'a>(&self, method: &'a str) -> Cow<'a, str> { if let Some(ns) = &self.namespace { format!("{}_{}", ns, method).into() diff --git a/proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.rs b/proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.rs new file mode 100644 index 0000000000..79fdd01972 --- /dev/null +++ b/proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.rs @@ -0,0 +1,67 @@ +//! Test that calling a deprecated method will generate warnings at compile-time. + +// Treat warnings as errors to fail the build. +#![deny(warnings)] + +use jsonrpsee::{ + proc_macros::rpc, + types::{async_trait, RpcResult}, + ws_client::*, + ws_server::WsServerBuilder, +}; +use std::net::SocketAddr; + +#[rpc(client, server)] +pub trait Deprecated { + // Deprecated method that is called by the client. + #[deprecated(since = "0.5.0", note = "please use `new_method` instead")] + #[method(name = "foo")] + async fn async_method(&self) -> RpcResult; + + // Deprecated methods that are not called should not generate warnings. + #[deprecated(since = "0.5.0", note = "please use `new_method` instead")] + #[method(name = "foo_unused")] + async fn async_method_unused(&self) -> RpcResult; + + // If the method is not marked as deprecated, should not generate warnings. + #[method(name = "bar")] + fn sync_method(&self) -> RpcResult; +} + +pub struct DeprecatedServerImpl; + +#[async_trait] +impl DeprecatedServer for DeprecatedServerImpl { + async fn async_method(&self) -> RpcResult { + Ok(16u8) + } + + async fn async_method_unused(&self) -> RpcResult { + Ok(32u8) + } + + fn sync_method(&self) -> RpcResult { + Ok(64u8) + } +} + +pub async fn websocket_server() -> SocketAddr { + let server = WsServerBuilder::default().build("127.0.0.1:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + + server.start(DeprecatedServerImpl.into_rpc()).unwrap(); + + addr +} + +#[tokio::main] +async fn main() { + let server_addr = websocket_server().await; + let server_url = format!("ws://{}", server_addr); + let client = WsClientBuilder::default().build(&server_url).await.unwrap(); + + // Calling this method should generate an warning. + assert_eq!(client.async_method().await.unwrap(), 16); + // Note: `async_method_unused` is not called, and should not generate warnings. + assert_eq!(client.sync_method().await.unwrap(), 64); +} diff --git a/proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.stderr b/proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.stderr new file mode 100644 index 0000000000..0f80a6749a --- /dev/null +++ b/proc-macros/tests/ui/incorrect/rpc/rpc_deprecated_method.stderr @@ -0,0 +1,12 @@ +error: use of deprecated associated function `DeprecatedClient::async_method`: please use `new_method` instead + --> $DIR/rpc_deprecated_method.rs:64:20 + | +64 | assert_eq!(client.async_method().await.unwrap(), 16); + | ^^^^^^^^^^^^ + | +note: the lint level is defined here + --> $DIR/rpc_deprecated_method.rs:4:9 + | +4 | #![deny(warnings)] + | ^^^^^^^^ + = note: `#[deny(deprecated)]` implied by `#[deny(warnings)]` From e19e5051145c89f86ea02d01f52800cce9d1a516 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Nov 2021 13:57:06 +0000 Subject: [PATCH 20/31] Update hyper-rustls requirement from 0.22 to 0.23 (#571) * Update hyper-rustls requirement from 0.22 to 0.23 Updates the requirements on [hyper-rustls](https://github.com/ctz/hyper-rustls) to permit the latest version. - [Release notes](https://github.com/ctz/hyper-rustls/releases) - [Commits](https://github.com/ctz/hyper-rustls/compare/v/0.22.0...v/0.23.0) --- updated-dependencies: - dependency-name: hyper-rustls dependency-type: direct:production ... Signed-off-by: dependabot[bot] * make it work Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Niklas Adolfsson --- http-client/Cargo.toml | 2 +- http-client/src/transport.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index 44f0feee5a..505f5e2b9f 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -13,7 +13,7 @@ documentation = "https://docs.rs/jsonrpsee-http-client" async-trait = "0.1" fnv = "1" hyper = { version = "0.14.10", features = ["client", "http1", "http2", "tcp"] } -hyper-rustls = { version = "0.22", features = ["webpki-tokio"] } +hyper-rustls = { version = "0.23", features = ["webpki-tokio"] } jsonrpsee-types = { path = "../types", version = "0.4.1" } jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["client", "http-helpers"] } serde = { version = "1.0", default-features = false, features = ["derive"] } diff --git a/http-client/src/transport.rs b/http-client/src/transport.rs index e46b3a597b..b512e6749d 100644 --- a/http-client/src/transport.rs +++ b/http-client/src/transport.rs @@ -8,7 +8,7 @@ use crate::types::error::GenericTransportError; use hyper::client::{Client, HttpConnector}; -use hyper_rustls::HttpsConnector; +use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; use jsonrpsee_types::CertificateStore; use jsonrpsee_utils::http_helpers; use thiserror::Error; @@ -36,11 +36,15 @@ impl HttpTransportClient { let target = url::Url::parse(target.as_ref()).map_err(|e| Error::Url(format!("Invalid URL: {}", e)))?; if target.scheme() == "http" || target.scheme() == "https" { let connector = match cert_store { - CertificateStore::Native => HttpsConnector::with_native_roots(), - CertificateStore::WebPki => HttpsConnector::with_webpki_roots(), + CertificateStore::Native => { + HttpsConnectorBuilder::new().with_native_roots().https_or_http().enable_http1() + } + CertificateStore::WebPki => { + HttpsConnectorBuilder::new().with_webpki_roots().https_or_http().enable_http1() + } _ => return Err(Error::InvalidCertficateStore), }; - let client = Client::builder().build::<_, hyper::Body>(connector); + let client = Client::builder().build::<_, hyper::Body>(connector.build()); Ok(HttpTransportClient { target, client, max_request_body_size }) } else { Err(Error::Url("URL scheme not supported, expects 'http' or 'https'".into())) From 085df4144e87be2a0ec547d12cbe390d90a8b038 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Tue, 23 Nov 2021 20:24:24 +0100 Subject: [PATCH 21/31] fix: better log for failed unsubscription call (#575) --- utils/src/server/rpc_module.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index 294b7f9a72..eca57ef0b9 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -616,8 +616,9 @@ impl RpcModule { Ok(sub_id) => sub_id, Err(_) => { tracing::error!( - "unsubscribe call '{}' failed: couldn't parse subscription id, request id={:?}", + "unsubscribe call '{}' failed: couldn't parse subscription id={:?} request id={:?}", unsubscribe_method_name, + params, id ); let err = to_json_raw_value(&"Invalid subscription ID type, must be integer").ok(); From 42ffbcc608afce97af4e8b394fb9d31920888346 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 24 Nov 2021 10:54:16 +0100 Subject: [PATCH 22/31] [chore] Release v0.5 (#574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version –> 0.5 Fix try-build tests * Changelog * Update CHANGELOG.md Co-authored-by: Niklas Adolfsson * Update CHANGELOG.md Co-authored-by: Niklas Adolfsson --- CHANGELOG.md | 36 +++++++++++++++++++ benches/Cargo.toml | 2 +- examples/Cargo.toml | 2 +- http-client/Cargo.toml | 6 ++-- http-server/Cargo.toml | 6 ++-- jsonrpsee/Cargo.toml | 16 ++++----- proc-macros/Cargo.toml | 2 +- .../method/method_unexpected_field.stderr | 2 +- .../sub/sub_dup_name_override.stderr | 2 +- .../ui/incorrect/sub/sub_name_override.stderr | 2 +- .../sub/sub_unsupported_field.stderr | 2 +- test-utils/Cargo.toml | 2 +- tests/Cargo.toml | 2 +- types/Cargo.toml | 2 +- utils/Cargo.toml | 4 +-- ws-client/Cargo.toml | 4 +-- ws-server/Cargo.toml | 6 ++-- 17 files changed, 67 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce1ebef086..53f96ed82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,42 @@ The format is based on [Keep a Changelog]. ## [Unreleased] +## [v0.5.0] – 2021-11-23 + +v0.5 is a breaking release + +### [Added] + +- Add register_blocking_method [#523](https://github.com/paritytech/jsonrpsee/pull/523) +- Re-introduce object param parsing [#526](https://github.com/paritytech/jsonrpsee/pull/526) +- clients: add support for webpki and native certificate stores [#533](https://github.com/paritytech/jsonrpsee/pull/533) +- feat(ws client): support custom headers. [#535](https://github.com/paritytech/jsonrpsee/pull/535) +- Proc macro support for map param [#544](https://github.com/paritytech/jsonrpsee/pull/544) +- feat: make it possible to try several sockaddrs when starting server [#567](https://github.com/paritytech/jsonrpsee/pull/567) +- feat: make it possible to override method name in subscriptions [#568](https://github.com/paritytech/jsonrpsee/pull/568) +- proc-macros: Support deprecated methods for rpc client [#570](https://github.com/paritytech/jsonrpsee/pull/570) + +### [Change] + +- DRY error handling for methods [#515](https://github.com/paritytech/jsonrpsee/pull/515) +- deps: replace log with tracing [#525](https://github.com/paritytech/jsonrpsee/pull/525) +- benches: add option to run benchmarks against jsonrpc crate servers [#527](https://github.com/paritytech/jsonrpsee/pull/527) +- clients: request ID as RAII guard [#543](https://github.com/paritytech/jsonrpsee/pull/543) +- Allow awaiting on server handles [#550](https://github.com/paritytech/jsonrpsee/pull/550) +- ws server: reject too big response [#553](https://github.com/paritytech/jsonrpsee/pull/553) +- Array syntax aliases [#557](https://github.com/paritytech/jsonrpsee/pull/557) +- rpc module: report error on invalid subscription [#561](https://github.com/paritytech/jsonrpsee/pull/561) +- [rpc module]: improve TestSubscription to return None when closed [#566](https://github.com/paritytech/jsonrpsee/pull/566) + +### [Fixed] + +- ws server: respect max limit for received messages [#537](https://github.com/paritytech/jsonrpsee/pull/537) +- fix(ws server): batch wait until all methods has been executed. [#542](https://github.com/paritytech/jsonrpsee/pull/542) +- Re-export tracing for macros [#555](https://github.com/paritytech/jsonrpsee/pull/555) +- Periodically wake DriverSelect so we can poll whether or not stop had been called. [#556](https://github.com/paritytech/jsonrpsee/pull/556) +- Implement SubscriptionClient for HttpClient [#563](https://github.com/paritytech/jsonrpsee/pull/563) +- fix: better log for failed unsubscription call [#575](https://github.com/paritytech/jsonrpsee/pull/575) + ## [v0.4.1] – 2021-10-12 The v0.4.1 release is a bug fix. diff --git a/benches/Cargo.toml b/benches/Cargo.toml index e2e3b02871..f4020ebb11 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-benchmarks" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies "] description = "Benchmarks for jsonrpsee" edition = "2018" diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 7eb987a911..bb9dbddbff 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-examples" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies "] description = "Examples for jsonrpsee" edition = "2018" diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index 505f5e2b9f..988e79d4f3 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-http-client" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "HTTP client for JSON-RPC" edition = "2018" @@ -14,8 +14,8 @@ async-trait = "0.1" fnv = "1" hyper = { version = "0.14.10", features = ["client", "http1", "http2", "tcp"] } hyper-rustls = { version = "0.23", features = ["webpki-tokio"] } -jsonrpsee-types = { path = "../types", version = "0.4.1" } -jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["client", "http-helpers"] } +jsonrpsee-types = { path = "../types", version = "0.5.0" } +jsonrpsee-utils = { path = "../utils", version = "0.5.0", features = ["client", "http-helpers"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" thiserror = "1.0" diff --git a/http-server/Cargo.toml b/http-server/Cargo.toml index b2bf840b77..937718b35e 100644 --- a/http-server/Cargo.toml +++ b/http-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-http-server" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "HTTP server for JSON-RPC" edition = "2018" @@ -13,8 +13,8 @@ documentation = "https://docs.rs/jsonrpsee-http-server" hyper = { version = "0.14.10", features = ["server", "http1", "http2", "tcp"] } futures-channel = "0.3.14" futures-util = { version = "0.3.14", default-features = false } -jsonrpsee-types = { path = "../types", version = "0.4.1" } -jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["server", "http-helpers"] } +jsonrpsee-types = { path = "../types", version = "0.5.0" } +jsonrpsee-utils = { path = "../utils", version = "0.5.0", features = ["server", "http-helpers"] } globset = "0.4" lazy_static = "1.4" tracing = "0.1" diff --git a/jsonrpsee/Cargo.toml b/jsonrpsee/Cargo.toml index 79c4a5f94c..317e2b34ef 100644 --- a/jsonrpsee/Cargo.toml +++ b/jsonrpsee/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "jsonrpsee" description = "JSON-RPC crate" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies ", "Pierre Krieger "] license = "MIT" edition = "2018" @@ -12,13 +12,13 @@ documentation = "https://docs.rs/jsonrpsee" [dependencies] # No support for namespaced features yet so workspace dependencies are prefixed with `jsonrpsee-`. # See https://github.com/rust-lang/cargo/issues/5565 for more details. -jsonrpsee-http-client = { path = "../http-client", version = "0.4.1", package = "jsonrpsee-http-client", optional = true } -jsonrpsee-http-server = { path = "../http-server", version = "0.4.1", package = "jsonrpsee-http-server", optional = true } -jsonrpsee-ws-client = { path = "../ws-client", version = "0.4.1", package = "jsonrpsee-ws-client", optional = true } -jsonrpsee-ws-server = { path = "../ws-server", version = "0.4.1", package = "jsonrpsee-ws-server", optional = true } -jsonrpsee-proc-macros = { path = "../proc-macros", version = "0.4.1", package = "jsonrpsee-proc-macros", optional = true } -jsonrpsee-utils = { path = "../utils", version = "0.4.1", package = "jsonrpsee-utils", optional = true } -jsonrpsee-types = { path = "../types", version = "0.4.1", package = "jsonrpsee-types", optional = true } +jsonrpsee-http-client = { path = "../http-client", version = "0.5.0", package = "jsonrpsee-http-client", optional = true } +jsonrpsee-http-server = { path = "../http-server", version = "0.5.0", package = "jsonrpsee-http-server", optional = true } +jsonrpsee-ws-client = { path = "../ws-client", version = "0.5.0", package = "jsonrpsee-ws-client", optional = true } +jsonrpsee-ws-server = { path = "../ws-server", version = "0.5.0", package = "jsonrpsee-ws-server", optional = true } +jsonrpsee-proc-macros = { path = "../proc-macros", version = "0.5.0", package = "jsonrpsee-proc-macros", optional = true } +jsonrpsee-utils = { path = "../utils", version = "0.5.0", package = "jsonrpsee-utils", optional = true } +jsonrpsee-types = { path = "../types", version = "0.5.0", package = "jsonrpsee-types", optional = true } [features] http-client = ["jsonrpsee-http-client", "jsonrpsee-types", "jsonrpsee-utils/client"] diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml index 05bf1e5787..5d2287ab64 100644 --- a/proc-macros/Cargo.toml +++ b/proc-macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "jsonrpsee-proc-macros" description = "Procedueral macros for jsonrpsee" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies ", "Pierre Krieger "] license = "MIT" edition = "2018" diff --git a/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr b/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr index 81b031b034..57c82ce5eb 100644 --- a/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr +++ b/proc-macros/tests/ui/incorrect/method/method_unexpected_field.stderr @@ -1,5 +1,5 @@ error: Unknown argument `magic`, expected one of: `aliases`, `blocking`, `name`, `param_kind`, `resources` - --> tests/ui/incorrect/method/method_unexpected_field.rs:6:25 + --> $DIR/method_unexpected_field.rs:6:25 | 6 | #[method(name = "foo", magic = false)] | ^^^^^ diff --git a/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr b/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr index a34210fe70..45e3a50301 100644 --- a/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr +++ b/proc-macros/tests/ui/incorrect/sub/sub_dup_name_override.stderr @@ -1,5 +1,5 @@ error: "override" is already defined - --> tests/ui/incorrect/sub/sub_dup_name_override.rs:9:5 + --> $DIR/sub_dup_name_override.rs:9:5 | 9 | fn two(&self) -> RpcResult<()>; | ^^^ diff --git a/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr b/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr index 719b2e88cf..0a46b0bcd0 100644 --- a/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr +++ b/proc-macros/tests/ui/incorrect/sub/sub_name_override.stderr @@ -1,5 +1,5 @@ error: "one" is already defined - --> tests/ui/incorrect/sub/sub_name_override.rs:7:5 + --> $DIR/sub_name_override.rs:7:5 | 7 | fn one(&self) -> RpcResult<()>; | ^^^ diff --git a/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr b/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr index 87e90136fe..d0613d1c12 100644 --- a/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr +++ b/proc-macros/tests/ui/incorrect/sub/sub_unsupported_field.stderr @@ -1,5 +1,5 @@ error: Unknown argument `magic`, expected one of: `aliases`, `item`, `name`, `param_kind`, `unsubscribe_aliases` - --> tests/ui/incorrect/sub/sub_unsupported_field.rs:6:42 + --> $DIR/sub_unsupported_field.rs:6:42 | 6 | #[subscription(name = "sub", item = u8, magic = true)] | ^^^^^ diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index bf9ff16001..ddcb9742d8 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-test-utils" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies "] license = "MIT" edition = "2018" diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 1a450dedf2..0a583376af 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-integration-tests" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies "] description = "Integration tests for jsonrpsee" edition = "2018" diff --git a/types/Cargo.toml b/types/Cargo.toml index 9b4fffcae6..b844464367 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-types" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies "] description = "Shared types for jsonrpsee" edition = "2018" diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 25cebb8825..89221d4149 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-utils" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies "] description = "Utilities for jsonrpsee" edition = "2018" @@ -13,7 +13,7 @@ thiserror = { version = "1", optional = true } futures-channel = { version = "0.3.14", default-features = false, optional = true } futures-util = { version = "0.3.14", default-features = false, optional = true } hyper = { version = "0.14.10", default-features = false, features = ["stream"], optional = true } -jsonrpsee-types = { path = "../types", version = "0.4.1", optional = true } +jsonrpsee-types = { path = "../types", version = "0.5.0", optional = true } tracing = { version = "0.1", optional = true } rustc-hash = { version = "1", optional = true } rand = { version = "0.8", optional = true } diff --git a/ws-client/Cargo.toml b/ws-client/Cargo.toml index b528b140b0..1da3069564 100644 --- a/ws-client/Cargo.toml +++ b/ws-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-ws-client" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "WebSocket client for JSON-RPC" edition = "2018" @@ -14,7 +14,7 @@ async-trait = "0.1" fnv = "1" futures = { version = "0.3.14", default-features = false, features = ["std"] } http = "0.2" -jsonrpsee-types = { path = "../types", version = "0.4.1" } +jsonrpsee-types = { path = "../types", version = "0.5.0" } pin-project = "1" rustls-native-certs = "0.6.0" serde = "1" diff --git a/ws-server/Cargo.toml b/ws-server/Cargo.toml index a0f21f5010..e310be2c71 100644 --- a/ws-server/Cargo.toml +++ b/ws-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-ws-server" -version = "0.4.1" +version = "0.5.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "WebSocket server for JSON-RPC" edition = "2018" @@ -12,8 +12,8 @@ documentation = "https://docs.rs/jsonrpsee-ws-server" [dependencies] futures-channel = "0.3.14" futures-util = { version = "0.3.14", default-features = false, features = ["io", "async-await-macro"] } -jsonrpsee-types = { path = "../types", version = "0.4.1" } -jsonrpsee-utils = { path = "../utils", version = "0.4.1", features = ["server"] } +jsonrpsee-types = { path = "../types", version = "0.5.0" } +jsonrpsee-utils = { path = "../utils", version = "0.5.0", features = ["server"] } tracing = "0.1" serde_json = { version = "1", features = ["raw_value"] } soketto = "0.7.1" From 6bcd60b060a1d289a031d841ef64b046afb7b2d1 Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko <85877331+sergejparity@users.noreply.github.com> Date: Thu, 25 Nov 2021 10:55:05 +0200 Subject: [PATCH 23/31] Add `CODEOWNERS` file (#572) * Add CODEOWNERS file * Fix typo * Add tools-team as main codeowner --- CODEOWNERS | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000..cf22bb7af4 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,25 @@ +# Lists some code owners. +# +# A codeowner just oversees some part of the codebase. If an owned file is changed then the +# corresponding codeowner receives a review request. An approval of the codeowner might be +# required for merging a PR (depends on repository settings). +# +# For details about syntax, see: +# https://help.github.com/en/articles/about-code-owners +# But here are some important notes: +# +# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` +# which can be everywhere. +# - Multiple owners are supported. +# - Either handle (e.g, @github_user or @github_org/team) or email can be used. Keep in mind, +# that handles might work better because they are more recognizable on GitHub, +# you can use them for mentioning unlike an email. +# - The latest matching rule, if multiple, takes precedence. + +# main codeowner @paritytech/tools-team +* @paritytech/tools-team + +# CI +/.github/ @paritytech/ci @paritytech/tools-team +/.scripts/ci/ @paritytech/ci @paritytech/tools-team +/.gitlab-ci.yml @paritytech/ci @paritytech/tools-team From d4e53f83c06bc2a477735f7cd9b6e18f311787dd Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Thu, 25 Nov 2021 20:15:57 +0100 Subject: [PATCH 24/31] fix rpc error: support unquoted strings (#578) * fix rpc error: support unquoted strings * Update types/src/v2/error.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> --- types/src/v2/error.rs | 43 +++++++++++++++++++++++++++++++------ utils/src/server/helpers.rs | 4 ++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/types/src/v2/error.rs b/types/src/v2/error.rs index 27380e2891..c5e1dc1ddd 100644 --- a/types/src/v2/error.rs +++ b/types/src/v2/error.rs @@ -25,6 +25,7 @@ // DEALINGS IN THE SOFTWARE. use crate::v2::params::{Id, TwoPointZero}; +use beef::Cow; use serde::de::Deserializer; use serde::ser::Serializer; use serde::{Deserialize, Serialize}; @@ -64,7 +65,8 @@ pub struct ErrorObject<'a> { /// Code pub code: ErrorCode, /// Message - pub message: &'a str, + #[serde(borrow)] + pub message: Cow<'a, str>, /// Optional data #[serde(skip_serializing_if = "Option::is_none")] #[serde(borrow)] @@ -74,13 +76,13 @@ pub struct ErrorObject<'a> { impl<'a> ErrorObject<'a> { /// Create a new `ErrorObject` with optional data. pub fn new(code: ErrorCode, data: Option<&'a RawValue>) -> ErrorObject<'a> { - Self { code, message: code.message(), data } + Self { code, message: code.message().into(), data } } } impl<'a> From for ErrorObject<'a> { fn from(code: ErrorCode) -> Self { - Self { code, message: code.message(), data: None } + Self { code, message: code.message().into(), data: None } } } @@ -242,7 +244,7 @@ mod tests { let ser = r#"{"jsonrpc":"2.0","error":{"code":-32700,"message":"Parse error"},"id":null}"#; let exp = RpcError { jsonrpc: TwoPointZero, - error: ErrorObject { code: ErrorCode::ParseError, message: "Parse error", data: None }, + error: ErrorObject { code: ErrorCode::ParseError, message: "Parse error".into(), data: None }, id: Id::Null, }; let err: RpcError = serde_json::from_str(ser).unwrap(); @@ -255,19 +257,48 @@ mod tests { let data = serde_json::value::to_raw_value(&"vegan").unwrap(); let exp = RpcError { jsonrpc: TwoPointZero, - error: ErrorObject { code: ErrorCode::ParseError, message: "Parse error", data: Some(&*data) }, + error: ErrorObject { code: ErrorCode::ParseError, message: "Parse error".into(), data: Some(&*data) }, id: Id::Null, }; let err: RpcError = serde_json::from_str(ser).unwrap(); assert_eq!(exp, err); } + #[test] + fn deserialized_error_with_quoted_str() { + let raw = r#"{ + "error": { + "code": 1002, + "message": "desc: \"Could not decode `ChargeAssetTxPayment::asset_id`\" } })", + "data": "\\\"validate_transaction\\\"" + }, + "id": 7, + "jsonrpc": "2.0" + }"#; + let err: RpcError = serde_json::from_str(raw).unwrap(); + + let data = serde_json::value::to_raw_value(&"\\\"validate_transaction\\\"").unwrap(); + + assert_eq!( + err, + RpcError { + error: ErrorObject { + code: 1002.into(), + message: "desc: \"Could not decode `ChargeAssetTxPayment::asset_id`\" } })".into(), + data: Some(&*data), + }, + id: Id::Number(7), + jsonrpc: TwoPointZero, + } + ); + } + #[test] fn serialize_works() { let exp = r#"{"jsonrpc":"2.0","error":{"code":-32603,"message":"Internal error"},"id":1337}"#; let err = RpcError { jsonrpc: TwoPointZero, - error: ErrorObject { code: ErrorCode::InternalError, message: "Internal error", data: None }, + error: ErrorObject { code: ErrorCode::InternalError, message: "Internal error".into(), data: None }, id: Id::Number(1337), }; let ser = serde_json::to_string(&err).unwrap(); diff --git a/utils/src/server/helpers.rs b/utils/src/server/helpers.rs index 99d2256f62..c6cfd1d212 100644 --- a/utils/src/server/helpers.rs +++ b/utils/src/server/helpers.rs @@ -98,7 +98,7 @@ pub fn send_response(id: Id, tx: &MethodSink, result: impl Serialize, max_respon let data = to_json_raw_value(&format!("Exceeded max limit {}", max_response_size)).ok(); let err = ErrorObject { code: ErrorCode::ServerError(OVERSIZED_RESPONSE_CODE), - message: OVERSIZED_RESPONSE_MSG, + message: OVERSIZED_RESPONSE_MSG.into(), data: data.as_deref(), }; return send_error(id, tx, err); @@ -140,7 +140,7 @@ pub fn send_call_error(id: Id, tx: &MethodSink, err: Error) { e => (ErrorCode::ServerError(UNKNOWN_ERROR_CODE), e.to_string(), None), }; - let err = ErrorObject { code, message: &message, data: data.as_deref() }; + let err = ErrorObject { code, message: message.into(), data: data.as_deref() }; send_error(id, tx, err) } From 8c8676999ea3ddc74ef907b1f27858405750c17f Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Fri, 26 Nov 2021 09:41:25 +0100 Subject: [PATCH 25/31] chore: release v0.5.1 (#579) --- CHANGELOG.md | 8 ++++++++ benches/Cargo.toml | 2 +- examples/Cargo.toml | 2 +- http-client/Cargo.toml | 6 +++--- http-server/Cargo.toml | 6 +++--- jsonrpsee/Cargo.toml | 16 ++++++++-------- proc-macros/Cargo.toml | 2 +- test-utils/Cargo.toml | 2 +- tests/Cargo.toml | 2 +- types/Cargo.toml | 2 +- utils/Cargo.toml | 4 ++-- ws-client/Cargo.toml | 4 ++-- ws-server/Cargo.toml | 6 +++--- 13 files changed, 35 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53f96ed82d..8aa70eae94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog]. ## [Unreleased] +## [v0.5.1] – 2021-11-26 + +The v0.5.1 release is a bug fix. + +### [Fixed] + +- rpc error: support escaped strings [#578](https://github.com/paritytech/jsonrpsee/pull/578) + ## [v0.5.0] – 2021-11-23 v0.5 is a breaking release diff --git a/benches/Cargo.toml b/benches/Cargo.toml index f4020ebb11..8526386980 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-benchmarks" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies "] description = "Benchmarks for jsonrpsee" edition = "2018" diff --git a/examples/Cargo.toml b/examples/Cargo.toml index bb9dbddbff..6428b298ea 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-examples" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies "] description = "Examples for jsonrpsee" edition = "2018" diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index 988e79d4f3..47d154e8ae 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-http-client" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies ", "Pierre Krieger "] description = "HTTP client for JSON-RPC" edition = "2018" @@ -14,8 +14,8 @@ async-trait = "0.1" fnv = "1" hyper = { version = "0.14.10", features = ["client", "http1", "http2", "tcp"] } hyper-rustls = { version = "0.23", features = ["webpki-tokio"] } -jsonrpsee-types = { path = "../types", version = "0.5.0" } -jsonrpsee-utils = { path = "../utils", version = "0.5.0", features = ["client", "http-helpers"] } +jsonrpsee-types = { path = "../types", version = "0.5.1" } +jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["client", "http-helpers"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" thiserror = "1.0" diff --git a/http-server/Cargo.toml b/http-server/Cargo.toml index 937718b35e..5434457125 100644 --- a/http-server/Cargo.toml +++ b/http-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-http-server" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies ", "Pierre Krieger "] description = "HTTP server for JSON-RPC" edition = "2018" @@ -13,8 +13,8 @@ documentation = "https://docs.rs/jsonrpsee-http-server" hyper = { version = "0.14.10", features = ["server", "http1", "http2", "tcp"] } futures-channel = "0.3.14" futures-util = { version = "0.3.14", default-features = false } -jsonrpsee-types = { path = "../types", version = "0.5.0" } -jsonrpsee-utils = { path = "../utils", version = "0.5.0", features = ["server", "http-helpers"] } +jsonrpsee-types = { path = "../types", version = "0.5.1" } +jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["server", "http-helpers"] } globset = "0.4" lazy_static = "1.4" tracing = "0.1" diff --git a/jsonrpsee/Cargo.toml b/jsonrpsee/Cargo.toml index 317e2b34ef..e06da8206c 100644 --- a/jsonrpsee/Cargo.toml +++ b/jsonrpsee/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "jsonrpsee" description = "JSON-RPC crate" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies ", "Pierre Krieger "] license = "MIT" edition = "2018" @@ -12,13 +12,13 @@ documentation = "https://docs.rs/jsonrpsee" [dependencies] # No support for namespaced features yet so workspace dependencies are prefixed with `jsonrpsee-`. # See https://github.com/rust-lang/cargo/issues/5565 for more details. -jsonrpsee-http-client = { path = "../http-client", version = "0.5.0", package = "jsonrpsee-http-client", optional = true } -jsonrpsee-http-server = { path = "../http-server", version = "0.5.0", package = "jsonrpsee-http-server", optional = true } -jsonrpsee-ws-client = { path = "../ws-client", version = "0.5.0", package = "jsonrpsee-ws-client", optional = true } -jsonrpsee-ws-server = { path = "../ws-server", version = "0.5.0", package = "jsonrpsee-ws-server", optional = true } -jsonrpsee-proc-macros = { path = "../proc-macros", version = "0.5.0", package = "jsonrpsee-proc-macros", optional = true } -jsonrpsee-utils = { path = "../utils", version = "0.5.0", package = "jsonrpsee-utils", optional = true } -jsonrpsee-types = { path = "../types", version = "0.5.0", package = "jsonrpsee-types", optional = true } +jsonrpsee-http-client = { path = "../http-client", version = "0.5.1", package = "jsonrpsee-http-client", optional = true } +jsonrpsee-http-server = { path = "../http-server", version = "0.5.1", package = "jsonrpsee-http-server", optional = true } +jsonrpsee-ws-client = { path = "../ws-client", version = "0.5.1", package = "jsonrpsee-ws-client", optional = true } +jsonrpsee-ws-server = { path = "../ws-server", version = "0.5.1", package = "jsonrpsee-ws-server", optional = true } +jsonrpsee-proc-macros = { path = "../proc-macros", version = "0.5.1", package = "jsonrpsee-proc-macros", optional = true } +jsonrpsee-utils = { path = "../utils", version = "0.5.1", package = "jsonrpsee-utils", optional = true } +jsonrpsee-types = { path = "../types", version = "0.5.1", package = "jsonrpsee-types", optional = true } [features] http-client = ["jsonrpsee-http-client", "jsonrpsee-types", "jsonrpsee-utils/client"] diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml index 5d2287ab64..4d5baf7d83 100644 --- a/proc-macros/Cargo.toml +++ b/proc-macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "jsonrpsee-proc-macros" description = "Procedueral macros for jsonrpsee" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies ", "Pierre Krieger "] license = "MIT" edition = "2018" diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index ddcb9742d8..15df549efe 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-test-utils" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies "] license = "MIT" edition = "2018" diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 0a583376af..516498d72b 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-integration-tests" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies "] description = "Integration tests for jsonrpsee" edition = "2018" diff --git a/types/Cargo.toml b/types/Cargo.toml index b844464367..381abdba50 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-types" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies "] description = "Shared types for jsonrpsee" edition = "2018" diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 89221d4149..af49be62d8 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-utils" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies "] description = "Utilities for jsonrpsee" edition = "2018" @@ -13,7 +13,7 @@ thiserror = { version = "1", optional = true } futures-channel = { version = "0.3.14", default-features = false, optional = true } futures-util = { version = "0.3.14", default-features = false, optional = true } hyper = { version = "0.14.10", default-features = false, features = ["stream"], optional = true } -jsonrpsee-types = { path = "../types", version = "0.5.0", optional = true } +jsonrpsee-types = { path = "../types", version = "0.5.1", optional = true } tracing = { version = "0.1", optional = true } rustc-hash = { version = "1", optional = true } rand = { version = "0.8", optional = true } diff --git a/ws-client/Cargo.toml b/ws-client/Cargo.toml index 1da3069564..59b56dd8dc 100644 --- a/ws-client/Cargo.toml +++ b/ws-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-ws-client" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies ", "Pierre Krieger "] description = "WebSocket client for JSON-RPC" edition = "2018" @@ -14,7 +14,7 @@ async-trait = "0.1" fnv = "1" futures = { version = "0.3.14", default-features = false, features = ["std"] } http = "0.2" -jsonrpsee-types = { path = "../types", version = "0.5.0" } +jsonrpsee-types = { path = "../types", version = "0.5.1" } pin-project = "1" rustls-native-certs = "0.6.0" serde = "1" diff --git a/ws-server/Cargo.toml b/ws-server/Cargo.toml index e310be2c71..b31466871a 100644 --- a/ws-server/Cargo.toml +++ b/ws-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-ws-server" -version = "0.5.0" +version = "0.5.1" authors = ["Parity Technologies ", "Pierre Krieger "] description = "WebSocket server for JSON-RPC" edition = "2018" @@ -12,8 +12,8 @@ documentation = "https://docs.rs/jsonrpsee-ws-server" [dependencies] futures-channel = "0.3.14" futures-util = { version = "0.3.14", default-features = false, features = ["io", "async-await-macro"] } -jsonrpsee-types = { path = "../types", version = "0.5.0" } -jsonrpsee-utils = { path = "../utils", version = "0.5.0", features = ["server"] } +jsonrpsee-types = { path = "../types", version = "0.5.1" } +jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["server"] } tracing = "0.1" serde_json = { version = "1", features = ["raw_value"] } soketto = "0.7.1" From 15b2f23858b06b6162b6821a7bbf0086f68c5eba Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Mon, 29 Nov 2021 22:30:34 +0100 Subject: [PATCH 26/31] fix(http client): impl Clone (#583) --- http-client/src/client.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/http-client/src/client.rs b/http-client/src/client.rs index 300c248d5e..7d28994ea4 100644 --- a/http-client/src/client.rs +++ b/http-client/src/client.rs @@ -33,7 +33,7 @@ use crate::types::{ use async_trait::async_trait; use fnv::FnvHashMap; use serde::de::DeserializeOwned; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; /// Http Client Builder. #[derive(Debug)] @@ -75,7 +75,7 @@ impl HttpClientBuilder { .map_err(|e| Error::Transport(e.into()))?; Ok(HttpClient { transport, - id_manager: RequestIdManager::new(self.max_concurrent_requests), + id_manager: Arc::new(RequestIdManager::new(self.max_concurrent_requests)), request_timeout: self.request_timeout, }) } @@ -93,14 +93,14 @@ impl Default for HttpClientBuilder { } /// JSON-RPC HTTP Client that provides functionality to perform method calls and notifications. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct HttpClient { /// HTTP transport client. transport: HttpTransportClient, /// Request timeout. Defaults to 60sec. request_timeout: Duration, /// Request ID manager. - id_manager: RequestIdManager, + id_manager: Arc, } #[async_trait] From 3c3f3ac9b6c12e81a39e845b898b085b9580b84e Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Tue, 30 Nov 2021 14:21:10 +0100 Subject: [PATCH 27/31] fix(types): use `Cow` for deserializing `str` (#584) * fix(types): use `Cow` for deserializing `str` * use ToString --- types/src/v2/params.rs | 8 ++--- types/src/v2/request.rs | 56 ++++++++++++++++++++++++++++------ utils/src/server/rpc_module.rs | 2 +- ws-client/src/helpers.rs | 6 ++-- 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/types/src/v2/params.rs b/types/src/v2/params.rs index b3a8fd799f..ea399be59b 100644 --- a/types/src/v2/params.rs +++ b/types/src/v2/params.rs @@ -383,17 +383,17 @@ mod test { let s = r#"[1337]"#; assert!(serde_json::from_str::(s).is_err()); - let s = r#"[null, 0, 2, "3"]"#; + let s = r#"[null, 0, 2, "\"3"]"#; let deserialized: Vec = serde_json::from_str(s).unwrap(); - assert_eq!(deserialized, vec![Id::Null, Id::Number(0), Id::Number(2), Id::Str("3".into())]); + assert_eq!(deserialized, vec![Id::Null, Id::Number(0), Id::Number(2), Id::Str("\"3".into())]); } #[test] fn id_serialization() { let d = - vec![Id::Null, Id::Number(0), Id::Number(2), Id::Number(3), Id::Str("3".into()), Id::Str("test".into())]; + vec![Id::Null, Id::Number(0), Id::Number(2), Id::Number(3), Id::Str("\"3".into()), Id::Str("test".into())]; let serialized = serde_json::to_string(&d).unwrap(); - assert_eq!(serialized, r#"[null,0,2,3,"3","test"]"#); + assert_eq!(serialized, r#"[null,0,2,3,"\"3","test"]"#); } #[test] diff --git a/types/src/v2/request.rs b/types/src/v2/request.rs index 19d382a562..e63ee3ef97 100644 --- a/types/src/v2/request.rs +++ b/types/src/v2/request.rs @@ -65,7 +65,8 @@ pub struct Notification<'a, T> { /// JSON-RPC version. pub jsonrpc: TwoPointZero, /// Name of the method to be invoked. - pub method: &'a str, + #[serde(borrow)] + pub method: Cow<'a, str>, /// Parameter values of the request. pub params: T, } @@ -78,6 +79,8 @@ pub struct RequestSer<'a> { /// Request ID pub id: Id<'a>, /// Name of the method to be invoked. + // NOTE: as this type only implements serialize + // `#[serde(borrow)]` and `Cow<'a, str>` is not needed. pub method: &'a str, /// Parameter values of the request. #[serde(skip_serializing_if = "Option::is_none")] @@ -97,6 +100,8 @@ pub struct NotificationSer<'a> { /// JSON-RPC version. pub jsonrpc: TwoPointZero, /// Name of the method to be invoked. + // NOTE: as this type only implements serialize + // `#[serde(borrow)]` and `Cow<'a, str>` is not needed. pub method: &'a str, /// Parameter values of the request. #[serde(skip_serializing_if = "Option::is_none")] @@ -124,23 +129,37 @@ mod test { /// Checks that we can deserialize the object with or without non-mandatory fields. #[test] - fn deserialize_request() { + fn deserialize_call() { let method = "subtract"; let params = "[42, 23]"; let test_vector = vec![ // With all fields set. - (r#"{"jsonrpc":"2.0", "method":"subtract", "params":[42, 23], "id":1}"#, Id::Number(1), Some(params)), + ( + r#"{"jsonrpc":"2.0", "method":"subtract", "params":[42, 23], "id":1}"#, + Id::Number(1), + Some(params), + method, + ), // Without params field - (r#"{"jsonrpc":"2.0", "method":"subtract", "id":null}"#, Id::Null, None), + (r#"{"jsonrpc":"2.0", "method":"subtract", "id":null}"#, Id::Null, None, method), + // Escaped method name. + (r#"{"jsonrpc":"2.0", "method":"\"m", "id":null}"#, Id::Null, None, "\"m"), ]; - for (ser, id, params) in test_vector.into_iter() { + for (ser, id, params, method) in test_vector.into_iter() { let request = serde_json::from_str(ser).unwrap(); assert_request(request, id, method, params); } } + #[test] + fn deserialize_call_escaped_method_name() { + let ser = r#"{"jsonrpc":"2.0","id":1,"method":"\"m\""}"#; + let req: Request = serde_json::from_str(ser).unwrap(); + assert_request(req, Id::Number(1), "\"m\"", None); + } + #[test] fn deserialize_valid_notif_works() { let ser = r#"{"jsonrpc":"2.0","method":"say_hello","params":[]}"#; @@ -149,6 +168,14 @@ mod test { assert_eq!(dsr.jsonrpc, TwoPointZero); } + #[test] + fn deserialize_valid_notif_escaped_method() { + let ser = r#"{"jsonrpc":"2.0","method":"\"m\"","params":[]}"#; + let dsr: Notification<&RawValue> = serde_json::from_str(ser).unwrap(); + assert_eq!(dsr.method, "\"m\""); + assert_eq!(dsr.jsonrpc, TwoPointZero); + } + #[test] fn deserialize_call_bad_id_should_fail() { let ser = r#"{"jsonrpc":"2.0","method":"say_hello","params":[],"id":{}}"#; @@ -174,16 +201,19 @@ mod test { r#"{"jsonrpc":"2.0","id":1,"method":"subtract","params":[42,23]}"#, Some(id.clone()), Some(params.clone()), + method, ), + // Escaped method name. + (r#"{"jsonrpc":"2.0","id":1,"method":"\"m"}"#, Some(id.clone()), None, "\"m"), // Without ID field. - (r#"{"jsonrpc":"2.0","id":null,"method":"subtract","params":[42,23]}"#, None, Some(params)), + (r#"{"jsonrpc":"2.0","id":null,"method":"subtract","params":[42,23]}"#, None, Some(params), method), // Without params field - (r#"{"jsonrpc":"2.0","id":1,"method":"subtract"}"#, Some(id), None), + (r#"{"jsonrpc":"2.0","id":1,"method":"subtract"}"#, Some(id), None, method), // Without params and ID. - (r#"{"jsonrpc":"2.0","id":null,"method":"subtract"}"#, None, None), + (r#"{"jsonrpc":"2.0","id":null,"method":"subtract"}"#, None, None, method), ]; - for (ser, id, params) in test_vector.iter().cloned() { + for (ser, id, params, method) in test_vector.iter().cloned() { let request = serde_json::to_string(&RequestSer { jsonrpc: TwoPointZero, method, @@ -203,4 +233,12 @@ mod test { let ser = serde_json::to_string(&req).unwrap(); assert_eq!(exp, ser); } + + #[test] + fn serialize_notif_escaped_method_name() { + let exp = r#"{"jsonrpc":"2.0","method":"\"method\""}"#; + let req = NotificationSer::new("\"method\"", None); + let ser = serde_json::to_string(&req).unwrap(); + assert_eq!(exp, ser); + } } diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index eca57ef0b9..3d0ee2849e 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -682,7 +682,7 @@ impl SubscriptionSink { fn build_message(&self, result: &T) -> Result { serde_json::to_string(&SubscriptionResponse { jsonrpc: TwoPointZero, - method: self.method, + method: self.method.into(), params: SubscriptionPayload { subscription: RpcSubscriptionId::Num(self.uniq_sub.sub_id), result }, }) .map_err(Into::into) diff --git a/ws-client/src/helpers.rs b/ws-client/src/helpers.rs index 9a7c711beb..7be177ff7c 100644 --- a/ws-client/src/helpers.rs +++ b/ws-client/src/helpers.rs @@ -103,18 +103,18 @@ pub fn process_subscription_response( /// Returns Ok() if the response was successfully handled /// Returns Err() if there was no handler for the method pub fn process_notification(manager: &mut RequestManager, notif: Notification) -> Result<(), Error> { - match manager.as_notification_handler_mut(notif.method.to_owned()) { + match manager.as_notification_handler_mut(notif.method.to_string()) { Some(send_back_sink) => match send_back_sink.try_send(notif.params) { Ok(()) => Ok(()), Err(err) => { tracing::error!("Error sending notification, dropping handler for {:?} error: {:?}", notif.method, err); - let _ = manager.remove_notification_handler(notif.method.to_owned()); + let _ = manager.remove_notification_handler(notif.method.into_owned()); Err(Error::Internal(err.into_send_error())) } }, None => { tracing::error!("Notification: {:?} not a registered method", notif.method); - Err(Error::UnregisteredNotification(notif.method.to_owned())) + Err(Error::UnregisteredNotification(notif.method.into_owned())) } } } From 1657e26b7461d5fe52d98615ce5064f18c829859 Mon Sep 17 00:00:00 2001 From: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Date: Wed, 1 Dec 2021 10:17:58 +0100 Subject: [PATCH 28/31] Middleware for metrics (#576) * Squashed MethodSink * Middleware WIP * Passing all the information through * Unnecessary `false` * Apply suggestions from code review Co-authored-by: David * Add a setter for middleware (#577) * Fix try-build tests * Add a middleware setter and an example * Actually add the example * Grumbles * Use an atomic * Set middleware with a constructor instead * Resolve a todo * Update ws-server/src/server.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * Update ws-server/src/server.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * Update ws-server/src/server.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * Middleware::on_response for batches * Middleware in HTTP * fmt * Server builder for HTTP * Use actual time in the example * HTTP example * Middleware to capture method not found calls * An example of adding multiple middlewares. (#581) * Add an example of adding multiple middlewares. * Update examples/multi-middleware.rs Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * Update examples/Cargo.toml Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> Co-authored-by: Maciej Hirsz <1096222+maciejhirsz@users.noreply.github.com> * Move `Middleware` to jsonrpsee-types (#582) * Move `Middleware` to jsonrpsee-types * Move Middleware trait to jsonrpsee-types * Add some docs. * Link middleware to `with_middleware` methods in docs * Doctests * Doc comment fixed * Clean up a TODO * Switch back to `set_middleware` * fmt * Tests * Add `on_connect` and `on_disconnect` * Add note to future selves Co-authored-by: David --- examples/Cargo.toml | 13 ++ examples/middleware_http.rs | 84 +++++++++++ examples/middleware_ws.rs | 84 +++++++++++ examples/multi_middleware.rs | 115 +++++++++++++++ http-server/src/server.rs | 133 +++++++++++++---- jsonrpsee/src/lib.rs | 3 + tests/tests/middleware.rs | 197 ++++++++++++++++++++++++++ types/src/lib.rs | 3 + types/src/middleware.rs | 92 ++++++++++++ utils/src/server/helpers.rs | 140 +++++++++++------- utils/src/server/resource_limiting.rs | 2 +- utils/src/server/rpc_module.rs | 163 ++++++++++++--------- ws-server/src/server.rs | 196 ++++++++++++++++++------- 13 files changed, 1026 insertions(+), 199 deletions(-) create mode 100644 examples/middleware_http.rs create mode 100644 examples/middleware_ws.rs create mode 100644 examples/multi_middleware.rs create mode 100644 tests/tests/middleware.rs create mode 100644 types/src/middleware.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 6428b298ea..e8704bb3a9 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -13,11 +13,24 @@ jsonrpsee = { path = "../jsonrpsee", features = ["full"] } tracing = "0.1" tracing-subscriber = "0.2" tokio = { version = "1", features = ["full"] } +palaver = "0.2" [[example]] name = "http" path = "http.rs" +[[example]] +name = "middleware_ws" +path = "middleware_ws.rs" + +[[example]] +name = "middleware_http" +path = "middleware_http.rs" + +[[example]] +name = "multi_middleware" +path = "multi_middleware.rs" + [[example]] name = "ws" path = "ws.rs" diff --git a/examples/middleware_http.rs b/examples/middleware_http.rs new file mode 100644 index 0000000000..c734ca5d47 --- /dev/null +++ b/examples/middleware_http.rs @@ -0,0 +1,84 @@ +// Copyright 2019-2021 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use jsonrpsee::{ + http_client::HttpClientBuilder, + http_server::{HttpServerBuilder, HttpServerHandle, RpcModule}, + types::{middleware, traits::Client}, +}; +use std::net::SocketAddr; +use std::time::Instant; + +#[derive(Clone)] +struct Timings; + +impl middleware::Middleware for Timings { + type Instant = Instant; + + fn on_request(&self) -> Self::Instant { + Instant::now() + } + + fn on_call(&self, name: &str) { + println!("[Middleware::on_call] '{}'", name); + } + + fn on_result(&self, name: &str, succeess: bool, started_at: Self::Instant) { + println!("[Middleware::on_result] '{}', worked? {}, time elapsed {:?}", name, succeess, started_at.elapsed()); + } + + fn on_response(&self, started_at: Self::Instant) { + println!("[Middleware::on_response] time elapsed {:?}", started_at.elapsed()); + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); + + let (addr, _handle) = run_server().await?; + let url = format!("http://{}", addr); + + let client = HttpClientBuilder::default().build(&url)?; + let response: String = client.request("say_hello", None).await?; + println!("response: {:?}", response); + let _response: Result = client.request("unknown_method", None).await; + let _ = client.request::("say_hello", None).await?; + + Ok(()) +} + +async fn run_server() -> anyhow::Result<(SocketAddr, HttpServerHandle)> { + let server = HttpServerBuilder::new().set_middleware(Timings).build("127.0.0.1:0")?; + let mut module = RpcModule::new(()); + module.register_method("say_hello", |_, _| Ok("lo"))?; + let addr = server.local_addr()?; + let server_handle = server.start(module)?; + Ok((addr, server_handle)) +} diff --git a/examples/middleware_ws.rs b/examples/middleware_ws.rs new file mode 100644 index 0000000000..19d262a2bd --- /dev/null +++ b/examples/middleware_ws.rs @@ -0,0 +1,84 @@ +// Copyright 2019-2021 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use jsonrpsee::{ + types::{middleware, traits::Client}, + ws_client::WsClientBuilder, + ws_server::{RpcModule, WsServerBuilder}, +}; +use std::net::SocketAddr; +use std::time::Instant; + +#[derive(Clone)] +struct Timings; + +impl middleware::Middleware for Timings { + type Instant = Instant; + + fn on_request(&self) -> Self::Instant { + Instant::now() + } + + fn on_call(&self, name: &str) { + println!("[Middleware::on_call] '{}'", name); + } + + fn on_result(&self, name: &str, succeess: bool, started_at: Self::Instant) { + println!("[Middleware::on_result] '{}', worked? {}, time elapsed {:?}", name, succeess, started_at.elapsed()); + } + + fn on_response(&self, started_at: Self::Instant) { + println!("[Middleware::on_response] time elapsed {:?}", started_at.elapsed()); + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); + + let addr = run_server().await?; + let url = format!("ws://{}", addr); + + let client = WsClientBuilder::default().build(&url).await?; + let response: String = client.request("say_hello", None).await?; + println!("response: {:?}", response); + let _response: Result = client.request("unknown_method", None).await; + let _ = client.request::("say_hello", None).await?; + + Ok(()) +} + +async fn run_server() -> anyhow::Result { + let server = WsServerBuilder::new().set_middleware(Timings).build("127.0.0.1:0").await?; + let mut module = RpcModule::new(()); + module.register_method("say_hello", |_, _| Ok("lo"))?; + let addr = server.local_addr()?; + server.start(module)?; + Ok(addr) +} diff --git a/examples/multi_middleware.rs b/examples/multi_middleware.rs new file mode 100644 index 0000000000..b240fffcff --- /dev/null +++ b/examples/multi_middleware.rs @@ -0,0 +1,115 @@ +// Copyright 2019-2021 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Example showing how to add multiple middlewares to the same server. + +use jsonrpsee::{ + rpc_params, + types::{middleware, traits::Client}, + ws_client::WsClientBuilder, + ws_server::{RpcModule, WsServerBuilder}, +}; +use std::net::SocketAddr; +use std::time::Instant; + +/// Example middleware to measure call execution time. +#[derive(Clone)] +struct Timings; + +impl middleware::Middleware for Timings { + type Instant = Instant; + + fn on_request(&self) -> Self::Instant { + Instant::now() + } + + fn on_call(&self, name: &str) { + println!("[Timings] They called '{}'", name); + } + + fn on_result(&self, name: &str, succeess: bool, started_at: Self::Instant) { + println!("[Timings] call={}, worked? {}, duration {:?}", name, succeess, started_at.elapsed()); + } + + fn on_response(&self, started_at: Self::Instant) { + println!("[Timings] Response duration {:?}", started_at.elapsed()); + } +} + +/// Example middleware to keep a watch on the number of total threads started in the system. +#[derive(Clone)] +struct ThreadWatcher; + +impl middleware::Middleware for ThreadWatcher { + type Instant = isize; + + fn on_request(&self) -> Self::Instant { + let threads = palaver::process::count_threads(); + println!("[ThreadWatcher] Threads running on the machine at the start of a call: {}", threads); + threads as isize + } + + fn on_response(&self, started_at: Self::Instant) { + let current_nr_threads = palaver::process::count_threads() as isize; + println!("[ThreadWatcher] Request started {} threads", current_nr_threads - started_at); + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .expect("setting default subscriber failed"); + + let addr = run_server().await?; + let url = format!("ws://{}", addr); + + let client = WsClientBuilder::default().build(&url).await?; + let response: String = client.request("say_hello", None).await?; + println!("response: {:?}", response); + let _response: Result = client.request("unknown_method", None).await; + let _ = client.request::("say_hello", None).await?; + let _ = client.request::<()>("thready", rpc_params![4]).await?; + + Ok(()) +} + +async fn run_server() -> anyhow::Result { + let server = WsServerBuilder::new().set_middleware((Timings, ThreadWatcher)).build("127.0.0.1:0").await?; + let mut module = RpcModule::new(()); + module.register_method("say_hello", |_, _| Ok("lo"))?; + module.register_method("thready", |params, _| { + let thread_count: usize = params.one().unwrap(); + for _ in 0..thread_count { + std::thread::spawn(|| std::thread::sleep(std::time::Duration::from_secs(1))); + } + Ok(()) + })?; + let addr = server.local_addr()?; + server.start(module)?; + Ok(addr) +} diff --git a/http-server/src/server.rs b/http-server/src/server.rs index 43269e9926..7ecb9f7025 100644 --- a/http-server/src/server.rs +++ b/http-server/src/server.rs @@ -34,14 +34,15 @@ use hyper::{ }; use jsonrpsee_types::{ error::{Error, GenericTransportError}, + middleware::Middleware, v2::{ErrorCode, Id, Notification, Request}, TEN_MB_SIZE_BYTES, }; use jsonrpsee_utils::http_helpers::read_body; use jsonrpsee_utils::server::{ - helpers::{collect_batch_response, prepare_error, send_error}, + helpers::{collect_batch_response, prepare_error, MethodSink}, resource_limiting::Resources, - rpc_module::Methods, + rpc_module::{MethodResult, Methods}, }; use serde_json::value::RawValue; @@ -56,16 +57,72 @@ use std::{ /// Builder to create JSON-RPC HTTP server. #[derive(Debug)] -pub struct Builder { +pub struct Builder { access_control: AccessControl, resources: Resources, max_request_body_size: u32, keep_alive: bool, /// Custom tokio runtime to run the server on. tokio_runtime: Option, + middleware: M, +} + +impl Default for Builder { + fn default() -> Self { + Self { + max_request_body_size: TEN_MB_SIZE_BYTES, + resources: Resources::default(), + access_control: AccessControl::default(), + keep_alive: true, + tokio_runtime: None, + middleware: (), + } + } } impl Builder { + /// Create a default server builder. + pub fn new() -> Self { + Self::default() + } +} + +impl Builder { + /// Add a middleware to the builder [`Middleware`](../jsonrpsee_types/middleware/trait.Middleware.html). + /// + /// ``` + /// use jsonrpsee_types::middleware::Middleware; + /// use jsonrpsee_http_server::HttpServerBuilder; + /// use std::time::Instant; + /// + /// #[derive(Clone)] + /// struct MyMiddleware; + /// + /// impl Middleware for MyMiddleware { + /// type Instant = Instant; + /// + /// fn on_request(&self) -> Instant { + /// Instant::now() + /// } + /// + /// fn on_result(&self, name: &str, success: bool, started_at: Instant) { + /// println!("Call to '{}' took {:?}", name, started_at.elapsed()); + /// } + /// } + /// + /// let builder = HttpServerBuilder::new().set_middleware(MyMiddleware); + /// ``` + pub fn set_middleware(self, middleware: T) -> Builder { + Builder { + max_request_body_size: self.max_request_body_size, + resources: self.resources, + access_control: self.access_control, + keep_alive: self.keep_alive, + tokio_runtime: self.tokio_runtime, + middleware, + } + } + /// Sets the maximum size of a request body in bytes (default is 10 MiB). pub fn max_request_body_size(mut self, size: u32) -> Self { self.max_request_body_size = size; @@ -120,7 +177,7 @@ impl Builder { /// assert!(jsonrpsee_http_server::HttpServerBuilder::default().build(addrs).is_ok()); /// } /// ``` - pub fn build(self, addrs: impl ToSocketAddrs) -> Result { + pub fn build(self, addrs: impl ToSocketAddrs) -> Result, Error> { let mut err: Option = None; for addr in addrs.to_socket_addrs()? { @@ -139,6 +196,7 @@ impl Builder { max_request_body_size: self.max_request_body_size, resources: self.resources, tokio_runtime: self.tokio_runtime, + middleware: self.middleware, }); } @@ -167,18 +225,6 @@ impl Builder { } } -impl Default for Builder { - fn default() -> Self { - Self { - max_request_body_size: TEN_MB_SIZE_BYTES, - resources: Resources::default(), - access_control: AccessControl::default(), - keep_alive: true, - tokio_runtime: None, - } - } -} - /// Handle used to run or stop the server. #[derive(Debug)] pub struct ServerHandle { @@ -212,7 +258,7 @@ impl Future for ServerHandle { /// An HTTP JSON RPC server. #[derive(Debug)] -pub struct Server { +pub struct Server { /// Hyper server. listener: HyperBuilder, /// Local address @@ -225,9 +271,10 @@ pub struct Server { resources: Resources, /// Custom tokio runtime to run the server on. tokio_runtime: Option, + middleware: M, } -impl Server { +impl Server { /// Returns socket address to which the server is bound. pub fn local_addr(&self) -> Result { self.local_addr.ok_or_else(|| Error::Custom("Local address not found".into())) @@ -240,18 +287,21 @@ impl Server { let (tx, mut rx) = mpsc::channel(1); let listener = self.listener; let resources = self.resources; + let middleware = self.middleware; let methods = methods.into().initialize_resources(&resources)?; let make_service = make_service_fn(move |_| { let methods = methods.clone(); let access_control = access_control.clone(); let resources = resources.clone(); + let middleware = middleware.clone(); async move { Ok::<_, HyperError>(service_fn(move |request| { let methods = methods.clone(); let access_control = access_control.clone(); let resources = resources.clone(); + let middleware = middleware.clone(); // Run some validation on the http request, then read the body and try to deserialize it into one of // two cases: a single RPC request or a batch of RPC requests. @@ -276,32 +326,60 @@ impl Server { } }; + let request_start = middleware.on_request(); + // NOTE(niklasad1): it's a channel because it's needed for batch requests. let (tx, mut rx) = mpsc::unbounded::(); + let sink = MethodSink::new_with_limit(tx, max_request_body_size); type Notif<'a> = Notification<'a, Option<&'a RawValue>>; // Single request or notification if is_single { if let Ok(req) = serde_json::from_slice::(&body) { + middleware.on_call(req.method.as_ref()); + // NOTE: we don't need to track connection id on HTTP, so using hardcoded 0 here. - if let Some(fut) = - methods.execute_with_resources(&tx, req, 0, &resources, max_request_body_size) - { - fut.await; + match methods.execute_with_resources(&sink, req, 0, &resources) { + Ok((name, MethodResult::Sync(success))) => { + middleware.on_result(name, success, request_start); + } + Ok((name, MethodResult::Async(fut))) => { + let success = fut.await; + + middleware.on_result(name, success, request_start); + } + Err(name) => { + middleware.on_result(name.as_ref(), false, request_start); + } } } else if let Ok(_req) = serde_json::from_slice::(&body) { return Ok::<_, HyperError>(response::ok_response("".into())); } else { let (id, code) = prepare_error(&body); - send_error(id, &tx, code.into()); + sink.send_error(id, code.into()); } // Batch of requests or notifications } else if let Ok(batch) = serde_json::from_slice::>(&body) { if !batch.is_empty() { - join_all(batch.into_iter().filter_map(|req| { - methods.execute_with_resources(&tx, req, 0, &resources, max_request_body_size) + let middleware = &middleware; + + join_all(batch.into_iter().filter_map(move |req| { + match methods.execute_with_resources(&sink, req, 0, &resources) { + Ok((name, MethodResult::Sync(success))) => { + middleware.on_result(name, success, request_start); + None + } + Ok((name, MethodResult::Async(fut))) => Some(async move { + let success = fut.await; + middleware.on_result(name, success, request_start); + }), + Err(name) => { + middleware.on_result(name.as_ref(), false, request_start); + None + } + } })) .await; } else { @@ -309,7 +387,7 @@ impl Server { // Array with at least one value, the response from the Server MUST be a single // Response object." – The Spec. is_single = true; - send_error(Id::Null, &tx, ErrorCode::InvalidRequest.into()); + sink.send_error(Id::Null, ErrorCode::InvalidRequest.into()); } } else if let Ok(_batch) = serde_json::from_slice::>(&body) { return Ok::<_, HyperError>(response::ok_response("".into())); @@ -319,7 +397,7 @@ impl Server { // Response object." – The Spec. is_single = true; let (id, code) = prepare_error(&body); - send_error(id, &tx, code.into()); + sink.send_error(id, code.into()); } // Closes the receiving half of a channel without dropping it. This prevents any further @@ -331,6 +409,7 @@ impl Server { collect_batch_response(rx).await }; tracing::debug!("[service_fn] sending back: {:?}", &response[..cmp::min(response.len(), 1024)]); + middleware.on_response(request_start); Ok::<_, HyperError>(response::ok_response(response)) } })) diff --git a/jsonrpsee/src/lib.rs b/jsonrpsee/src/lib.rs index 4fc8cbddc5..5d2e97ca5f 100644 --- a/jsonrpsee/src/lib.rs +++ b/jsonrpsee/src/lib.rs @@ -76,6 +76,9 @@ pub use jsonrpsee_types as types; #[cfg(any(feature = "http-server", feature = "ws-server"))] pub use jsonrpsee_utils::server::rpc_module::{RpcModule, SubscriptionSink}; +#[cfg(any(feature = "http-server", feature = "ws-server"))] +pub use jsonrpsee_utils as utils; + #[cfg(feature = "http-server")] pub use http_server::tracing; diff --git a/tests/tests/middleware.rs b/tests/tests/middleware.rs new file mode 100644 index 0000000000..6767b3968f --- /dev/null +++ b/tests/tests/middleware.rs @@ -0,0 +1,197 @@ +// Copyright 2019-2021 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use jsonrpsee::{ + http_client::HttpClientBuilder, + http_server::{HttpServerBuilder, HttpServerHandle}, + proc_macros::rpc, + types::{middleware::Middleware, traits::Client, Error}, + ws_client::WsClientBuilder, + ws_server::{WsServerBuilder, WsServerHandle}, + RpcModule, +}; +use tokio::time::sleep; + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +#[derive(Clone, Default)] +struct Counter { + inner: Arc>, +} + +#[derive(Default)] +struct CounterInner { + /// (Number of started connections, number of finished connections) + connections: (u32, u32), + /// (Number of started requests, number of finished requests) + requests: (u32, u32), + /// Mapping method names to (number of calls, ids of successfully completed calls) + calls: HashMap)>, +} + +impl Middleware for Counter { + /// Auto-incremented id of the call + type Instant = u32; + + fn on_connect(&self) { + self.inner.lock().unwrap().connections.0 += 1; + } + + fn on_request(&self) -> u32 { + let mut inner = self.inner.lock().unwrap(); + let n = inner.requests.0; + + inner.requests.0 += 1; + + n + } + + fn on_call(&self, name: &str) { + let mut inner = self.inner.lock().unwrap(); + let entry = inner.calls.entry(name.into()).or_insert((0, Vec::new())); + + entry.0 += 1; + } + + fn on_result(&self, name: &str, success: bool, n: u32) { + if success { + self.inner.lock().unwrap().calls.get_mut(name).unwrap().1.push(n); + } + } + + fn on_response(&self, _: u32) { + self.inner.lock().unwrap().requests.1 += 1; + } + + fn on_disconnect(&self) { + self.inner.lock().unwrap().connections.1 += 1; + } +} + +fn test_module() -> RpcModule<()> { + #[rpc(server)] + pub trait Rpc { + #[method(name = "say_hello")] + async fn hello(&self) -> Result<&'static str, Error> { + sleep(Duration::from_millis(50)).await; + Ok("hello") + } + } + + impl RpcServer for () {} + + ().into_rpc() +} + +async fn websocket_server(module: RpcModule<()>, counter: Counter) -> Result<(SocketAddr, WsServerHandle), Error> { + let server = WsServerBuilder::default() + .register_resource("CPU", 6, 2)? + .register_resource("MEM", 10, 1)? + .set_middleware(counter) + .build("127.0.0.1:0") + .await?; + + let addr = server.local_addr()?; + let handle = server.start(module)?; + + Ok((addr, handle)) +} + +async fn http_server(module: RpcModule<()>, counter: Counter) -> Result<(SocketAddr, HttpServerHandle), Error> { + let server = HttpServerBuilder::default() + .register_resource("CPU", 6, 2)? + .register_resource("MEM", 10, 1)? + .set_middleware(counter) + .build("127.0.0.1:0")?; + + let addr = server.local_addr()?; + let handle = server.start(module)?; + + Ok((addr, handle)) +} + +#[tokio::test] +async fn ws_server_middleware() { + let counter = Counter::default(); + let (server_addr, server_handle) = websocket_server(test_module(), counter.clone()).await.unwrap(); + + let server_url = format!("ws://{}", server_addr); + let client = WsClientBuilder::default().build(&server_url).await.unwrap(); + + assert_eq!(client.request::("say_hello", None).await.unwrap(), "hello"); + + assert!(client.request::("unknown_method", None).await.is_err()); + + assert_eq!(client.request::("say_hello", None).await.unwrap(), "hello"); + assert_eq!(client.request::("say_hello", None).await.unwrap(), "hello"); + + assert!(client.request::("unknown_method", None).await.is_err()); + + { + let inner = counter.inner.lock().unwrap(); + + assert_eq!(inner.connections, (1, 0)); + assert_eq!(inner.requests, (5, 5)); + assert_eq!(inner.calls["say_hello"], (3, vec![0, 2, 3])); + assert_eq!(inner.calls["unknown_method"], (2, vec![])); + } + + server_handle.stop().unwrap().await; + + assert_eq!(counter.inner.lock().unwrap().connections, (1, 1)); +} + +#[tokio::test] +async fn http_server_middleware() { + let counter = Counter::default(); + let (server_addr, server_handle) = http_server(test_module(), counter.clone()).await.unwrap(); + + let server_url = format!("http://{}", server_addr); + let client = HttpClientBuilder::default().build(&server_url).unwrap(); + + assert_eq!(client.request::("say_hello", None).await.unwrap(), "hello"); + + assert!(client.request::("unknown_method", None).await.is_err()); + + assert_eq!(client.request::("say_hello", None).await.unwrap(), "hello"); + assert_eq!(client.request::("say_hello", None).await.unwrap(), "hello"); + + assert!(client.request::("unknown_method", None).await.is_err()); + + let inner = counter.inner.lock().unwrap(); + + assert_eq!(inner.requests, (5, 5)); + assert_eq!(inner.calls["say_hello"], (3, vec![0, 2, 3])); + assert_eq!(inner.calls["unknown_method"], (2, vec![])); + + server_handle.stop().unwrap().await.unwrap(); + + // HTTP server doesn't track connections + assert_eq!(inner.connections, (0, 0)); +} diff --git a/types/src/lib.rs b/types/src/lib.rs index 649096da57..a932e94d5c 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -46,6 +46,9 @@ mod client; /// Traits pub mod traits; +/// Middleware trait and implementation. +pub mod middleware; + pub use async_trait::async_trait; pub use beef::Cow; pub use client::*; diff --git a/types/src/middleware.rs b/types/src/middleware.rs new file mode 100644 index 0000000000..2ca8f3db5f --- /dev/null +++ b/types/src/middleware.rs @@ -0,0 +1,92 @@ +// Copyright 2019-2021 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Middleware for `jsonrpsee` servers. + +/// Defines a middleware with callbacks during the RPC request life-cycle. The primary use case for +/// this is to collect timings for a larger metrics collection solution but the only constraints on +/// the associated type is that it be [`Send`] and [`Copy`], giving users some freedom to do what +/// they need to do. +/// +/// See the [`WsServerBuilder::set_middleware`](../../jsonrpsee_ws_server/struct.WsServerBuilder.html#method.set_middleware) +/// or the [`HttpServerBuilder::set_middleware`](../../jsonrpsee_http_server/struct.HttpServerBuilder.html#method.set_middleware) method +/// for examples. +pub trait Middleware: Send + Sync + Clone + 'static { + /// Intended to carry timestamp of a request, for example `std::time::Instant`. How the middleware + /// measures time, if at all, is entirely up to the implementation. + type Instant: Send + Copy; + + /// Called when a new client connects (WebSocket only) + fn on_connect(&self) {} + + /// Called when a new JSON-RPC comes to the server. + fn on_request(&self) -> Self::Instant; + + /// Called on each JSON-RPC method call, batch requests will trigger `on_call` multiple times. + fn on_call(&self, _name: &str) {} + + /// Called on each JSON-RPC method completion, batch requests will trigger `on_result` multiple times. + fn on_result(&self, _name: &str, _success: bool, _started_at: Self::Instant) {} + + /// Called once the JSON-RPC request is finished and response is sent to the output buffer. + fn on_response(&self, _started_at: Self::Instant) {} + + /// Called when a client disconnects (WebSocket only) + fn on_disconnect(&self) {} +} + +impl Middleware for () { + type Instant = (); + + fn on_request(&self) -> Self::Instant {} +} + +impl Middleware for (A, B) +where + A: Middleware, + B: Middleware, +{ + type Instant = (A::Instant, B::Instant); + + fn on_request(&self) -> Self::Instant { + (self.0.on_request(), self.1.on_request()) + } + + fn on_call(&self, name: &str) { + self.0.on_call(name); + self.1.on_call(name); + } + + fn on_result(&self, name: &str, success: bool, started_at: Self::Instant) { + self.0.on_result(name, success, started_at.0); + self.1.on_result(name, success, started_at.1); + } + + fn on_response(&self, started_at: Self::Instant) { + self.0.on_response(started_at.0); + self.1.on_response(started_at.1); + } +} diff --git a/utils/src/server/helpers.rs b/utils/src/server/helpers.rs index c6cfd1d212..25d863f07e 100644 --- a/utils/src/server/helpers.rs +++ b/utils/src/server/helpers.rs @@ -24,7 +24,6 @@ // IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -use crate::server::rpc_module::MethodSink; use futures_channel::mpsc; use futures_util::stream::StreamExt; use jsonrpsee_types::error::{CallError, Error}; @@ -82,67 +81,108 @@ impl<'a> io::Write for &'a mut BoundedWriter { } } -/// Helper for sending JSON-RPC responses to the client -pub fn send_response(id: Id, tx: &MethodSink, result: impl Serialize, max_response_size: u32) { - let mut writer = BoundedWriter::new(max_response_size as usize); +/// Sink that is used to send back the result to the server for a specific method. +#[derive(Clone, Debug)] +pub struct MethodSink { + /// Channel sender + tx: mpsc::UnboundedSender, + /// Max response size in bytes for a executed call. + max_response_size: u32, +} - let json = match serde_json::to_writer(&mut writer, &Response { jsonrpc: TwoPointZero, id: id.clone(), result }) { - Ok(_) => { - // Safety - serde_json does not emit invalid UTF-8. - unsafe { String::from_utf8_unchecked(writer.into_bytes()) } - } - Err(err) => { - tracing::error!("Error serializing response: {:?}", err); - - if err.is_io() { - let data = to_json_raw_value(&format!("Exceeded max limit {}", max_response_size)).ok(); - let err = ErrorObject { - code: ErrorCode::ServerError(OVERSIZED_RESPONSE_CODE), - message: OVERSIZED_RESPONSE_MSG.into(), - data: data.as_deref(), - }; - return send_error(id, tx, err); - } else { - return send_error(id, tx, ErrorCode::InternalError.into()); +impl MethodSink { + /// Create a new `MethodSink` with unlimited response size + pub fn new(tx: mpsc::UnboundedSender) -> Self { + MethodSink { tx, max_response_size: u32::MAX } + } + + /// Create a new `MethodSink` with a limited response size + pub fn new_with_limit(tx: mpsc::UnboundedSender, max_response_size: u32) -> Self { + MethodSink { tx, max_response_size } + } + + /// Send a JSON-RPC response to the client. If the serialization of `result` exceeds `max_response_size`, + /// an error will be sent instead. + pub fn send_response(&self, id: Id, result: impl Serialize) -> bool { + let mut writer = BoundedWriter::new(self.max_response_size as usize); + + let json = match serde_json::to_writer(&mut writer, &Response { jsonrpc: TwoPointZero, id: id.clone(), result }) + { + Ok(_) => { + // Safety - serde_json does not emit invalid UTF-8. + unsafe { String::from_utf8_unchecked(writer.into_bytes()) } } - } - }; + Err(err) => { + tracing::error!("Error serializing response: {:?}", err); + + if err.is_io() { + let data = to_json_raw_value(&format!("Exceeded max limit {}", self.max_response_size)).ok(); + let err = ErrorObject { + code: ErrorCode::ServerError(OVERSIZED_RESPONSE_CODE), + message: OVERSIZED_RESPONSE_MSG.into(), + data: data.as_deref(), + }; + return self.send_error(id, err); + } else { + return self.send_error(id, ErrorCode::InternalError.into()); + } + } + }; - if let Err(err) = tx.unbounded_send(json) { - tracing::error!("Error sending response to the client: {:?}", err) + if let Err(err) = self.tx.unbounded_send(json) { + tracing::error!("Error sending response to the client: {:?}", err); + false + } else { + true + } } -} -/// Helper for sending JSON-RPC errors to the client -pub fn send_error(id: Id, tx: &MethodSink, error: ErrorObject) { - let json = match serde_json::to_string(&RpcError { jsonrpc: TwoPointZero, error, id }) { - Ok(json) => json, - Err(err) => { - tracing::error!("Error serializing error message: {:?}", err); + /// Send a JSON-RPC error to the client + pub fn send_error(&self, id: Id, error: ErrorObject) -> bool { + let json = match serde_json::to_string(&RpcError { jsonrpc: TwoPointZero, error, id }) { + Ok(json) => json, + Err(err) => { + tracing::error!("Error serializing error message: {:?}", err); + + return false; + } + }; - return; + if let Err(err) = self.tx.unbounded_send(json) { + tracing::error!("Could not send error response to the client: {:?}", err) } - }; - if let Err(err) = tx.unbounded_send(json) { - tracing::error!("Could not send error response to the client: {:?}", err) + false } -} -/// Helper for sending the general purpose `Error` as a JSON-RPC errors to the client -pub fn send_call_error(id: Id, tx: &MethodSink, err: Error) { - let (code, message, data) = match err { - Error::Call(CallError::InvalidParams(e)) => (ErrorCode::InvalidParams, e.to_string(), None), - Error::Call(CallError::Failed(e)) => (ErrorCode::ServerError(CALL_EXECUTION_FAILED_CODE), e.to_string(), None), - Error::Call(CallError::Custom { code, message, data }) => (code.into(), message, data), - // This should normally not happen because the most common use case is to - // return `Error::Call` in `register_async_method`. - e => (ErrorCode::ServerError(UNKNOWN_ERROR_CODE), e.to_string(), None), - }; + /// Helper for sending the general purpose `Error` as a JSON-RPC errors to the client + pub fn send_call_error(&self, id: Id, err: Error) -> bool { + let (code, message, data) = match err { + Error::Call(CallError::InvalidParams(e)) => (ErrorCode::InvalidParams, e.to_string(), None), + Error::Call(CallError::Failed(e)) => { + (ErrorCode::ServerError(CALL_EXECUTION_FAILED_CODE), e.to_string(), None) + } + Error::Call(CallError::Custom { code, message, data }) => (code.into(), message, data), + // This should normally not happen because the most common use case is to + // return `Error::Call` in `register_async_method`. + e => (ErrorCode::ServerError(UNKNOWN_ERROR_CODE), e.to_string(), None), + }; + + let err = ErrorObject { code, message: message.into(), data: data.as_deref() }; - let err = ErrorObject { code, message: message.into(), data: data.as_deref() }; + self.send_error(id, err) + } - send_error(id, tx, err) + /// Send a raw JSON-RPC message to the client, `MethodSink` does not check verify the validity + /// of the JSON being sent. + pub fn send_raw(&self, raw_json: String) -> Result<(), mpsc::TrySendError> { + self.tx.unbounded_send(raw_json) + } + + /// Close the channel for any further messages. + pub fn close(&self) { + self.tx.close_channel(); + } } /// Figure out if this is a sufficiently complete request that we can extract an [`Id`] out of, or just plain diff --git a/utils/src/server/resource_limiting.rs b/utils/src/server/resource_limiting.rs index a7fe3dd71c..3127176e5e 100644 --- a/utils/src/server/resource_limiting.rs +++ b/utils/src/server/resource_limiting.rs @@ -38,7 +38,7 @@ //! and then defining your units such that the limits (`capacity`) can be adjusted for different hardware configurations. //! //! Up to 8 resources can be defined using the [`WsServerBuilder::register_resource`](../../../jsonrpsee_ws_server/struct.WsServerBuilder.html#method.register_resource) -//! or [`HttpServerBuilder::register_resource`](../../../jsonrpsee_ws_server/struct.WsServerBuilder.html#method.register_resource) method +//! or [`HttpServerBuilder::register_resource`](../../../jsonrpsee_http_server/struct.HttpServerBuilder.html#method.register_resource) method //! for the WebSocket and HTTP server respectively. //! //! Each method will claim the specified number of units (or the default) for the duration of its execution. diff --git a/utils/src/server/rpc_module.rs b/utils/src/server/rpc_module.rs index 3d0ee2849e..9ba5aaf7ed 100644 --- a/utils/src/server/rpc_module.rs +++ b/utils/src/server/rpc_module.rs @@ -24,7 +24,7 @@ // IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -use crate::server::helpers::{send_call_error, send_error, send_response}; +use crate::server::helpers::MethodSink; use crate::server::resource_limiting::{ResourceGuard, ResourceTable, ResourceVec, Resources}; use beef::Cow; use futures_channel::{mpsc, oneshot}; @@ -46,7 +46,7 @@ use rustc_hash::FxHashMap; use serde::Serialize; use serde_json::value::RawValue; use std::collections::hash_map::Entry; -use std::fmt::Debug; +use std::fmt::{self, Debug}; use std::future::Future; use std::ops::{Deref, DerefMut}; use std::sync::Arc; @@ -55,20 +55,15 @@ use std::sync::Arc; /// implemented as a function pointer to a `Fn` function taking four arguments: /// the `id`, `params`, a channel the function uses to communicate the result (or error) /// back to `jsonrpsee`, and the connection ID (useful for the websocket transport). -pub type SyncMethod = Arc; +pub type SyncMethod = Arc bool>; /// Similar to [`SyncMethod`], but represents an asynchronous handler and takes an additional argument containing a [`ResourceGuard`] if configured. -pub type AsyncMethod<'a> = Arc< - dyn Send + Sync + Fn(Id<'a>, Params<'a>, MethodSink, Option, MaxResponseSize) -> BoxFuture<'a, ()>, ->; +pub type AsyncMethod<'a> = + Arc, Params<'a>, MethodSink, Option) -> BoxFuture<'a, bool>>; /// Connection ID, used for stateful protocol such as WebSockets. /// For stateless protocols such as http it's unused, so feel free to set it some hardcoded value. pub type ConnectionId = usize; /// Subscription ID. pub type SubscriptionId = u64; -/// Sink that is used to send back the result to the server for a specific method. -pub type MethodSink = mpsc::UnboundedSender; -/// Max response size in bytes for a executed call. -pub type MaxResponseSize = u32; type Subscribers = Arc)>>>; @@ -105,6 +100,23 @@ pub struct MethodCallback { resources: MethodResources, } +/// Result of a method, either direct value or a future of one. +pub enum MethodResult { + /// Result by value + Sync(T), + /// Future of a value + Async(BoxFuture<'static, T>), +} + +impl Debug for MethodResult { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + MethodResult::Sync(result) => result.fmt(f), + MethodResult::Async(_) => f.write_str(""), + } + } +} + /// Builder for configuring resources used by a method. #[derive(Debug)] pub struct MethodResourcesBuilder<'a> { @@ -147,16 +159,15 @@ impl MethodCallback { /// Execute the callback, sending the resulting JSON (success or error) to the specified sink. pub fn execute( &self, - tx: &MethodSink, + sink: &MethodSink, req: Request<'_>, conn_id: ConnectionId, claimed: Option, - max_response_size: MaxResponseSize, - ) -> Option> { + ) -> MethodResult { let id = req.id.clone(); let params = Params::new(req.params.map(|params| params.get())); - match &self.callback { + let result = match &self.callback { MethodKind::Sync(callback) => { tracing::trace!( "[MethodCallback::execute] Executing sync callback, params={:?}, req.id={:?}, conn_id={:?}", @@ -164,15 +175,16 @@ impl MethodCallback { id, conn_id ); - (callback)(id, params, tx, conn_id, max_response_size); + + let result = (callback)(id, params, sink, conn_id); // Release claimed resources drop(claimed); - None + MethodResult::Sync(result) } MethodKind::Async(callback) => { - let tx = tx.clone(); + let sink = sink.clone(); let params = params.into_owned(); let id = id.into_owned(); tracing::trace!( @@ -182,9 +194,11 @@ impl MethodCallback { conn_id ); - Some((callback)(id, params, tx, claimed, max_response_size)) + MethodResult::Async((callback)(id, params, sink, claimed)) } - } + }; + + result } } @@ -288,46 +302,46 @@ impl Methods { self.callbacks.get(method_name) } + /// Returns the method callback along with its name. The returned name is same as the + /// `method_name`, but its lifetime bound is `'static`. + pub fn method_with_name(&self, method_name: &str) -> Option<(&'static str, &MethodCallback)> { + self.callbacks.get_key_value(method_name).map(|(k, v)| (*k, v)) + } + /// Attempt to execute a callback, sending the resulting JSON (success or error) to the specified sink. - pub fn execute( - &self, - tx: &MethodSink, - req: Request, - conn_id: ConnectionId, - max_response_size: MaxResponseSize, - ) -> Option> { + pub fn execute(&self, sink: &MethodSink, req: Request, conn_id: ConnectionId) -> MethodResult { tracing::trace!("[Methods::execute] Executing request: {:?}", req); match self.callbacks.get(&*req.method) { - Some(callback) => callback.execute(tx, req, conn_id, None, max_response_size), + Some(callback) => callback.execute(sink, req, conn_id, None), None => { - send_error(req.id, tx, ErrorCode::MethodNotFound.into()); - None + sink.send_error(req.id, ErrorCode::MethodNotFound.into()); + MethodResult::Sync(false) } } } - /// Attempt to execute a callback while checking that the call does not exhaust the available resources, sending the resulting JSON (success or error) to the specified sink. - pub fn execute_with_resources( + /// Attempt to execute a callback while checking that the call does not exhaust the available resources, + // sending the resulting JSON (success or error) to the specified sink. + pub fn execute_with_resources<'r>( &self, - tx: &MethodSink, - req: Request, + sink: &MethodSink, + req: Request<'r>, conn_id: ConnectionId, resources: &Resources, - max_response_size: MaxResponseSize, - ) -> Option> { + ) -> Result<(&'static str, MethodResult), Cow<'r, str>> { tracing::trace!("[Methods::execute_with_resources] Executing request: {:?}", req); - match self.callbacks.get(&*req.method) { - Some(callback) => match callback.claim(&req.method, resources) { - Ok(guard) => callback.execute(tx, req, conn_id, Some(guard), max_response_size), + match self.callbacks.get_key_value(&*req.method) { + Some((&name, callback)) => match callback.claim(&req.method, resources) { + Ok(guard) => Ok((name, callback.execute(sink, req, conn_id, Some(guard)))), Err(err) => { tracing::error!("[Methods::execute_with_resources] failed to lock resources: {:?}", err); - send_error(req.id, tx, ErrorCode::ServerIsBusy.into()); - None + sink.send_error(req.id, ErrorCode::ServerIsBusy.into()); + Ok((name, MethodResult::Sync(false))) } }, None => { - send_error(req.id, tx, ErrorCode::MethodNotFound.into()); - None + sink.send_error(req.id, ErrorCode::MethodNotFound.into()); + Err(req.method) } } } @@ -351,8 +365,9 @@ impl Methods { }; let (tx, mut rx) = mpsc::unbounded(); + let sink = MethodSink::new(tx); - if let Some(fut) = self.execute(&tx, req, 0, MaxResponseSize::MAX) { + if let MethodResult::Async(fut) = self.execute(&sink, req, 0) { fut.await; } @@ -368,8 +383,9 @@ impl Methods { Request { jsonrpc: TwoPointZero, id: Id::Number(0), method: Cow::borrowed(method), params: Some(¶ms) }; let (tx, mut rx) = mpsc::unbounded(); + let sink = MethodSink::new(tx.clone()); - if let Some(fut) = self.execute(&tx, req, 0, MaxResponseSize::MAX) { + if let MethodResult::Async(fut) = self.execute(&sink, req, 0) { fut.await; } let response = rx.next().await.expect("Could not establish subscription."); @@ -436,11 +452,9 @@ impl RpcModule { let ctx = self.ctx.clone(); let callback = self.methods.verify_and_insert( method_name, - MethodCallback::new_sync(Arc::new(move |id, params, tx, _, max_response_size| { - match callback(params, &*ctx) { - Ok(res) => send_response(id, tx, res, max_response_size), - Err(err) => send_call_error(id, tx, err), - }; + MethodCallback::new_sync(Arc::new(move |id, params, sink, _| match callback(params, &*ctx) { + Ok(res) => sink.send_response(id, res), + Err(err) => sink.send_call_error(id, err), })), )?; @@ -461,16 +475,18 @@ impl RpcModule { let ctx = self.ctx.clone(); let callback = self.methods.verify_and_insert( method_name, - MethodCallback::new_async(Arc::new(move |id, params, tx, claimed, max_response_size| { + MethodCallback::new_async(Arc::new(move |id, params, sink, claimed| { let ctx = ctx.clone(); let future = async move { - match callback(params, ctx).await { - Ok(res) => send_response(id, &tx, res, max_response_size), - Err(err) => send_call_error(id, &tx, err), + let result = match callback(params, ctx).await { + Ok(res) => sink.send_response(id, res), + Err(err) => sink.send_call_error(id, err), }; // Release claimed resources drop(claimed); + + result }; future.boxed() })), @@ -494,20 +510,26 @@ impl RpcModule { let ctx = self.ctx.clone(); let callback = self.methods.verify_and_insert( method_name, - MethodCallback::new_async(Arc::new(move |id, params, tx, claimed, max_response_size| { + MethodCallback::new_async(Arc::new(move |id, params, sink, claimed| { let ctx = ctx.clone(); tokio::task::spawn_blocking(move || { - match callback(params, ctx) { - Ok(res) => send_response(id, &tx, res, max_response_size), - Err(err) => send_call_error(id, &tx, err), + let result = match callback(params, ctx) { + Ok(res) => sink.send_response(id, res), + Err(err) => sink.send_call_error(id, err), }; // Release claimed resources drop(claimed); + + result }) - .map(|err| { - tracing::error!("Join error for blocking RPC method: {:?}", err); + .map(|result| match result { + Ok(r) => r, + Err(err) => { + tracing::error!("Join error for blocking RPC method: {:?}", err); + false + } }) .boxed() })), @@ -574,7 +596,7 @@ impl RpcModule { let subscribers = subscribers.clone(); self.methods.mut_callbacks().insert( subscribe_method_name, - MethodCallback::new_sync(Arc::new(move |id, params, method_sink, conn_id, max_response_size| { + MethodCallback::new_sync(Arc::new(move |id, params, method_sink, conn_id| { let (conn_tx, conn_rx) = oneshot::channel::<()>(); let sub_id = { const JS_NUM_MASK: SubscriptionId = !0 >> 11; @@ -586,7 +608,7 @@ impl RpcModule { sub_id }; - send_response(id.clone(), method_sink, sub_id, max_response_size); + method_sink.send_response(id.clone(), sub_id); let sink = SubscriptionSink { inner: method_sink.clone(), @@ -602,7 +624,9 @@ impl RpcModule { err, id ); - send_error(id, method_sink, ErrorCode::ServerError(CALL_EXECUTION_FAILED_CODE).into()); + method_sink.send_error(id, ErrorCode::ServerError(CALL_EXECUTION_FAILED_CODE).into()) + } else { + true } })), ); @@ -611,7 +635,7 @@ impl RpcModule { { self.methods.mut_callbacks().insert( unsubscribe_method_name, - MethodCallback::new_sync(Arc::new(move |id, params, tx, conn_id, max_response_size| { + MethodCallback::new_sync(Arc::new(move |id, params, sink, conn_id| { let sub_id = match params.one() { Ok(sub_id) => sub_id, Err(_) => { @@ -622,16 +646,15 @@ impl RpcModule { id ); let err = to_json_raw_value(&"Invalid subscription ID type, must be integer").ok(); - send_error(id, tx, invalid_subscription_err(err.as_deref())); - return; + return sink.send_error(id, invalid_subscription_err(err.as_deref())); } }; if subscribers.lock().remove(&SubscriptionKey { conn_id, sub_id }).is_some() { - send_response(id, tx, "Unsubscribed", max_response_size); + sink.send_response(id, "Unsubscribed") } else { let err = to_json_raw_value(&format!("Invalid subscription ID={}", sub_id)).ok(); - send_error(id, tx, invalid_subscription_err(err.as_deref())) + sink.send_error(id, invalid_subscription_err(err.as_deref())) } })), ); @@ -659,7 +682,7 @@ impl RpcModule { #[derive(Debug)] pub struct SubscriptionSink { /// Sink. - inner: mpsc::UnboundedSender, + inner: MethodSink, /// MethodCallback. method: &'static str, /// Unique subscription. @@ -692,7 +715,7 @@ impl SubscriptionSink { let res = match self.is_connected.as_ref() { Some(conn) if !conn.is_canceled() => { // unbounded send only fails if the receiver has been dropped. - self.inner.unbounded_send(msg).map_err(|_| { + self.inner.send_raw(msg).map_err(|_| { Some(SubscriptionClosedError::new("Closed by the client (connection reset)", self.uniq_sub.sub_id)) }) } @@ -722,7 +745,7 @@ impl SubscriptionSink { if let Some((sink, _)) = self.subscribers.lock().remove(&self.uniq_sub) { tracing::debug!("Closing subscription: {:?}", self.uniq_sub.sub_id); let msg = self.build_message(err).expect("valid json infallible; qed"); - let _ = sink.unbounded_send(msg); + let _ = sink.send_raw(msg); } } } diff --git a/ws-server/src/server.rs b/ws-server/src/server.rs index 6594f0d144..f60df7e682 100644 --- a/ws-server/src/server.rs +++ b/ws-server/src/server.rs @@ -32,36 +32,41 @@ use std::task::{Context, Poll}; use crate::future::{FutureDriver, ServerHandle, StopMonitor}; use crate::types::{ error::Error, + middleware::Middleware, v2::{ErrorCode, Id, Request}, TEN_MB_SIZE_BYTES, }; use futures_channel::mpsc; +use futures_util::future::join_all; use futures_util::future::FutureExt; use futures_util::io::{BufReader, BufWriter}; -use futures_util::stream::{self, StreamExt}; +use futures_util::stream::StreamExt; use soketto::connection::Error as SokettoError; use soketto::handshake::{server::Response, Server as SokettoServer}; use soketto::Sender; use tokio::net::{TcpListener, TcpStream, ToSocketAddrs}; use tokio_util::compat::{Compat, TokioAsyncReadCompatExt}; -use jsonrpsee_utils::server::helpers::{collect_batch_response, prepare_error, send_error}; -use jsonrpsee_utils::server::resource_limiting::Resources; -use jsonrpsee_utils::server::rpc_module::{ConnectionId, Methods}; +use jsonrpsee_utils::server::{ + helpers::{collect_batch_response, prepare_error, MethodSink}, + resource_limiting::Resources, + rpc_module::{ConnectionId, MethodResult, Methods}, +}; /// Default maximum connections allowed. const MAX_CONNECTIONS: u64 = 100; /// A WebSocket JSON RPC server. #[derive(Debug)] -pub struct Server { +pub struct Server { listener: TcpListener, cfg: Settings, stop_monitor: StopMonitor, resources: Resources, + middleware: M, } -impl Server { +impl Server { /// Returns socket address to which the server is bound. pub fn local_addr(&self) -> Result { self.listener.local_addr().map_err(Into::into) @@ -88,6 +93,7 @@ impl Server { async fn start_inner(self, methods: Methods) { let stop_monitor = self.stop_monitor; let resources = self.resources; + let middleware = self.middleware; let mut id = 0; let mut connections = FutureDriver::default(); @@ -118,6 +124,7 @@ impl Server { resources: &resources, cfg, stop_monitor: &stop_monitor, + middleware: middleware.clone(), }, ))); @@ -184,7 +191,7 @@ where } } -enum HandshakeResponse<'a> { +enum HandshakeResponse<'a, M> { Reject { status_code: u16, }, @@ -194,10 +201,14 @@ enum HandshakeResponse<'a> { resources: &'a Resources, cfg: &'a Settings, stop_monitor: &'a StopMonitor, + middleware: M, }, } -async fn handshake(socket: tokio::net::TcpStream, mode: HandshakeResponse<'_>) -> Result<(), Error> { +async fn handshake(socket: tokio::net::TcpStream, mode: HandshakeResponse<'_, M>) -> Result<(), Error> +where + M: Middleware, +{ // For each incoming background_task we perform a handshake. let mut server = SokettoServer::new(BufReader::new(BufWriter::new(socket.compat()))); @@ -214,7 +225,7 @@ async fn handshake(socket: tokio::net::TcpStream, mode: HandshakeResponse<'_>) - Ok(()) } - HandshakeResponse::Accept { conn_id, methods, resources, cfg, stop_monitor } => { + HandshakeResponse::Accept { conn_id, methods, resources, cfg, stop_monitor, middleware } => { tracing::debug!("Accepting new connection: {}", conn_id); let key = { let req = server.receive_request().await?; @@ -244,6 +255,7 @@ async fn handshake(socket: tokio::net::TcpStream, mode: HandshakeResponse<'_>) - resources.clone(), cfg.max_request_body_size, stop_monitor.clone(), + middleware, )) .await; @@ -262,6 +274,7 @@ async fn background_task( resources: Resources, max_request_body_size: u32, stop_server: StopMonitor, + middleware: impl Middleware, ) -> Result<(), Error> { // And we can finally transition to a websocket background_task. let mut builder = server.into_builder(); @@ -269,6 +282,9 @@ async fn background_task( let (mut sender, mut receiver) = builder.finish(); let (tx, mut rx) = mpsc::unbounded::(); let stop_server2 = stop_server.clone(); + let sink = MethodSink::new_with_limit(tx, max_request_body_size); + + middleware.on_connect(); // Send results back to the client. tokio::spawn(async move { @@ -293,8 +309,9 @@ async fn background_task( // Buffer for incoming data. let mut data = Vec::with_capacity(100); let mut method_executors = FutureDriver::default(); + let middleware = &middleware; - loop { + let result = loop { data.clear(); { @@ -307,8 +324,8 @@ async fn background_task( match err { MonitoredError::Selector(SokettoError::Closed) => { tracing::debug!("WS transport error: remote peer terminated the connection: {}", conn_id); - tx.close_channel(); - return Ok(()); + sink.close(); + break Ok(()); } MonitoredError::Selector(SokettoError::MessageTooLarge { current, maximum }) => { tracing::warn!( @@ -316,35 +333,56 @@ async fn background_task( current, maximum ); - send_error(Id::Null, &tx, ErrorCode::OversizedRequest.into()); + sink.send_error(Id::Null, ErrorCode::OversizedRequest.into()); continue; } // These errors can not be gracefully handled, so just log them and terminate the connection. MonitoredError::Selector(err) => { tracing::error!("WS transport error: {:?} => terminating connection {}", err, conn_id); - tx.close_channel(); - return Err(err.into()); + sink.close(); + break Err(err.into()); } - MonitoredError::Shutdown => break, + MonitoredError::Shutdown => break Ok(()), }; }; }; tracing::debug!("recv {} bytes", data.len()); + let request_start = middleware.on_request(); + match data.get(0) { Some(b'{') => { if let Ok(req) = serde_json::from_slice::(&data) { + middleware.on_call(req.method.as_ref()); + tracing::debug!("recv method call={}", req.method); tracing::trace!("recv: req={:?}", req); - if let Some(fut) = - methods.execute_with_resources(&tx, req, conn_id, &resources, max_request_body_size) - { - method_executors.add(fut); + match methods.execute_with_resources(&sink, req, conn_id, &resources) { + Ok((name, MethodResult::Sync(success))) => { + middleware.on_result(name, success, request_start); + middleware.on_response(request_start); + } + Ok((name, MethodResult::Async(fut))) => { + let request_start = request_start; + + let fut = async move { + let success = fut.await; + middleware.on_result(name, success, request_start); + middleware.on_response(request_start); + }; + + method_executors.add(fut.boxed()); + } + Err(name) => { + middleware.on_result(name.as_ref(), false, request_start); + middleware.on_response(request_start); + } } } else { let (id, code) = prepare_error(&data); - send_error(id, &tx, code.into()); + sink.send_error(id, code.into()); + middleware.on_response(request_start); } } Some(b'[') => { @@ -352,57 +390,71 @@ async fn background_task( let d = std::mem::take(&mut data); let resources = &resources; let methods = &methods; - let tx2 = tx.clone(); + let sink = sink.clone(); let fut = async move { // Batch responses must be sent back as a single message so we read the results from each // request in the batch and read the results off of a new channel, `rx_batch`, and then send the // complete batch response back to the client over `tx`. let (tx_batch, mut rx_batch) = mpsc::unbounded(); + let sink_batch = MethodSink::new_with_limit(tx_batch, max_request_body_size); if let Ok(batch) = serde_json::from_slice::>(&d) { tracing::debug!("recv batch len={}", batch.len()); tracing::trace!("recv: batch={:?}", batch); if !batch.is_empty() { - let methods_stream = stream::iter(batch.into_iter().filter_map(|req| { - methods.execute_with_resources( - &tx_batch, - req, - conn_id, - resources, - max_request_body_size, - ) - })); - - let results = methods_stream - .for_each_concurrent(None, |item| item) - .then(|_| { - rx_batch.close(); - collect_batch_response(rx_batch) - }) - .await; - - if let Err(err) = tx2.unbounded_send(results) { + join_all(batch.into_iter().filter_map(move |req| { + match methods.execute_with_resources(&sink_batch, req, conn_id, resources) { + Ok((name, MethodResult::Sync(success))) => { + middleware.on_result(name, success, request_start); + None + } + Ok((name, MethodResult::Async(fut))) => Some(async move { + let success = fut.await; + middleware.on_result(name, success, request_start); + }), + Err(name) => { + middleware.on_result(name.as_ref(), false, request_start); + None + } + } + })) + .await; + + rx_batch.close(); + let results = collect_batch_response(rx_batch).await; + + if let Err(err) = sink.send_raw(results) { tracing::error!("Error sending batch response to the client: {:?}", err) + } else { + middleware.on_response(request_start); } } else { - send_error(Id::Null, &tx2, ErrorCode::InvalidRequest.into()); + sink.send_error(Id::Null, ErrorCode::InvalidRequest.into()); + middleware.on_response(request_start); } } else { let (id, code) = prepare_error(&d); - send_error(id, &tx2, code.into()); + sink.send_error(id, code.into()); + middleware.on_response(request_start); } }; method_executors.add(Box::pin(fut)); } - _ => send_error(Id::Null, &tx, ErrorCode::ParseError.into()), + _ => { + sink.send_error(Id::Null, ErrorCode::ParseError.into()); + } } - } + }; - // Drive all running methods to completion + middleware.on_disconnect(); + + // Drive all running methods to completion. + // **NOTE** Do not return early in this function. This `await` needs to run to guarantee + // proper drop behaviour. method_executors.await; - Ok(()) + result } #[derive(Debug, Clone)] @@ -453,13 +505,27 @@ impl Default for Settings { } /// Builder to configure and create a JSON-RPC Websocket server -#[derive(Debug, Default)] -pub struct Builder { +#[derive(Debug)] +pub struct Builder { settings: Settings, resources: Resources, + middleware: M, +} + +impl Default for Builder { + fn default() -> Self { + Builder { settings: Settings::default(), resources: Resources::default(), middleware: () } + } } impl Builder { + /// Create a default server builder. + pub fn new() -> Self { + Self::default() + } +} + +impl Builder { /// Set the maximum size of a request body in bytes. Default is 10 MiB. pub fn max_request_body_size(mut self, size: u32) -> Self { self.settings.max_request_body_size = size; @@ -511,6 +577,34 @@ impl Builder { Ok(self) } + /// Add a middleware to the builder [`Middleware`](../jsonrpsee_types/middleware/trait.Middleware.html). + /// + /// ``` + /// use jsonrpsee_types::middleware::Middleware; + /// use jsonrpsee_ws_server::WsServerBuilder; + /// use std::time::Instant; + /// + /// #[derive(Clone)] + /// struct MyMiddleware; + /// + /// impl Middleware for MyMiddleware { + /// type Instant = Instant; + /// + /// fn on_request(&self) -> Instant { + /// Instant::now() + /// } + /// + /// fn on_result(&self, name: &str, success: bool, started_at: Instant) { + /// println!("Call to '{}' took {:?}", name, started_at.elapsed()); + /// } + /// } + /// + /// let builder = WsServerBuilder::new().set_middleware(MyMiddleware); + /// ``` + pub fn set_middleware(self, middleware: T) -> Builder { + Builder { settings: self.settings, resources: self.resources, middleware } + } + /// Restores the default behavior of allowing connections with `Origin` header /// containing any value. This will undo any list set by [`set_allowed_origins`](Builder::set_allowed_origins). pub fn allow_all_origins(mut self) -> Self { @@ -578,11 +672,11 @@ impl Builder { /// } /// ``` /// - pub async fn build(self, addrs: impl ToSocketAddrs) -> Result { + pub async fn build(self, addrs: impl ToSocketAddrs) -> Result, Error> { let listener = TcpListener::bind(addrs).await?; let stop_monitor = StopMonitor::new(); let resources = self.resources; - Ok(Server { listener, cfg: self.settings, stop_monitor, resources }) + Ok(Server { listener, cfg: self.settings, stop_monitor, resources, middleware: self.middleware }) } } From bdc25a88616571bcd2fe5dc7b1bf2605b65b51ad Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Wed, 1 Dec 2021 11:59:47 +0100 Subject: [PATCH 29/31] deps: tokio ^1.8 (#586) --- benches/Cargo.toml | 2 +- examples/Cargo.toml | 2 +- http-client/Cargo.toml | 4 ++-- http-server/Cargo.toml | 2 +- proc-macros/Cargo.toml | 2 +- test-utils/Cargo.toml | 2 +- tests/Cargo.toml | 2 +- utils/Cargo.toml | 4 ++-- ws-client/Cargo.toml | 4 ++-- ws-server/Cargo.toml | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 8526386980..c4ffeb994f 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -17,7 +17,7 @@ jsonrpc-http-server = "18.0.0" jsonrpc-pubsub = "18.0.0" num_cpus = "1" serde_json = "1" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1.8", features = ["full"] } [[bench]] name = "bench" diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e8704bb3a9..cdd8205db6 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -12,7 +12,7 @@ env_logger = "0.9" jsonrpsee = { path = "../jsonrpsee", features = ["full"] } tracing = "0.1" tracing-subscriber = "0.2" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1.8", features = ["full"] } palaver = "0.2" [[example]] diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index 47d154e8ae..9e7251c5ee 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -19,10 +19,10 @@ jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["client", serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1.8", features = ["time"] } tracing = "0.1" url = "2.2" [dev-dependencies] jsonrpsee-test-utils = { path = "../test-utils" } -tokio = { package = "tokio", version = "1", features = ["net", "rt-multi-thread", "macros"] } +tokio = { version = "1.8", features = ["net", "rt-multi-thread", "macros"] } diff --git a/http-server/Cargo.toml b/http-server/Cargo.toml index 5434457125..d28802c9fc 100644 --- a/http-server/Cargo.toml +++ b/http-server/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.4" tracing = "0.1" serde_json = "1" socket2 = "0.4" -tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +tokio = { version = "1.8", features = ["rt-multi-thread", "macros"] } unicase = "2.6.0" [dev-dependencies] diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml index 4d5baf7d83..c8878dcc2b 100644 --- a/proc-macros/Cargo.toml +++ b/proc-macros/Cargo.toml @@ -21,5 +21,5 @@ proc-macro-crate = "1" [dev-dependencies] jsonrpsee = { path = "../jsonrpsee", features = ["full"] } trybuild = "1.0" -tokio = { version = "1", features = ["rt", "macros"] } +tokio = { version = "1.8", features = ["rt", "macros"] } futures-channel = { version = "0.3.14", default-features = false } diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 15df549efe..fb883a4094 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -16,5 +16,5 @@ tracing = "0.1" serde = { version = "1", default-features = false, features = ["derive"] } serde_json = "1" soketto = { version = "0.7.1", features = ["http"] } -tokio = { version = "1", features = ["net", "rt-multi-thread", "macros", "time"] } +tokio = { version = "1.8", features = ["net", "rt-multi-thread", "macros", "time"] } tokio-util = { version = "0.6", features = ["compat"] } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 516498d72b..b6bb22fd01 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -12,6 +12,6 @@ env_logger = "0.8" beef = { version = "0.5.1", features = ["impl_serde"] } futures = { version = "0.3.14", default-features = false, features = ["std"] } jsonrpsee = { path = "../jsonrpsee", features = ["full"] } -tokio = { version = "1", features = ["full"] } +tokio = { version = "1.8", features = ["full"] } tracing = "0.1" serde_json = "1" diff --git a/utils/Cargo.toml b/utils/Cargo.toml index af49be62d8..fb6671fabf 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -20,7 +20,7 @@ rand = { version = "0.8", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } serde_json = { version = "1", features = ["raw_value"], optional = true } parking_lot = { version = "0.11", optional = true } -tokio = { version = "1", features = ["rt"], optional = true } +tokio = { version = "1.8", features = ["rt"], optional = true } [features] default = [] @@ -42,5 +42,5 @@ client = ["jsonrpsee-types"] [dev-dependencies] serde_json = "1.0" -tokio = { version = "1", features = ["macros", "rt"] } +tokio = { version = "1.8", features = ["macros", "rt"] } jsonrpsee = { path = "../jsonrpsee", features = ["server", "macros"] } diff --git a/ws-client/Cargo.toml b/ws-client/Cargo.toml index 59b56dd8dc..e19b901b4b 100644 --- a/ws-client/Cargo.toml +++ b/ws-client/Cargo.toml @@ -21,7 +21,7 @@ serde = "1" serde_json = "1" soketto = "0.7.1" thiserror = "1" -tokio = { version = "1", features = ["net", "time", "rt-multi-thread", "macros"] } +tokio = { version = "1.8", features = ["net", "time", "rt-multi-thread", "macros"] } tokio-rustls = "0.23" tokio-util = { version = "0.6", features = ["compat"] } tracing = "0.1" @@ -31,4 +31,4 @@ webpki-roots = "0.22.0" env_logger = "0.9" jsonrpsee-test-utils = { path = "../test-utils" } jsonrpsee-utils = { path = "../utils", features = ["client"] } -tokio = { version = "1", features = ["macros"] } +tokio = { version = "1.8", features = ["macros"] } diff --git a/ws-server/Cargo.toml b/ws-server/Cargo.toml index b31466871a..69efb66c19 100644 --- a/ws-server/Cargo.toml +++ b/ws-server/Cargo.toml @@ -17,7 +17,7 @@ jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["server"] tracing = "0.1" serde_json = { version = "1", features = ["raw_value"] } soketto = "0.7.1" -tokio = { version = "1", features = ["net", "rt-multi-thread", "macros", "time"] } +tokio = { version = "1.8", features = ["net", "rt-multi-thread", "macros", "time"] } tokio-util = { version = "0.6", features = ["compat"] } [dev-dependencies] From be6f64ae65baf5ad1a5a0de8487aaf3407d39c5f Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Wed, 1 Dec 2021 12:41:26 +0100 Subject: [PATCH 30/31] chore: release v0.6.0 (#587) --- CHANGELOG.md | 14 ++++++++++++++ benches/Cargo.toml | 2 +- examples/Cargo.toml | 2 +- http-client/Cargo.toml | 6 +++--- http-server/Cargo.toml | 6 +++--- jsonrpsee/Cargo.toml | 16 ++++++++-------- proc-macros/Cargo.toml | 2 +- test-utils/Cargo.toml | 2 +- tests/Cargo.toml | 2 +- types/Cargo.toml | 2 +- utils/Cargo.toml | 4 ++-- ws-client/Cargo.toml | 4 ++-- ws-server/Cargo.toml | 6 +++--- 13 files changed, 41 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa70eae94..7d96d4eb20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ The format is based on [Keep a Changelog]. ## [Unreleased] +## [v0.6.0] – 2021-12-01 + +v0.6 is a breaking release + +### [Added] + +- Servers: Middleware for metrics [#576](https://github.com/paritytech/jsonrpsee/pull/576) +- http client: impl Clone [#583](https://github.com/paritytech/jsonrpsee/pull/583) + +### [Fixed] +- types: use Cow for deserializing str [#584](https://github.com/paritytech/jsonrpsee/pull/584) +- deps: require tokio ^1.8 [#586](https://github.com/paritytech/jsonrpsee/pull/586) + + ## [v0.5.1] – 2021-11-26 The v0.5.1 release is a bug fix. diff --git a/benches/Cargo.toml b/benches/Cargo.toml index c4ffeb994f..35d3fb454d 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-benchmarks" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies "] description = "Benchmarks for jsonrpsee" edition = "2018" diff --git a/examples/Cargo.toml b/examples/Cargo.toml index cdd8205db6..734a1e2197 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-examples" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies "] description = "Examples for jsonrpsee" edition = "2018" diff --git a/http-client/Cargo.toml b/http-client/Cargo.toml index 9e7251c5ee..87369fd228 100644 --- a/http-client/Cargo.toml +++ b/http-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-http-client" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "HTTP client for JSON-RPC" edition = "2018" @@ -14,8 +14,8 @@ async-trait = "0.1" fnv = "1" hyper = { version = "0.14.10", features = ["client", "http1", "http2", "tcp"] } hyper-rustls = { version = "0.23", features = ["webpki-tokio"] } -jsonrpsee-types = { path = "../types", version = "0.5.1" } -jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["client", "http-helpers"] } +jsonrpsee-types = { path = "../types", version = "0.6.0" } +jsonrpsee-utils = { path = "../utils", version = "0.6.0", features = ["client", "http-helpers"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" thiserror = "1.0" diff --git a/http-server/Cargo.toml b/http-server/Cargo.toml index d28802c9fc..49b61e7c9f 100644 --- a/http-server/Cargo.toml +++ b/http-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-http-server" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "HTTP server for JSON-RPC" edition = "2018" @@ -13,8 +13,8 @@ documentation = "https://docs.rs/jsonrpsee-http-server" hyper = { version = "0.14.10", features = ["server", "http1", "http2", "tcp"] } futures-channel = "0.3.14" futures-util = { version = "0.3.14", default-features = false } -jsonrpsee-types = { path = "../types", version = "0.5.1" } -jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["server", "http-helpers"] } +jsonrpsee-types = { path = "../types", version = "0.6.0" } +jsonrpsee-utils = { path = "../utils", version = "0.6.0", features = ["server", "http-helpers"] } globset = "0.4" lazy_static = "1.4" tracing = "0.1" diff --git a/jsonrpsee/Cargo.toml b/jsonrpsee/Cargo.toml index e06da8206c..87bc2c0247 100644 --- a/jsonrpsee/Cargo.toml +++ b/jsonrpsee/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "jsonrpsee" description = "JSON-RPC crate" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies ", "Pierre Krieger "] license = "MIT" edition = "2018" @@ -12,13 +12,13 @@ documentation = "https://docs.rs/jsonrpsee" [dependencies] # No support for namespaced features yet so workspace dependencies are prefixed with `jsonrpsee-`. # See https://github.com/rust-lang/cargo/issues/5565 for more details. -jsonrpsee-http-client = { path = "../http-client", version = "0.5.1", package = "jsonrpsee-http-client", optional = true } -jsonrpsee-http-server = { path = "../http-server", version = "0.5.1", package = "jsonrpsee-http-server", optional = true } -jsonrpsee-ws-client = { path = "../ws-client", version = "0.5.1", package = "jsonrpsee-ws-client", optional = true } -jsonrpsee-ws-server = { path = "../ws-server", version = "0.5.1", package = "jsonrpsee-ws-server", optional = true } -jsonrpsee-proc-macros = { path = "../proc-macros", version = "0.5.1", package = "jsonrpsee-proc-macros", optional = true } -jsonrpsee-utils = { path = "../utils", version = "0.5.1", package = "jsonrpsee-utils", optional = true } -jsonrpsee-types = { path = "../types", version = "0.5.1", package = "jsonrpsee-types", optional = true } +jsonrpsee-http-client = { path = "../http-client", version = "0.6.0", package = "jsonrpsee-http-client", optional = true } +jsonrpsee-http-server = { path = "../http-server", version = "0.6.0", package = "jsonrpsee-http-server", optional = true } +jsonrpsee-ws-client = { path = "../ws-client", version = "0.6.0", package = "jsonrpsee-ws-client", optional = true } +jsonrpsee-ws-server = { path = "../ws-server", version = "0.6.0", package = "jsonrpsee-ws-server", optional = true } +jsonrpsee-proc-macros = { path = "../proc-macros", version = "0.6.0", package = "jsonrpsee-proc-macros", optional = true } +jsonrpsee-utils = { path = "../utils", version = "0.6.0", package = "jsonrpsee-utils", optional = true } +jsonrpsee-types = { path = "../types", version = "0.6.0", package = "jsonrpsee-types", optional = true } [features] http-client = ["jsonrpsee-http-client", "jsonrpsee-types", "jsonrpsee-utils/client"] diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml index c8878dcc2b..4e350017ab 100644 --- a/proc-macros/Cargo.toml +++ b/proc-macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "jsonrpsee-proc-macros" description = "Procedueral macros for jsonrpsee" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies ", "Pierre Krieger "] license = "MIT" edition = "2018" diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index fb883a4094..b4e76853e6 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-test-utils" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies "] license = "MIT" edition = "2018" diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b6bb22fd01..c35161ffb2 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-integration-tests" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies "] description = "Integration tests for jsonrpsee" edition = "2018" diff --git a/types/Cargo.toml b/types/Cargo.toml index 381abdba50..b4488810ed 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-types" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies "] description = "Shared types for jsonrpsee" edition = "2018" diff --git a/utils/Cargo.toml b/utils/Cargo.toml index fb6671fabf..bc6d7b2779 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-utils" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies "] description = "Utilities for jsonrpsee" edition = "2018" @@ -13,7 +13,7 @@ thiserror = { version = "1", optional = true } futures-channel = { version = "0.3.14", default-features = false, optional = true } futures-util = { version = "0.3.14", default-features = false, optional = true } hyper = { version = "0.14.10", default-features = false, features = ["stream"], optional = true } -jsonrpsee-types = { path = "../types", version = "0.5.1", optional = true } +jsonrpsee-types = { path = "../types", version = "0.6.0", optional = true } tracing = { version = "0.1", optional = true } rustc-hash = { version = "1", optional = true } rand = { version = "0.8", optional = true } diff --git a/ws-client/Cargo.toml b/ws-client/Cargo.toml index e19b901b4b..2b054d7aaa 100644 --- a/ws-client/Cargo.toml +++ b/ws-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-ws-client" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "WebSocket client for JSON-RPC" edition = "2018" @@ -14,7 +14,7 @@ async-trait = "0.1" fnv = "1" futures = { version = "0.3.14", default-features = false, features = ["std"] } http = "0.2" -jsonrpsee-types = { path = "../types", version = "0.5.1" } +jsonrpsee-types = { path = "../types", version = "0.6.0" } pin-project = "1" rustls-native-certs = "0.6.0" serde = "1" diff --git a/ws-server/Cargo.toml b/ws-server/Cargo.toml index 69efb66c19..f7f327f1a5 100644 --- a/ws-server/Cargo.toml +++ b/ws-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jsonrpsee-ws-server" -version = "0.5.1" +version = "0.6.0" authors = ["Parity Technologies ", "Pierre Krieger "] description = "WebSocket server for JSON-RPC" edition = "2018" @@ -12,8 +12,8 @@ documentation = "https://docs.rs/jsonrpsee-ws-server" [dependencies] futures-channel = "0.3.14" futures-util = { version = "0.3.14", default-features = false, features = ["io", "async-await-macro"] } -jsonrpsee-types = { path = "../types", version = "0.5.1" } -jsonrpsee-utils = { path = "../utils", version = "0.5.1", features = ["server"] } +jsonrpsee-types = { path = "../types", version = "0.6.0" } +jsonrpsee-utils = { path = "../utils", version = "0.6.0", features = ["server"] } tracing = "0.1" serde_json = { version = "1", features = ["raw_value"] } soketto = "0.7.1" From 66aa6c49175da7195d4ced15543d4a90a69cd015 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Thu, 2 Dec 2021 16:33:52 +0100 Subject: [PATCH 31/31] Create gitlab pipeline (#534) * add badge to readme * first version of pipeline * Update .gitlab-ci.yml Co-authored-by: Niklas Adolfsson * add pre-cache script * fmt and clippy stable * add check and test * remove output text file from bench * Update scripts/ci/pre_cache.sh Co-authored-by: David * Update .gitlab-ci.yml Co-authored-by: Niklas Adolfsson * small fix * fix test and schedule * CI: verbose is a surplus * CI: separately check rustdoc linx * fix refs * add bench to gh-pages * fix refs * fix benchmarks * added vault to ci * fix vars * comment bench * fix benches name * added script to push benchmark results to VM * make script executable * change bench psuh executor * changed benchmark task to run on a dedicated node pool * change prometheus metric name for benchmarks * send 2 metrics with benchmark results * disable non-schedule jobs from schedule run * empty commit for benchmark test * change metric name * empty commit for benchmark test * empty commit for benchmark test * add cirunner label to vm metric * split vm metric to 2 metrics * change runner description to runner tag in ci scripts * add pass runner tags from benchmark to publish job * change runner tag to runner description * add debug message * empty commit for test * empty commit for test * Update .scripts/ci/push_bench_results.sh Co-authored-by: Denis Pisarev <17856421+TriplEight@users.noreply.github.com> * add defaults, remove dups, change ci image for publish-bench * remove pre_cache.sh * move interruptible to defaults * add issue to fixme comment Co-authored-by: Niklas Adolfsson Co-authored-by: David Co-authored-by: Denis P Co-authored-by: Denis Pisarev <17856421+TriplEight@users.noreply.github.com> --- .gitlab-ci.yml | 149 ++++++++++++++++++++++++++++++ .scripts/ci/push_bench_results.sh | 28 ++++++ README.md | 2 + benches/bench.rs | 6 +- 4 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100755 .scripts/ci/push_bench_results.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..e859b9320a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,149 @@ +default: + interruptible: true + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + +stages: + - lint + - test + - benchmark + - publish + +variables: &default-vars + GIT_STRATEGY: fetch + GIT_DEPTH: 100 + CARGO_INCREMENTAL: 0 + CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" + CI_IMAGE: "paritytech/ci-linux:production" + VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io" + VAULT_AUTH_PATH: "gitlab-parity-io-jwt" + VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}" + +.vault-secrets: &vault-secrets + secrets: + GITHUB_TOKEN: + vault: cicd/gitlab/parity/GITHUB_TOKEN@kv + file: false + GITHUB_USER: + vault: cicd/gitlab/parity/GITHUB_USER@kv + file: false + +.common-refs: &common-refs + rules: + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_COMMIT_REF_NAME == "main" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_PIPELINE_SOURCE == "schedule" + when: never + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + +# run nightly by schedule +.schedule-refs: &schedule-refs + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + +.rust-info-script: &rust-info-script + - rustup show + - cargo --version + - rustup +nightly show + - cargo +nightly --version + - bash --version + - sccache -s + +.docker-env: &docker-env + image: "${CI_IMAGE}" + before_script: + - *rust-info-script + - sccache -s + tags: + - linux-docker + +.kubernetes-env: &kubernetes-env + image: "${CI_IMAGE}" + tags: + - kubernetes-parity-build + +.collect-artifacts: &collect-artifacts + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" + when: on_success + expire_in: 28 days + paths: + - ./artifacts/ + reports: + dotenv: runner.env +#### stage: lint + +fmt: + stage: lint + <<: *docker-env + <<: *common-refs + script: + # FIXME: remove component add after https://github.com/paritytech/substrate/issues/10411 is fixed + - rustup component add rustfmt + - cargo fmt --all -- --check + +clippy: + stage: lint + <<: *docker-env + <<: *common-refs + script: + # FIXME: remove component add after https://github.com/paritytech/substrate/issues/10411 is fixed + - rustup component add clippy + - cargo clippy --all-targets + +check-rustdoc-links: + stage: lint + <<: *docker-env + <<: *common-refs + script: + - RUSTDOCFLAGS="--deny broken_intra_doc_links" cargo doc --workspace --no-deps --document-private-items + +#### stage: test + +check-code: + stage: test + <<: *docker-env + <<: *common-refs + script: + - cargo install cargo-hack + - cargo hack check --workspace --each-feature + +test: + stage: test + <<: *docker-env + <<: *common-refs + script: + - cargo test + +benchmarks: + stage: benchmark + <<: *docker-env + <<: *collect-artifacts + <<: *schedule-refs + script: + - cargo bench -p jsonrpsee-benchmarks -- --output-format bencher | tee output.txt + - mkdir artifacts + - cp output.txt artifacts/ + - echo ${CI_RUNNER_DESCRIPTION} + - echo "RUNNER_NAME=${CI_RUNNER_DESCRIPTION}" > runner.env + tags: + - linux-docker-benches + +publish-bench: + stage: publish + variables: + CI_IMAGE: "paritytech/tools:latest" + <<: *kubernetes-env + <<: *schedule-refs + needs: + - job: benchmarks + artifacts: true + script: + - echo $RUNNER_NAME + - .scripts/ci/push_bench_results.sh artifacts/output.txt + diff --git a/.scripts/ci/push_bench_results.sh b/.scripts/ci/push_bench_results.sh new file mode 100755 index 0000000000..27abcbce58 --- /dev/null +++ b/.scripts/ci/push_bench_results.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# The script takes output.txt, removes every line that doesn't have "test" +# in it and pushes benchmark result to Victoria Metrics +# Benchmark name should have underscores in the name instead of spaces (e.g. async/http_concurrent_round_trip/8) + +RESULT_FILE=$1 +CURRENT_DIR=$(pwd) + +if [ -z "$RESULT_FILE" ] +then + RESULT_FILE="output.txt" +fi + +cat $RESULT_FILE | grep test > $CURRENT_DIR/output_redacted.txt + +INPUT="output_redacted.txt" + +while IFS= read -r line +do + BENCH_NAME=$(echo $line | cut -f 2 -d ' ') + BENCH_RESULT=$(echo $line | cut -f 5 -d ' ') + # send metric with common results + curl -d 'parity_benchmark_common_result_ns{project="'${CI_PROJECT_NAME}'",benchmark="'$BENCH_NAME'"} '$BENCH_RESULT'' \ + -X POST 'http://vm-longterm.parity-build.parity.io/api/v1/import/prometheus' + # send metric with detailed results + curl -d 'parity_benchmark_specific_result_ns{project="'${CI_PROJECT_NAME}'",benchmark="'$BENCH_NAME'",commit="'${CI_COMMIT_SHORT_SHA}'",cirunner="'${RUNNER_NAME}'"} '$BENCH_RESULT'' \ + -X POST 'http://vm-longterm.parity-build.parity.io/api/v1/import/prometheus' +done < "$INPUT" diff --git a/README.md b/README.md index cdff998ae2..edc52cd3dc 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![GitLab Status](https://gitlab.parity.io/parity/jsonrpsee/badges/master/pipeline.svg)](https://gitlab.parity.io/parity/jsonrpsee/pipelines) + # jsonrpsee JSON-RPC library designed for async/await in Rust. diff --git a/benches/bench.rs b/benches/bench.rs index f69b7905ea..a24c40823f 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -95,7 +95,7 @@ trait RequestBencher { let rt = TokioRuntime::new().unwrap(); let (url, _server) = rt.block_on(helpers::http_server(rt.handle().clone())); let client = Arc::new(HttpClientBuilder::default().max_concurrent_requests(1024 * 1024).build(&url).unwrap()); - run_round_trip_with_batch(&rt, crit, client, "http batch requests", Self::REQUEST_TYPE); + run_round_trip_with_batch(&rt, crit, client, "http_batch_requests", Self::REQUEST_TYPE); } fn websocket_requests(crit: &mut Criterion) { @@ -113,7 +113,7 @@ trait RequestBencher { let (url, _server) = rt.block_on(helpers::ws_server(rt.handle().clone())); let client = Arc::new(rt.block_on(WsClientBuilder::default().max_concurrent_requests(1024 * 1024).build(&url)).unwrap()); - run_round_trip_with_batch(&rt, crit, client, "ws batch requests", Self::REQUEST_TYPE); + run_round_trip_with_batch(&rt, crit, client, "ws_batch_requests", Self::REQUEST_TYPE); } fn subscriptions(crit: &mut Criterion) { @@ -188,7 +188,7 @@ fn run_sub_round_trip(rt: &TokioRuntime, crit: &mut Criterion, client: Arc