From decd30feb691cccb923477a686608c2466bfc814 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 21 Jan 2021 15:07:21 +0100 Subject: [PATCH 01/11] Add Agreement lock to send message to provider and update Agreement state atomically --- core/market/src/negotiation/common.rs | 6 ++ core/market/src/negotiation/requestor.rs | 122 +++++++++++++---------- core/market/src/utils.rs | 3 + core/market/src/utils/agreement_lock.rs | 45 +++++++++ 4 files changed, 125 insertions(+), 51 deletions(-) create mode 100644 core/market/src/utils/agreement_lock.rs diff --git a/core/market/src/negotiation/common.rs b/core/market/src/negotiation/common.rs index 12d46d5a8a..3b0f7f1bca 100644 --- a/core/market/src/negotiation/common.rs +++ b/core/market/src/negotiation/common.rs @@ -43,6 +43,7 @@ use crate::protocol::negotiation::{ messages::{AgreementTerminated, ProposalReceived}, }; use crate::utils::display::EnableDisplay; +use crate::utils::AgreementLock; type IsFirst = bool; @@ -54,6 +55,7 @@ pub struct CommonBroker { pub(super) session_notifier: EventNotifier, pub(super) agreement_notifier: EventNotifier, pub(super) config: Arc, + pub(super) agreement_lock: AgreementLock, } impl CommonBroker { @@ -70,6 +72,7 @@ impl CommonBroker { session_notifier, agreement_notifier: EventNotifier::new(), config, + agreement_lock: AgreementLock::new(), } } @@ -370,7 +373,9 @@ impl CommonBroker { dao.terminate(&agreement.id, reason_string, agreement.id.owner()) .await .map_err(|e| AgreementError::UpdateState((&agreement.id).clone(), e))?; + self.notify_agreement(&agreement).await; + self.agreement_lock.clear_locks(&agreement.id).await; inc_terminate_metrics(&reason, agreement.id.owner()); log::info!( @@ -457,6 +462,7 @@ impl CommonBroker { })?; self.notify_agreement(&agreement).await; + self.agreement_lock.clear_locks(&agreement_id).await; inc_terminate_metrics(&msg.reason, agreement.id.owner()); log::info!( diff --git a/core/market/src/negotiation/requestor.rs b/core/market/src/negotiation/requestor.rs index 2f1c2c76a9..c3666e6fc0 100644 --- a/core/market/src/negotiation/requestor.rs +++ b/core/market/src/negotiation/requestor.rs @@ -365,15 +365,27 @@ impl RequestorBroker { Some(agreement) => agreement, }; - validate_transition(&agreement, AgreementState::Pending)?; + { + // We won't be able to process `on_agreement_approved`, before we + // finish execution under this lock. This avoids errors related to + // Provider approving Agreement before we set proper state in database. + self.common + .agreement_lock + .get_lock(&agreement_id) + .await + .lock() + .await; - // TODO : possible race condition here ISSUE#430 - // 1. this state check should be also `db.update_state` - // 2. `db.update_state` must be invoked after successful propose_agreement - self.api.propose_agreement(&agreement).await?; - dao.confirm(agreement_id, &app_session_id) - .await - .map_err(|e| AgreementError::UpdateState(agreement_id.clone(), e))?; + validate_transition(&agreement, AgreementState::Pending)?; + + // TODO : possible race condition here ISSUE#430 + // 1. this state check should be also `db.update_state` + // 2. `dao.confirm` must be invoked after successful propose_agreement + self.api.propose_agreement(&agreement).await?; + dao.confirm(agreement_id, &app_session_id) + .await + .map_err(|e| AgreementError::UpdateState(agreement_id.clone(), e))?; + } counter!("market.agreements.requestor.confirmed", 1); log::info!( @@ -415,55 +427,63 @@ async fn agreement_approved( caller: NodeId, msg: AgreementApproved, ) -> Result<(), RemoteAgreementError> { - let agreement = broker - .db - .as_dao::() - .select(&msg.agreement_id, None, Utc::now().naive_utc()) - .await - .map_err(|_e| RemoteAgreementError::NotFound(msg.agreement_id.clone()))? - .ok_or(RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; + let agreement = { + // We aren't sure, if `confirm_agreement` execution is finished, + // so we must lock, to avoid attempt to change database state before. + broker + .agreement_lock + .get_lock(&msg.agreement_id) + .await + .lock() + .await; - if agreement.provider_id != caller { - // Don't reveal, that we know this Agreement id. - Err(RemoteAgreementError::NotFound(msg.agreement_id.clone()))? - } + let agreement = broker + .db + .as_dao::() + .select(&msg.agreement_id, None, Utc::now().naive_utc()) + .await + .map_err(|_e| RemoteAgreementError::NotFound(msg.agreement_id.clone()))? + .ok_or(RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; - // TODO: Validate agreement signature. - // Note: session must be None, because either we already set this value in ConfirmAgreement, - // or we purposely left it None. - broker - .db - .as_dao::() - .approve(&msg.agreement_id, &None) - .await - .map_err(|err| match err { - AgreementDaoError::InvalidTransition { from, .. } => { - match from { - // Expired Agreement could be InvalidState either, but we want to explicit - // say to provider, that Agreement has expired. - AgreementState::Expired => { - RemoteAgreementError::Expired(msg.agreement_id.clone()) + if agreement.provider_id != caller { + // Don't reveal, that we know this Agreement id. + Err(RemoteAgreementError::NotFound(msg.agreement_id.clone()))? + } + + // TODO: Validate agreement signature. + // Note: session must be None, because either we already set this value in ConfirmAgreement, + // or we purposely left it None. + broker + .db + .as_dao::() + .approve(&msg.agreement_id, &None) + .await + .map_err(|err| match err { + AgreementDaoError::InvalidTransition { from, .. } => { + match from { + // Expired Agreement could be InvalidState either, but we want to explicit + // say to provider, that Agreement has expired. + AgreementState::Expired => { + RemoteAgreementError::Expired(msg.agreement_id.clone()) + } + _ => RemoteAgreementError::InvalidState(msg.agreement_id.clone(), from), } - _ => RemoteAgreementError::InvalidState(msg.agreement_id.clone(), from), } - } - e => { - // Log our internal error, but don't reveal error message to Provider. - log::warn!( - "Approve Agreement [{}] internal error: {}", - &msg.agreement_id, - e - ); - RemoteAgreementError::InternalError(msg.agreement_id.clone()) - } - })?; + e => { + // Log our internal error, but don't reveal error message to Provider. + log::warn!( + "Approve Agreement [{}] internal error: {}", + &msg.agreement_id, + e + ); + RemoteAgreementError::InternalError(msg.agreement_id.clone()) + } + })?; + agreement + }; broker.notify_agreement(&agreement).await; - log::info!( - "Agreement [{}] approved by [{}].", - &msg.agreement_id, - &caller - ); + log::info!("Agreement [{}] approved by [{}].", &agreement.id, &caller); Ok(()) } diff --git a/core/market/src/utils.rs b/core/market/src/utils.rs index 8754563b71..7f6d88797a 100644 --- a/core/market/src/utils.rs +++ b/core/market/src/utils.rs @@ -1 +1,4 @@ +mod agreement_lock; pub mod display; + +pub use agreement_lock::AgreementLock; diff --git a/core/market/src/utils/agreement_lock.rs b/core/market/src/utils/agreement_lock.rs new file mode 100644 index 0000000000..34c11d07a2 --- /dev/null +++ b/core/market/src/utils/agreement_lock.rs @@ -0,0 +1,45 @@ +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{Mutex, RwLock}; + +use crate::db::model::AgreementId; + +#[derive(Clone)] +pub struct AgreementLock { + lock_map: Arc>>>>, +} + +impl AgreementLock { + pub fn new() -> AgreementLock { + AgreementLock { + lock_map: Arc::new(RwLock::new(HashMap::new())), + } + } + + pub async fn get_lock(&self, agreement_id: &AgreementId) -> Arc> { + // Note how important are '{}' around this statement. Otherwise lock isn't freed + // and we can't aqquire write lock + let potencial_lock = { + self.lock_map + .read() + .await + .get(agreement_id) + .map(|lock| lock.clone()) + }; + match potencial_lock { + Some(mutex) => mutex, + None => { + let mut lock_map = self.lock_map.write().await; + lock_map + .entry(agreement_id.clone()) + .or_insert(Arc::new(Mutex::new(()))) + .clone() + } + } + .clone() + } + + pub async fn clear_locks(&self, agreement_id: &AgreementId) { + self.lock_map.write().await.remove(agreement_id); + } +} From e68a33ff068b7baba46b74d8ffe8319e79ea0150 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 21 Jan 2021 15:22:42 +0100 Subject: [PATCH 02/11] Fix other potential race conditions, when changing Agreement state --- core/market/src/negotiation/common.rs | 56 +++++++++++++++++-------- core/market/src/negotiation/provider.rs | 27 +++++++----- 2 files changed, 56 insertions(+), 27 deletions(-) diff --git a/core/market/src/negotiation/common.rs b/core/market/src/negotiation/common.rs index b533f612c1..061a44099d 100644 --- a/core/market/src/negotiation/common.rs +++ b/core/market/src/negotiation/common.rs @@ -352,15 +352,26 @@ impl CommonBroker { Some(agreement) => agreement, }; - validate_transition(&agreement, AgreementState::Terminated)?; + { + // This must be under lock to avoid conflicts with `terminate_agreement`, + // that could have been called by other party at the same time. Termination + // consists of 2 operations: sending message to other party and updating state. + // Race conditions could appear in this situation. + self.agreement_lock + .get_lock(&agreement.id) + .await + .lock() + .await; - protocol_common::propagate_terminate_agreement(&agreement, reason.clone()).await?; + validate_transition(&agreement, AgreementState::Terminated)?; - let reason_string = CommonBroker::reason2string(&reason); + protocol_common::propagate_terminate_agreement(&agreement, reason.clone()).await?; - dao.terminate(&agreement.id, reason_string, agreement.id.owner()) - .await - .map_err(|e| AgreementError::UpdateState((&agreement.id).clone(), e))?; + let reason_string = CommonBroker::reason2string(&reason); + dao.terminate(&agreement.id, reason_string, agreement.id.owner()) + .await + .map_err(|e| AgreementError::UpdateState((&agreement.id).clone(), e))?; + } self.notify_agreement(&agreement).await; self.agreement_lock.clear_locks(&agreement.id).await; @@ -436,18 +447,29 @@ impl CommonBroker { Err(RemoteAgreementError::NotFound(agreement_id.clone()))? } - let reason_string = CommonBroker::reason2string(&msg.reason); + { + // This must be under lock to avoid conflicts with `terminate_agreement`, + // that could have been called by one of our Agents. Termination consists + // of 2 operations: sending message to other party and updating state. + // Race conditions could appear in this situation. + self.agreement_lock + .get_lock(&agreement.id) + .await + .lock() + .await; - dao.terminate(&agreement_id, reason_string, caller_role) - .await - .map_err(|e| { - log::warn!( - "Couldn't terminate agreement. id: {}, e: {}", - agreement_id, - e - ); - RemoteAgreementError::InternalError(agreement_id.clone()) - })?; + let reason_string = CommonBroker::reason2string(&msg.reason); + dao.terminate(&agreement_id, reason_string, caller_role) + .await + .map_err(|e| { + log::warn!( + "Couldn't terminate agreement. id: {}, e: {}", + agreement_id, + e + ); + RemoteAgreementError::InternalError(agreement_id.clone()) + })?; + } self.notify_agreement(&agreement).await; self.agreement_lock.clear_locks(&agreement_id).await; diff --git a/core/market/src/negotiation/provider.rs b/core/market/src/negotiation/provider.rs index de32f1bf0d..16f16009ae 100644 --- a/core/market/src/negotiation/provider.rs +++ b/core/market/src/negotiation/provider.rs @@ -219,16 +219,23 @@ impl ProviderBroker { Some(agreement) => agreement, }; - validate_transition(&agreement, AgreementState::Approved)?; - - // TODO: possible race condition here ISSUE#430 - // 1. this state check should be also `db.update_state` - // 2. `db.update_state` must be invoked after successful propose_agreement - // TODO: if dao.approve fails, Provider and Requestor have inconsistent state. - self.api.approve_agreement(&agreement, timeout).await?; - dao.approve(agreement_id, &app_session_id) - .await - .map_err(|e| AgreementError::UpdateState(agreement_id.clone(), e))?; + { + self.common + .agreement_lock + .get_lock(&agreement_id) + .await + .lock() + .await; + + validate_transition(&agreement, AgreementState::Approved)?; + + // `db.update_state` must be invoked after successful approve_agreement + // TODO: if dao.approve fails, Provider and Requestor have inconsistent state. + self.api.approve_agreement(&agreement, timeout).await?; + dao.approve(agreement_id, &app_session_id) + .await + .map_err(|e| AgreementError::UpdateState(agreement_id.clone(), e))?; + } self.common.notify_agreement(&agreement).await; From 2f13cfdd38bc57dea45bd6cb7fd7545577d942dd Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Tue, 26 Jan 2021 13:40:24 +0100 Subject: [PATCH 03/11] [mkt] agreement locks with wider scope --- Cargo.lock | 2 +- core/market/src/negotiation/common.rs | 39 +++++++++++++----------- core/market/src/negotiation/provider.rs | 25 ++++++++------- core/market/src/negotiation/requestor.rs | 27 ++++++++-------- core/market/src/utils/agreement_lock.rs | 2 +- 5 files changed, 50 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bc658e61a..1fe255eb2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,7 +41,7 @@ dependencies = [ "parking_lot 0.11.1", "pin-project 0.4.27", "smallvec 1.6.1", - "tokio", + "tokio 0.2.24", "tokio-util 0.3.1", ] diff --git a/core/market/src/negotiation/common.rs b/core/market/src/negotiation/common.rs index b2e9540cc2..dbef66c8ee 100644 --- a/core/market/src/negotiation/common.rs +++ b/core/market/src/negotiation/common.rs @@ -431,33 +431,34 @@ impl CommonBroker { ) -> Result<(), RemoteAgreementError> { let dao = self.db.as_dao::(); let agreement_id = msg.agreement_id.clone(); - let agreement = dao - .select(&agreement_id, None, Utc::now().naive_utc()) - .await - .map_err(|_e| RemoteAgreementError::NotFound(agreement_id.clone()))? - .ok_or(RemoteAgreementError::NotFound(agreement_id.clone()))?; - - let auth_id = match caller_role { - Owner::Provider => agreement.provider_id, - Owner::Requestor => agreement.requestor_id, - }; - - if auth_id != caller_id { - // Don't reveal, that we know this Agreement id. - Err(RemoteAgreementError::NotFound(agreement_id.clone()))? - } - { + let agreement = { // This must be under lock to avoid conflicts with `terminate_agreement`, // that could have been called by one of our Agents. Termination consists // of 2 operations: sending message to other party and updating state. // Race conditions could appear in this situation. self.agreement_lock - .get_lock(&agreement.id) + .get_lock(&agreement_id) .await .lock() .await; + let agreement = dao + .select(&agreement_id, None, Utc::now().naive_utc()) + .await + .map_err(|_e| RemoteAgreementError::NotFound(agreement_id.clone()))? + .ok_or(RemoteAgreementError::NotFound(agreement_id.clone()))?; + + let auth_id = match caller_role { + Owner::Provider => agreement.provider_id, + Owner::Requestor => agreement.requestor_id, + }; + + if auth_id != caller_id { + // Don't reveal, that we know this Agreement id. + Err(RemoteAgreementError::NotFound(agreement_id.clone()))? + } + let reason_string = CommonBroker::reason2string(&msg.reason); dao.terminate(&agreement_id, reason_string, caller_role) .await @@ -469,7 +470,9 @@ impl CommonBroker { ); RemoteAgreementError::InternalError(agreement_id.clone()) })?; - } + + agreement + }; self.notify_agreement(&agreement).await; self.agreement_lock.clear_locks(&agreement_id).await; diff --git a/core/market/src/negotiation/provider.rs b/core/market/src/negotiation/provider.rs index 16f16009ae..7e44674846 100644 --- a/core/market/src/negotiation/provider.rs +++ b/core/market/src/negotiation/provider.rs @@ -210,16 +210,7 @@ impl ProviderBroker { timeout: f32, ) -> Result { let dao = self.common.db.as_dao::(); - let agreement = match dao - .select(agreement_id, None, Utc::now().naive_utc()) - .await - .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? - { - None => Err(AgreementError::NotFound(agreement_id.to_string()))?, - Some(agreement) => agreement, - }; - - { + let agreement = { self.common .agreement_lock .get_lock(&agreement_id) @@ -227,6 +218,15 @@ impl ProviderBroker { .lock() .await; + let agreement = match dao + .select(agreement_id, None, Utc::now().naive_utc()) + .await + .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? + { + None => Err(AgreementError::NotFound(agreement_id.to_string()))?, + Some(agreement) => agreement, + }; + validate_transition(&agreement, AgreementState::Approved)?; // `db.update_state` must be invoked after successful approve_agreement @@ -235,7 +235,9 @@ impl ProviderBroker { dao.approve(agreement_id, &app_session_id) .await .map_err(|e| AgreementError::UpdateState(agreement_id.clone(), e))?; - } + + agreement + }; self.common.notify_agreement(&agreement).await; @@ -364,6 +366,7 @@ async fn agreement_received( Err(RemoteProposeAgreementError::InvalidId(id.clone()))? } + // This is creation of Agreement, so lock is not needed yet. let agreement = broker .db .as_dao::() diff --git a/core/market/src/negotiation/requestor.rs b/core/market/src/negotiation/requestor.rs index 673f559c26..5f6ec9506a 100644 --- a/core/market/src/negotiation/requestor.rs +++ b/core/market/src/negotiation/requestor.rs @@ -352,20 +352,6 @@ impl RequestorBroker { app_session_id: AppSessionId, ) -> Result<(), AgreementError> { let dao = self.common.db.as_dao::(); - - let agreement = match dao - .select( - agreement_id, - Some(id.identity.clone()), - Utc::now().naive_utc(), - ) - .await - .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? - { - None => return Err(AgreementError::NotFound(agreement_id.to_string())), - Some(agreement) => agreement, - }; - { // We won't be able to process `on_agreement_approved`, before we // finish execution under this lock. This avoids errors related to @@ -377,6 +363,19 @@ impl RequestorBroker { .lock() .await; + let agreement = match dao + .select( + agreement_id, + Some(id.identity.clone()), + Utc::now().naive_utc(), + ) + .await + .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? + { + None => return Err(AgreementError::NotFound(agreement_id.to_string())), + Some(agreement) => agreement, + }; + validate_transition(&agreement, AgreementState::Pending)?; // TODO : possible race condition here ISSUE#430 diff --git a/core/market/src/utils/agreement_lock.rs b/core/market/src/utils/agreement_lock.rs index 34c11d07a2..7f0c84e73c 100644 --- a/core/market/src/utils/agreement_lock.rs +++ b/core/market/src/utils/agreement_lock.rs @@ -18,7 +18,7 @@ impl AgreementLock { pub async fn get_lock(&self, agreement_id: &AgreementId) -> Arc> { // Note how important are '{}' around this statement. Otherwise lock isn't freed - // and we can't aqquire write lock + // and we can't acquire write lock let potencial_lock = { self.lock_map .read() From 0e5007d1682b22d1f8ab0804971a2d5a3f5492db Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 27 Jan 2021 10:52:57 +0100 Subject: [PATCH 04/11] Fix mock net after gsb started using flatbuffers --- core/market/src/testing/mock_net.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/core/market/src/testing/mock_net.rs b/core/market/src/testing/mock_net.rs index 406a4a9396..1cc78b1f60 100644 --- a/core/market/src/testing/mock_net.rs +++ b/core/market/src/testing/mock_net.rs @@ -8,13 +8,14 @@ use std::sync::{Arc, Mutex}; use ya_client::model::NodeId; use ya_core_model::net; use ya_core_model::net::{local as local_net, local::SendBroadcastMessage}; -use ya_service_bus::{typed as bus, untyped as local_bus, Error, RpcMessage}; +use ya_service_bus::{serialization, typed as bus, untyped as local_bus, Error, RpcMessage}; #[cfg(feature = "bcast-singleton")] use super::bcast::singleton::BCastService; use super::bcast::BCast; #[cfg(not(feature = "bcast-singleton"))] use super::bcast::BCastService; +use ya_core_model::net::local::SendBroadcastStub; #[derive(Clone)] pub struct MockNet { @@ -117,7 +118,7 @@ impl MockNetInner { let bcast = BCastService::default(); log::info!("initializing BCast on mock net"); - let bcast_service_id = as RpcMessage>::ID; + let bcast_service_id = as RpcMessage>::ID; let bcast1 = bcast.clone(); let _ = bus::bind(local_net::BUS_ID, move |subscribe: local_net::Subscribe| { @@ -130,7 +131,7 @@ impl MockNetInner { }); let addr = format!("{}/{}", local_net::BUS_ID, bcast_service_id); - let resp: Rc<[u8]> = serde_json::to_vec(&Ok::<(), ()>(())).unwrap().into(); + let resp: Rc<[u8]> = serialization::to_vec(&Ok::<(), ()>(())).unwrap().into(); let _ = local_bus::subscribe( &addr, move |caller: &str, _addr: &str, msg: &[u8]| { @@ -138,12 +139,12 @@ impl MockNetInner { let resp = resp.clone(); let bcast = bcast.clone(); - let msg_json: SendBroadcastMessage = - serde_json::from_slice(msg).unwrap(); + let stub: SendBroadcastStub = serialization::from_slice(msg).unwrap(); let caller = caller.to_string(); - let msg = serde_json::to_vec(&msg_json).unwrap(); - let topic = msg_json.topic().to_owned(); + let msg = msg.iter().copied().collect::>(); + + let topic = stub.topic; let endpoints = bcast.resolve(&caller, &topic); log::debug!("BCasting on {} to {:?} from {}", topic, endpoints, caller); @@ -155,7 +156,7 @@ impl MockNetInner { None => { log::debug!( "Not broadcasting on topic {} to {}. Node not found on list. \ - Probably networking was disabled for this Node.", + Probably networking was disabled for this Node.", topic, addr ); From d32192038c1ba6c410ab4ab1b2062ada3f414e5b Mon Sep 17 00:00:00 2001 From: Pawel Nowosielski Date: Wed, 27 Jan 2021 12:12:34 +0100 Subject: [PATCH 05/11] Doc-strings to driver's functions (#967) * docs: add docstrings to driver's functions --- core/payment-driver/base/src/bus.rs | 5 +++++ core/payment-driver/base/src/cron.rs | 3 +++ core/payment-driver/base/src/driver.rs | 23 ++++++++++++++++++++ core/payment/examples/README.md | 29 +++++++++++++++++++++----- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/core/payment-driver/base/src/bus.rs b/core/payment-driver/base/src/bus.rs index 7e31242281..5b02c9128a 100644 --- a/core/payment-driver/base/src/bus.rs +++ b/core/payment-driver/base/src/bus.rs @@ -114,6 +114,7 @@ pub async fn list_unlocked_identities() -> Result, GenericError> { Ok(unlocked_list) } +/// Notifies the Payment service that the account is ready to sending / receiving funds. pub async fn register_account( driver: &(dyn PaymentDriver), address: &str, @@ -136,6 +137,7 @@ pub async fn register_account( Ok(()) } +/// Delegates signing of the transaction's payload to the Identity service. pub async fn sign(node_id: NodeId, payload: Vec) -> Result, GenericError> { let signature = service(identity::BUS_ID) .send(identity::Sign { node_id, payload }) @@ -145,6 +147,9 @@ pub async fn sign(node_id: NodeId, payload: Vec) -> Result, GenericE Ok(signature) } +/// Notifies the Payment service that the scheduled payment is processed successfully. +/// It also links the PaymentDriver::schedule_payment `order_id` with the `confirmation` (e.g. transaction's hash). +/// This notification leads to `verify_payment` call on the provider side. pub async fn notify_payment( driver_name: &str, platform: &str, diff --git a/core/payment-driver/base/src/cron.rs b/core/payment-driver/base/src/cron.rs index ca1698d58b..80e7d0eca5 100644 --- a/core/payment-driver/base/src/cron.rs +++ b/core/payment-driver/base/src/cron.rs @@ -16,7 +16,10 @@ pub use async_trait::async_trait; #[async_trait(?Send)] pub trait PaymentDriverCron { + /// Confirms scheduled payments. async fn confirm_payments(&self); + + /// Processes scheduled payments. async fn process_payments(&self); } diff --git a/core/payment-driver/base/src/driver.rs b/core/payment-driver/base/src/driver.rs index 59c234007a..6bf2b15116 100644 --- a/core/payment-driver/base/src/driver.rs +++ b/core/payment-driver/base/src/driver.rs @@ -20,6 +20,9 @@ pub use ya_core_model::payment::local::Network; #[async_trait(?Send)] pub trait PaymentDriver { + /// Called by the Identity service to notify the driver that specified + /// account is _locked_ / _unlocked_. Identity service holds + /// accounts private keys and signs transactions. async fn account_event( &self, _db: DbExecutor, @@ -27,6 +30,7 @@ pub trait PaymentDriver { msg: IdentityEvent, ) -> Result<(), IdentityError>; + /// Gets the balance of the account. async fn get_account_balance( &self, db: DbExecutor, @@ -34,6 +38,7 @@ pub trait PaymentDriver { msg: GetAccountBalance, ) -> Result; + /// Deposits the funds into the driver's supported network. Called by CLI. async fn enter( &self, db: DbExecutor, @@ -41,6 +46,8 @@ pub trait PaymentDriver { msg: Enter, ) -> Result; + /// Exits the funds outside the driver's supported network (most likely L1). + /// Called by CLI. async fn exit(&self, db: DbExecutor, caller: String, msg: Exit) -> Result; @@ -48,8 +55,12 @@ pub trait PaymentDriver { fn get_name(&self) -> String; fn get_default_network(&self) -> String; fn get_networks(&self) -> HashMap; + + /// Tells whether account initialization is needed for receiving payments. fn recv_init_required(&self) -> bool; + /// NOTE: DEPRECATED. Drivers should return very big number (e.g. `1_000_000_000_000_000_000u64` or the whole token supply) + /// Gets the balance of the funds sent from the sender to the recipient. async fn get_transaction_balance( &self, db: DbExecutor, @@ -57,10 +68,15 @@ pub trait PaymentDriver { msg: GetTransactionBalance, ) -> Result; + /// Initializes the account to be used with the driver service. It should call + /// `bus::register_account` to notify Payment service about the driver readiness. Driver can handle multiple accounts. async fn init(&self, db: DbExecutor, caller: String, msg: Init) -> Result; + + /// Funds the account from faucet when run on testnet. Provides instructions how to fund on mainnet. async fn fund(&self, db: DbExecutor, caller: String, msg: Fund) -> Result; + /// Transfers the funds between specified accounts. Called by CLI. async fn transfer( &self, db: DbExecutor, @@ -68,6 +84,9 @@ pub trait PaymentDriver { msg: Transfer, ) -> Result; + /// Schedules the payment between specified accounts. Payments + /// are processed by the `cron` job. Payment tracking is done by + /// cron job, see: `PaymentDriverCron::confirm_payments`. async fn schedule_payment( &self, db: DbExecutor, @@ -75,6 +94,7 @@ pub trait PaymentDriver { msg: SchedulePayment, ) -> Result; + /// Verifies the payment transaction by transaction's confirmation (transaction's identifier). async fn verify_payment( &self, db: DbExecutor, @@ -82,6 +102,9 @@ pub trait PaymentDriver { msg: VerifyPayment, ) -> Result; + /// Validates that allocated funds are still sufficient to cover + /// the costs of the task (including the transaction fees, e.g. Ethereum's Gas). + /// Allocation is created when the requestor publishes the task on the market. async fn validate_allocation( &self, db: DbExecutor, diff --git a/core/payment/examples/README.md b/core/payment/examples/README.md index 6ca6c2a390..96e10db20a 100644 --- a/core/payment/examples/README.md +++ b/core/payment/examples/README.md @@ -8,7 +8,18 @@ cd core/payment cp ../../.env-template .env cargo run --example payment_api ``` -To use GNT instead of dummy driver us `cargo run --example payment_api -- --driver=gnt` instead. +To use GLM instead of dummy driver use: +- ERC-20 driver on rinkeby: + `cargo run --example payment_api -- --driver=erc20 --platform=erc20-rinkeby-tglm` +- ERC-20 driver on mainnet: + `cargo run --example payment_api -- --driver=erc20 --network=mainnet --platform=erc20-mainnet-glm` +- ZkSync driver on rinkeby: + `cargo run --example payment_api -- --driver=zksync --platform=zksync-rinkeby-tglm` +- ZkSync driver on mainnet: + `cargo run --example payment_api -- --driver=zksync --network=mainnet --platform=zksync-mainnet-glm` + + +**:warning: Remember! Each example expects a clean database so might need to remove payment.db and restart the API server.** ### Debit note flow @@ -16,11 +27,14 @@ To test the whole flow start the API server (see above) and run the debit_note_f example in another terminal: ```shell script cd core/payment -cargo run --example debit_note_flow +cargo run --example debit_note_flow -- --platform=dummy-glm ``` -(**NOTE:** The example expects a clean database so might need to remove `payment.db` +(**:warning: NOTE:** The example expects a clean database so might need to remove `payment.db` and restart the API server.) +
+ Example could be also run via REST API + ##### Issue a debit node: `POST` `http://127.0.0.1:7465/payment-api/v1/provider/debitNotes` @@ -63,17 +77,21 @@ Payload: ##### Listen for provider's debit note events: `GET` `http://127.0.0.1:7465/payment-api/v1/provider/debitNoteEvents?timeout=` +
### Invoice flow To test the whole flow start the API server (see above) and run the invoice_flow example in another terminal: ```shell script -cargo run --example invoice_flow +cargo run --example invoice_flow -- --platform=dummy-glm ``` -(**NOTE:** The example expects a clean database so might need to remove `payment.db` +(**:warning: NOTE:** The example expects a clean database so might need to remove `payment.db` and restart the API server.) +
+ Example could be also run via REST API + ##### Issue an invoice: `POST` `http://127.0.0.1:7465/payment-api/v1/provider/invoices` @@ -144,3 +162,4 @@ Don't forget to copy `allocationId` from the response! `GET` `http://127.0.0.1:7465/payment-api/v1/provider/payments` One can also listen for payments by adding `?timeout=` parameter. +
From 0d153ded2e1fc87868de5ad57e6c6cafe4b61993 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 28 Jan 2021 16:42:30 +0100 Subject: [PATCH 06/11] Hide time based PaymentChecker tests behind compilation flag --- agent/provider/Cargo.toml | 3 +++ agent/provider/src/payments/payment_checker.rs | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index 859bc6dbb3..1d24dd459a 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -5,6 +5,9 @@ version = "0.1.0" authors = ["Golem Factory "] edition = "2018" +[features] +time-dependent-tests = [] + [dependencies] ya-agreement-utils = "0.2" ya-client = { version = "0.4", features = ['cli'] } diff --git a/agent/provider/src/payments/payment_checker.rs b/agent/provider/src/payments/payment_checker.rs index f79c02fe0d..b6f51b6aca 100644 --- a/agent/provider/src/payments/payment_checker.rs +++ b/agent/provider/src/payments/payment_checker.rs @@ -273,6 +273,7 @@ mod test { checker } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_single_agreement() { let receiver = DeadlineReceiver::new(); @@ -326,6 +327,7 @@ mod test { assert_eq!(deadlined[0].id, 6.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_near_deadlines() { let receiver = DeadlineReceiver::new(); @@ -366,6 +368,7 @@ mod test { assert_eq!(deadlined[5].id, 5.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_insert_deadlines_between() { let receiver = DeadlineReceiver::new(); @@ -424,6 +427,7 @@ mod test { assert_eq!(deadlined[3].id, 3.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_multi_agreements() { let receiver = DeadlineReceiver::new(); @@ -455,6 +459,7 @@ mod test { assert_eq!(deadlined[4].id, 1.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_stop_tracking() { let receiver = DeadlineReceiver::new(); From b5dcb8e7780bdf01bc9b9f3587c9f1c60416e8d5 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 28 Jan 2021 16:42:30 +0100 Subject: [PATCH 07/11] Hide time based PaymentChecker tests behind compilation flag --- agent/provider/Cargo.toml | 3 +++ agent/provider/src/payments/payment_checker.rs | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index 859bc6dbb3..1d24dd459a 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -5,6 +5,9 @@ version = "0.1.0" authors = ["Golem Factory "] edition = "2018" +[features] +time-dependent-tests = [] + [dependencies] ya-agreement-utils = "0.2" ya-client = { version = "0.4", features = ['cli'] } diff --git a/agent/provider/src/payments/payment_checker.rs b/agent/provider/src/payments/payment_checker.rs index f79c02fe0d..b6f51b6aca 100644 --- a/agent/provider/src/payments/payment_checker.rs +++ b/agent/provider/src/payments/payment_checker.rs @@ -273,6 +273,7 @@ mod test { checker } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_single_agreement() { let receiver = DeadlineReceiver::new(); @@ -326,6 +327,7 @@ mod test { assert_eq!(deadlined[0].id, 6.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_near_deadlines() { let receiver = DeadlineReceiver::new(); @@ -366,6 +368,7 @@ mod test { assert_eq!(deadlined[5].id, 5.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_insert_deadlines_between() { let receiver = DeadlineReceiver::new(); @@ -424,6 +427,7 @@ mod test { assert_eq!(deadlined[3].id, 3.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_multi_agreements() { let receiver = DeadlineReceiver::new(); @@ -455,6 +459,7 @@ mod test { assert_eq!(deadlined[4].id, 1.to_string()); } + #[cfg_attr(not(feature = "time-dependent-tests"), ignore)] #[actix_rt::test] async fn test_deadline_checker_stop_tracking() { let receiver = DeadlineReceiver::new(); From 1a35c2617bad0b9a5dae52691b2856e313635089 Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Thu, 28 Jan 2021 17:23:16 +0100 Subject: [PATCH 08/11] enable `cargo test -p ya-provider --features ya-provider/time-dependent-tests` --- Cargo.lock | 7 ++++--- Cargo.toml | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3d2997fa7..6d19e16ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,7 +41,7 @@ dependencies = [ "parking_lot 0.11.1", "pin-project 0.4.27", "smallvec 1.6.1", - "tokio", + "tokio 0.2.24", "tokio-util 0.3.1", ] @@ -5338,6 +5338,7 @@ checksum = "2932dc07acd2066ff2e3921a4419606b220ba6cd03a9935123856cc534877056" dependencies = [ "rand 0.6.5", "secp256k1-sys 0.1.2", + "serde", ] [[package]] @@ -5348,7 +5349,6 @@ checksum = "c6179428c22c73ac0fbb7b5579a56353ce78ba29759b3b8575183336ea74cdfb" dependencies = [ "rand 0.6.5", "secp256k1-sys 0.3.0", - "serde", ] [[package]] @@ -7159,7 +7159,7 @@ dependencies = [ "hex", "openssl", "rand 0.7.3", - "secp256k1 0.19.0", + "secp256k1 0.17.2", "serde", "serde_json", "strum", @@ -7995,6 +7995,7 @@ dependencies = [ "ya-net", "ya-payment", "ya-persistence", + "ya-provider", "ya-sb-proto", "ya-sb-router", "ya-service-api", diff --git a/Cargo.toml b/Cargo.toml index cb97182325..71e80242bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ name = "yagna" path = "core/serv/src/main.rs" [dependencies] -gftp = "0.1" # just to enable gftp build for cargo-deb ya-activity = "0.2" ya-compile-time-utils = "0.1" ya-dummy-driver = { version = "0.1", optional = true } @@ -48,6 +47,9 @@ ya-utils-futures = "0.1" ya-utils-process = { version = "0.1", features = ["lock"] } ya-version = "0.1" +gftp = "0.1" # just to enable gftp build for cargo-deb +ya-provider = { version = "0.1", optional = true } # just to enable conditionally running some tests + actix-rt = "1.0" actix-service = "1.0" actix-web = "3.2" @@ -190,3 +192,4 @@ ya-utils-path = { path = "utils/path"} ya-utils-process = { path = "utils/process"} ya-diesel-utils = { path = "utils/diesel-utils"} ya-metrics = { path = "core/metrics" } +ya-provider = { path = "agent/provider"} From 9d145007cfb20587b5a4d0135b401bb6a98012e8 Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Thu, 28 Jan 2021 17:34:09 +0100 Subject: [PATCH 09/11] gftp made optional for yagna build --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 71e80242bf..c1e0183217 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ ya-utils-futures = "0.1" ya-utils-process = { version = "0.1", features = ["lock"] } ya-version = "0.1" -gftp = "0.1" # just to enable gftp build for cargo-deb +gftp = { version = "0.1", optional = true } # just to enable gftp build for cargo-deb ya-provider = { version = "0.1", optional = true } # just to enable conditionally running some tests actix-rt = "1.0" From 03125ba78fbf41853c6bf238aeddb19eb246535d Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Thu, 28 Jan 2021 21:46:28 +0100 Subject: [PATCH 10/11] fix secp ver in Cargo.lock... again :/ --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d19e16ebf..15f2342e50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5338,7 +5338,6 @@ checksum = "2932dc07acd2066ff2e3921a4419606b220ba6cd03a9935123856cc534877056" dependencies = [ "rand 0.6.5", "secp256k1-sys 0.1.2", - "serde", ] [[package]] @@ -5349,6 +5348,7 @@ checksum = "c6179428c22c73ac0fbb7b5579a56353ce78ba29759b3b8575183336ea74cdfb" dependencies = [ "rand 0.6.5", "secp256k1-sys 0.3.0", + "serde", ] [[package]] @@ -7159,7 +7159,7 @@ dependencies = [ "hex", "openssl", "rand 0.7.3", - "secp256k1 0.17.2", + "secp256k1 0.19.0", "serde", "serde_json", "strum", From f9ff3273e953b48c67615e22b813ef77106bda16 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 29 Jan 2021 12:50:38 +0100 Subject: [PATCH 11/11] Revert "Doc-strings to driver's functions (#967)" This reverts commit d32192038c1ba6c410ab4ab1b2062ada3f414e5b. --- core/payment-driver/base/src/bus.rs | 5 ----- core/payment-driver/base/src/cron.rs | 3 --- core/payment-driver/base/src/driver.rs | 23 -------------------- core/payment/examples/README.md | 29 +++++--------------------- 4 files changed, 5 insertions(+), 55 deletions(-) diff --git a/core/payment-driver/base/src/bus.rs b/core/payment-driver/base/src/bus.rs index 5b02c9128a..7e31242281 100644 --- a/core/payment-driver/base/src/bus.rs +++ b/core/payment-driver/base/src/bus.rs @@ -114,7 +114,6 @@ pub async fn list_unlocked_identities() -> Result, GenericError> { Ok(unlocked_list) } -/// Notifies the Payment service that the account is ready to sending / receiving funds. pub async fn register_account( driver: &(dyn PaymentDriver), address: &str, @@ -137,7 +136,6 @@ pub async fn register_account( Ok(()) } -/// Delegates signing of the transaction's payload to the Identity service. pub async fn sign(node_id: NodeId, payload: Vec) -> Result, GenericError> { let signature = service(identity::BUS_ID) .send(identity::Sign { node_id, payload }) @@ -147,9 +145,6 @@ pub async fn sign(node_id: NodeId, payload: Vec) -> Result, GenericE Ok(signature) } -/// Notifies the Payment service that the scheduled payment is processed successfully. -/// It also links the PaymentDriver::schedule_payment `order_id` with the `confirmation` (e.g. transaction's hash). -/// This notification leads to `verify_payment` call on the provider side. pub async fn notify_payment( driver_name: &str, platform: &str, diff --git a/core/payment-driver/base/src/cron.rs b/core/payment-driver/base/src/cron.rs index 80e7d0eca5..ca1698d58b 100644 --- a/core/payment-driver/base/src/cron.rs +++ b/core/payment-driver/base/src/cron.rs @@ -16,10 +16,7 @@ pub use async_trait::async_trait; #[async_trait(?Send)] pub trait PaymentDriverCron { - /// Confirms scheduled payments. async fn confirm_payments(&self); - - /// Processes scheduled payments. async fn process_payments(&self); } diff --git a/core/payment-driver/base/src/driver.rs b/core/payment-driver/base/src/driver.rs index 6bf2b15116..59c234007a 100644 --- a/core/payment-driver/base/src/driver.rs +++ b/core/payment-driver/base/src/driver.rs @@ -20,9 +20,6 @@ pub use ya_core_model::payment::local::Network; #[async_trait(?Send)] pub trait PaymentDriver { - /// Called by the Identity service to notify the driver that specified - /// account is _locked_ / _unlocked_. Identity service holds - /// accounts private keys and signs transactions. async fn account_event( &self, _db: DbExecutor, @@ -30,7 +27,6 @@ pub trait PaymentDriver { msg: IdentityEvent, ) -> Result<(), IdentityError>; - /// Gets the balance of the account. async fn get_account_balance( &self, db: DbExecutor, @@ -38,7 +34,6 @@ pub trait PaymentDriver { msg: GetAccountBalance, ) -> Result; - /// Deposits the funds into the driver's supported network. Called by CLI. async fn enter( &self, db: DbExecutor, @@ -46,8 +41,6 @@ pub trait PaymentDriver { msg: Enter, ) -> Result; - /// Exits the funds outside the driver's supported network (most likely L1). - /// Called by CLI. async fn exit(&self, db: DbExecutor, caller: String, msg: Exit) -> Result; @@ -55,12 +48,8 @@ pub trait PaymentDriver { fn get_name(&self) -> String; fn get_default_network(&self) -> String; fn get_networks(&self) -> HashMap; - - /// Tells whether account initialization is needed for receiving payments. fn recv_init_required(&self) -> bool; - /// NOTE: DEPRECATED. Drivers should return very big number (e.g. `1_000_000_000_000_000_000u64` or the whole token supply) - /// Gets the balance of the funds sent from the sender to the recipient. async fn get_transaction_balance( &self, db: DbExecutor, @@ -68,15 +57,10 @@ pub trait PaymentDriver { msg: GetTransactionBalance, ) -> Result; - /// Initializes the account to be used with the driver service. It should call - /// `bus::register_account` to notify Payment service about the driver readiness. Driver can handle multiple accounts. async fn init(&self, db: DbExecutor, caller: String, msg: Init) -> Result; - - /// Funds the account from faucet when run on testnet. Provides instructions how to fund on mainnet. async fn fund(&self, db: DbExecutor, caller: String, msg: Fund) -> Result; - /// Transfers the funds between specified accounts. Called by CLI. async fn transfer( &self, db: DbExecutor, @@ -84,9 +68,6 @@ pub trait PaymentDriver { msg: Transfer, ) -> Result; - /// Schedules the payment between specified accounts. Payments - /// are processed by the `cron` job. Payment tracking is done by - /// cron job, see: `PaymentDriverCron::confirm_payments`. async fn schedule_payment( &self, db: DbExecutor, @@ -94,7 +75,6 @@ pub trait PaymentDriver { msg: SchedulePayment, ) -> Result; - /// Verifies the payment transaction by transaction's confirmation (transaction's identifier). async fn verify_payment( &self, db: DbExecutor, @@ -102,9 +82,6 @@ pub trait PaymentDriver { msg: VerifyPayment, ) -> Result; - /// Validates that allocated funds are still sufficient to cover - /// the costs of the task (including the transaction fees, e.g. Ethereum's Gas). - /// Allocation is created when the requestor publishes the task on the market. async fn validate_allocation( &self, db: DbExecutor, diff --git a/core/payment/examples/README.md b/core/payment/examples/README.md index 96e10db20a..6ca6c2a390 100644 --- a/core/payment/examples/README.md +++ b/core/payment/examples/README.md @@ -8,18 +8,7 @@ cd core/payment cp ../../.env-template .env cargo run --example payment_api ``` -To use GLM instead of dummy driver use: -- ERC-20 driver on rinkeby: - `cargo run --example payment_api -- --driver=erc20 --platform=erc20-rinkeby-tglm` -- ERC-20 driver on mainnet: - `cargo run --example payment_api -- --driver=erc20 --network=mainnet --platform=erc20-mainnet-glm` -- ZkSync driver on rinkeby: - `cargo run --example payment_api -- --driver=zksync --platform=zksync-rinkeby-tglm` -- ZkSync driver on mainnet: - `cargo run --example payment_api -- --driver=zksync --network=mainnet --platform=zksync-mainnet-glm` - - -**:warning: Remember! Each example expects a clean database so might need to remove payment.db and restart the API server.** +To use GNT instead of dummy driver us `cargo run --example payment_api -- --driver=gnt` instead. ### Debit note flow @@ -27,14 +16,11 @@ To test the whole flow start the API server (see above) and run the debit_note_f example in another terminal: ```shell script cd core/payment -cargo run --example debit_note_flow -- --platform=dummy-glm +cargo run --example debit_note_flow ``` -(**:warning: NOTE:** The example expects a clean database so might need to remove `payment.db` +(**NOTE:** The example expects a clean database so might need to remove `payment.db` and restart the API server.) -
- Example could be also run via REST API - ##### Issue a debit node: `POST` `http://127.0.0.1:7465/payment-api/v1/provider/debitNotes` @@ -77,21 +63,17 @@ Payload: ##### Listen for provider's debit note events: `GET` `http://127.0.0.1:7465/payment-api/v1/provider/debitNoteEvents?timeout=` -
### Invoice flow To test the whole flow start the API server (see above) and run the invoice_flow example in another terminal: ```shell script -cargo run --example invoice_flow -- --platform=dummy-glm +cargo run --example invoice_flow ``` -(**:warning: NOTE:** The example expects a clean database so might need to remove `payment.db` +(**NOTE:** The example expects a clean database so might need to remove `payment.db` and restart the API server.) -
- Example could be also run via REST API - ##### Issue an invoice: `POST` `http://127.0.0.1:7465/payment-api/v1/provider/invoices` @@ -162,4 +144,3 @@ Don't forget to copy `allocationId` from the response! `GET` `http://127.0.0.1:7465/payment-api/v1/provider/payments` One can also listen for payments by adding `?timeout=` parameter. -