From 8760067f05e3bbf9e2a54f855c6b90e8790b92e7 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 22 Dec 2023 18:17:39 +0100 Subject: [PATCH 01/52] Add progress watch channel to TransferContext --- exe-unit/components/transfer/src/lib.rs | 20 +++++++++++++++++-- exe-unit/components/transfer/src/progress.rs | 10 ++++++++++ exe-unit/components/transfer/src/transfer.rs | 17 ++++++++++++++-- .../components/transfer/tests/test_deploy.rs | 2 ++ .../transfer/tests/test_transfer_abort.rs | 1 + .../transfer/tests/test_transfer_resume.rs | 1 + .../transfer/tests/test_transfer_service.rs | 3 +++ exe-unit/src/lib.rs | 6 +++++- 8 files changed, 55 insertions(+), 5 deletions(-) diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 34e055680a..ffc45e133c 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -15,6 +15,7 @@ mod traverse; use std::cell::RefCell; use std::pin::Pin; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use bytes::Bytes; @@ -38,6 +39,8 @@ pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; use crate::hash::with_hash_stream; +use crate::progress::progress_report_channel; +use crate::transfer::Progress; use ya_client_model::activity::TransferArgs; /// Transfers data from `stream` to a `TransferSink` @@ -73,7 +76,7 @@ where log::debug!("Transferring from offset: {}", ctx.state.offset()); let stream = with_hash_stream(src.source(&src_url.url, ctx), src_url, dst_url, ctx)?; - let sink = dst.destination(&dst_url.url, ctx); + let sink = progress_report_channel(dst.destination(&dst_url.url, ctx), ctx); transfer(stream, sink).await?; Ok::<_, Error>(()) @@ -296,6 +299,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, + pub report: Arc>>>, } impl TransferContext { @@ -304,7 +308,19 @@ impl TransferContext { let state = TransferState::default(); state.set_offset(offset); - Self { args, state } + Self { + args, + state, + report: Arc::new(std::sync::Mutex::new(None)), + } + } + + pub fn register_reporter(&self, report: tokio::sync::watch::Sender) { + *self.report.lock().unwrap() = Some(report); + } + + pub fn take_reporter(&self) -> Option> { + self.report.lock().unwrap().take() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 71edaa7419..5c0ba866f5 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -1,4 +1,5 @@ use crate::error::Error; +use crate::transfer::Progress; use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; use futures::{SinkExt, StreamExt, TryFutureExt}; @@ -48,6 +49,15 @@ where type Sink = TransferSink; +pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { + let report = ctx.take_reporter(); + wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { + if let Some(report) = &report { + report.send(Progress { progress, size }).ok(); + } + }) +} + /// Wraps a sink to report progress. /// The `report` function is called with the current offset and the total size. /// The total size is 0 if the size is unknown. (For example, when the source is a directory.) diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index bc24672a44..4f33688a02 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -35,12 +35,22 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Message)] +#[derive(Clone, Debug)] +pub struct Progress { + pub progress: u64, + pub size: Option, +} + +#[derive(Debug, Message)] #[rtype(result = "Result<()>")] pub struct TransferResource { pub from: String, pub to: String, pub args: TransferArgs, + + /// Channel for watching for transfer progress. `None` means that there + /// will be no progress updates. + pub progress: Option>, } #[derive(Message)] @@ -53,10 +63,13 @@ impl AddVolumes { } } -#[derive(Clone, Debug, Message)] +#[derive(Debug, Message)] #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, + /// Channel for watching for deploy progress. `None` means that there + /// will be no progress updates. + pub progress: Option>, } #[derive(Clone, Debug, Message)] diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 53dab763a3..96ed3ed77e 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -67,6 +67,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re let result = addr .send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await?; log::info!("Deployment stopped"); @@ -76,6 +77,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re log::info!("Re-deploying the same image"); addr.send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 69e864f2d7..63f167d3a2 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,6 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), + progress: None, }) .await?; diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 2986cc4471..9b561d2ef2 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -113,6 +113,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, + progress: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index a96e6f7a00..e92d5809b3 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -31,6 +31,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, + progress: None, }) .await??; @@ -98,6 +99,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment with hash verification"); addr.send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await??; log::warn!("Deployment complete"); @@ -106,6 +108,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment from cache"); addr.send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await??; log::warn!("Deployment from cache complete"); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index d92d97eebc..56a70f27ac 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -308,13 +308,17 @@ impl RuntimeRef { from: from.clone(), to: to.clone(), args: args.clone(), + progress: None, }; transfer_service.send(msg).await??; } ExeScriptCommand::Deploy { net, hosts } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. let task_package = transfer_service - .send(DeployImage { task_package: None }) + .send(DeployImage { + task_package: None, + progress: None, + }) .await??; runtime .send(UpdateDeployment { From 732872817558ac11c4f6b81cc39c345bc2b251bc Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 14:16:14 +0100 Subject: [PATCH 02/52] Register progress reporter for transfers and deploy --- exe-unit/components/transfer/src/lib.rs | 4 ++-- exe-unit/components/transfer/src/transfer.rs | 10 +++++++--- exe-unit/components/transfer/tests/test_deploy.rs | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index ffc45e133c..b30befac1c 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -315,8 +315,8 @@ impl TransferContext { } } - pub fn register_reporter(&self, report: tokio::sync::watch::Sender) { - *self.report.lock().unwrap() = Some(report); + pub fn register_reporter(&self, report: Option>) { + *self.report.lock().unwrap() = report; } pub fn take_reporter(&self) -> Option> { diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 4f33688a02..4c27964b94 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -35,7 +35,7 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Progress { pub progress: u64, pub size: Option, @@ -173,6 +173,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, + progress: Option>, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -194,6 +195,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, + progress: Option>, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -206,6 +208,7 @@ impl TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.deploy_retry.clone()); + ctx.register_reporter(progress); // Using partially downloaded image from previous executions could speed up deploy // process, but it comes with the cost: If image under URL changed, Requestor will get @@ -283,10 +286,10 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); #[cfg(not(feature = "sgx"))] - return self.deploy_no_sgx(src_url, src_name, path); + return self.deploy_no_sgx(src_url, src_name, path, deploy.progress); #[cfg(feature = "sgx")] - return self.deploy_sgx(src_url, src_name, path); + return self.deploy_sgx(src_url, src_name, path, deploy.progress); } } @@ -301,6 +304,7 @@ impl Handler for TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); + ctx.register_reporter(msg.progress); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 96ed3ed77e..25e3d145c2 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -17,7 +17,7 @@ use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, Transf #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("deploy-restart")?; let temp_dir = dir.path(); From 6c6f3f1556b987b4589ef40bd5e447dd7eece3c1 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 14:53:03 +0100 Subject: [PATCH 03/52] Fix http tranfer provider to always ask server for file size --- exe-unit/components/transfer/src/http.rs | 11 +-- .../components/transfer/tests/test_deploy.rs | 88 +++++++++++++++++-- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/exe-unit/components/transfer/src/http.rs b/exe-unit/components/transfer/src/http.rs index 97bb1b2026..03e1acfd7e 100644 --- a/exe-unit/components/transfer/src/http.rs +++ b/exe-unit/components/transfer/src/http.rs @@ -119,10 +119,6 @@ impl TransferProvider for HttpTransferProvider { url: &Url, ctx: &TransferContext, ) -> LocalBoxFuture<'a, Result<(), Error>> { - if ctx.state.offset() == 0 { - return futures::future::ok(()).boxed_local(); - } - let url = url.clone(); let state = ctx.state.clone(); @@ -137,8 +133,13 @@ impl TransferProvider for HttpTransferProvider { .get(header::CONTENT_LENGTH) .and_then(|v| v.to_str().ok().and_then(|s| u64::from_str(s).ok())); + match &size { + None => log::info!("File size unknown. Http source server didn't respond with CONTENT_LENGTH header."), + Some(size) => log::info!("Http source size reported by server: {size} B"), + }; + state.set_size(size); - if !ranges { + if state.offset() != 0 && !ranges { log::warn!("Transfer resuming is not supported by the server"); state.set_offset(0); } diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 25e3d145c2..9fe2ad70db 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -9,7 +9,9 @@ use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; +use ya_transfer::transfer::{ + AbortTransfers, DeployImage, Progress, TransferService, TransferServiceContext, +}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -31,10 +33,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re std::fs::create_dir_all(dir)?; } - let chunk_size = 4096_usize; - let chunk_count = 1024 * 10; - - let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); @@ -83,3 +82,82 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re Ok(()) } + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("deploy-restart")?; + let temp_dir = dir.path(); + + log::debug!("Creating directories in: {}", temp_dir.display()); + let work_dir = temp_dir.join("work_dir"); + let cache_dir = temp_dir.join("cache_dir"); + let sub_dir = temp_dir.join("sub_dir"); + + for dir in vec![work_dir.clone(), cache_dir.clone(), sub_dir.clone()] { + std::fs::create_dir_all(dir)?; + } + + let chunk_size = 4096_usize; + let chunk_count = 1024 * 1; + let file_size = (chunk_size * chunk_count) as u64; + let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + + log::debug!("Starting HTTP servers"); + let path = temp_dir.to_path_buf(); + start_http(ctx, path) + .await + .expect("unable to start http servers"); + + let task_package = Some(format!( + "hash://sha3:{}:http://127.0.0.1:8001/rnd", + hex::encode(hash) + )); + + log::debug!("Starting TransferService"); + let exe_ctx = TransferServiceContext { + work_dir: work_dir.clone(), + cache_dir, + ..TransferServiceContext::default() + }; + let addr = TransferService::new(exe_ctx).start(); + + log::info!("[>>] Deployment with hash verification"); + let (tx, mut rx) = tokio::sync::watch::channel::(Progress::default()); + + tokio::task::spawn_local(async move { + let _result = addr + .send(DeployImage { + task_package: task_package.clone(), + progress: Some(tx), + }) + .await??; + log::info!("Deployment stopped"); + anyhow::Ok(()) + }); + + let mut last_progress = 0u64; + loop { + if rx.changed().await.is_err() { + break; + } + + let progress = rx.borrow_and_update(); + + assert_eq!(progress.size.unwrap(), file_size); + assert!(progress.progress >= last_progress); + + last_progress = progress.progress; + + log::info!( + "Progress: {}/{}", + progress.progress, + progress.size.unwrap_or(0) + ); + } + + Ok(()) +} From ffc31ffe9629e192278560cda51bf8adcd423f51 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 15:10:41 +0100 Subject: [PATCH 04/52] Prettify progress loop in test --- exe-unit/components/transfer/tests/test_deploy.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 9fe2ad70db..323bb318a6 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -87,7 +87,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("deploy-restart")?; let temp_dir = dir.path(); @@ -140,11 +140,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< }); let mut last_progress = 0u64; - loop { - if rx.changed().await.is_err() { - break; - } - + while let Ok(_) = rx.changed().await { let progress = rx.borrow_and_update(); assert_eq!(progress.size.unwrap(), file_size); From 1428ffe17b5ebf679863fb84576a6e8a7fe97c9a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 16:38:26 +0100 Subject: [PATCH 05/52] Progress struct is serializable --- exe-unit/components/transfer/src/transfer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 4c27964b94..615dd17bf1 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,6 +5,7 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; +use serde::{Deserialize, Serialize}; use url::Url; use crate::cache::{Cache, CachePath}; @@ -35,7 +36,7 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct Progress { pub progress: u64, pub size: Option, From 319c7bf3b6d9806fd9e567242c56779819e0513a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 3 Jan 2024 17:54:22 +0100 Subject: [PATCH 06/52] Move CommandProgress to ya-client-model --- Cargo.lock | 478 ++++++++++-------- Cargo.toml | 8 +- agent/provider/Cargo.toml | 4 +- core/activity/Cargo.toml | 2 +- core/gsb-api/Cargo.toml | 2 +- core/identity/Cargo.toml | 2 +- core/market/Cargo.toml | 2 +- core/model/Cargo.toml | 2 +- core/net/Cargo.toml | 2 +- core/payment-driver/base/Cargo.toml | 2 +- core/payment-driver/erc20/Cargo.toml | 2 +- core/payment-driver/erc20next/Cargo.toml | 2 +- core/payment/Cargo.toml | 4 +- core/persistence/Cargo.toml | 2 +- core/serv-api/web/Cargo.toml | 2 +- core/sgx/Cargo.toml | 2 +- core/version/Cargo.toml | 2 +- core/vpn/Cargo.toml | 4 +- exe-unit/Cargo.toml | 2 +- exe-unit/components/transfer/Cargo.toml | 2 +- exe-unit/components/transfer/src/lib.rs | 16 +- exe-unit/components/transfer/src/progress.rs | 6 +- exe-unit/components/transfer/src/transfer.rs | 14 +- .../components/transfer/tests/test_deploy.rs | 4 +- exe-unit/src/state.rs | 1 + golem_cli/Cargo.toml | 2 +- .../test-framework/framework-basic/Cargo.toml | 2 +- utils/agreement-utils/Cargo.toml | 2 +- utils/manifest-utils/Cargo.toml | 2 +- 29 files changed, 303 insertions(+), 274 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b84e7e8269..fd96f033fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ "actix_derive", "bitflags 2.4.1", "bytes 1.5.0", - "crossbeam-channel 0.5.9", + "crossbeam-channel 0.5.10", "futures-core", "futures-sink", "futures-task", @@ -84,16 +84,16 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" +checksum = "129d4c88e98860e1758c5de288d1632b07970a16d59bdf7b8d66053d582bb71f" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-tls", "actix-utils", - "ahash 0.8.6", + "ahash 0.8.7", "base64 0.21.5", "bitflags 2.4.1", "brotli", @@ -154,14 +154,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" +checksum = "e43428f3bf11dee6d166b00ec2df4e3aa8cc1606aaa0b7433c146852e2f4e03b" dependencies = [ "actix-codec", "actix-http", @@ -284,7 +284,7 @@ dependencies = [ "actix-tls", "actix-utils", "actix-web-codegen", - "ahash 0.8.6", + "ahash 0.8.7", "bytes 1.5.0", "bytestring", "cfg-if 1.0.0", @@ -336,7 +336,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if 1.0.0", "getrandom 0.2.11", @@ -515,9 +515,9 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anyhow" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "appdirs" @@ -597,13 +597,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -652,9 +652,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "awc" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa3c705a9c7917ac0f41c0757a0a747b43bbc29b0b364b081bd7c5fc67fb223" +checksum = "b625cad34428b3b82d0bd548b26a1cd0a3d70b6109e9b4e3355d8f1802a8b1c6" dependencies = [ "actix-codec", "actix-http", @@ -941,7 +941,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "syn_derive", ] @@ -968,9 +968,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", "regex-automata", @@ -1333,12 +1333,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" +checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.17", + "crossbeam-utils 0.8.18", ] [[package]] @@ -1380,12 +1380,12 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.17", + "crossbeam-utils 0.8.18", ] [[package]] @@ -1401,9 +1401,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if 1.0.0", ] @@ -1517,12 +1517,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e95fbd621905b854affdc67943b043a0fbb6ed7385fd5a25650d19a8a6cfdf" +checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ "nix 0.27.1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1612,7 +1612,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -1914,7 +1914,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -1927,7 +1927,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -1986,7 +1986,7 @@ dependencies = [ "erc20_payment_lib_common", "erc20_rpc_pool", "fastrand", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "hex", "humantime 2.1.0", @@ -2025,7 +2025,7 @@ dependencies = [ "chrono", "dotenv", "fastrand", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "hex", "humantime 2.1.0", @@ -2070,7 +2070,7 @@ dependencies = [ "env_logger 0.10.1", "erc20_payment_lib_common", "fastrand", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "hex", "humantime 2.1.0", @@ -2508,9 +2508,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2523,9 +2523,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2533,15 +2533,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2561,32 +2561,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -2596,9 +2596,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures 0.1.31", "futures-channel", @@ -2672,7 +2672,7 @@ dependencies = [ "digest 0.8.1", "dotenv", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "log", "rand 0.8.5", "serde", @@ -2711,7 +2711,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -2752,7 +2752,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.5.0", ] [[package]] @@ -2769,7 +2769,7 @@ dependencies = [ "directories", "dotenv", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "libc", "log", @@ -2857,7 +2857,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", "allocator-api2", ] @@ -3125,9 +3125,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3350,13 +3350,13 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi 0.3.3", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3425,7 +3425,7 @@ version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" dependencies = [ - "futures 0.3.29", + "futures 0.3.30", "futures-executor", "futures-util", "log", @@ -3646,9 +3646,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -3703,8 +3703,8 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ - "ahash 0.8.6", - "metrics-macros 0.7.0", + "ahash 0.8.7", + "metrics-macros 0.7.1", "portable-atomic", ] @@ -3729,13 +3729,13 @@ dependencies = [ [[package]] name = "metrics-macros" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4228,9 +4228,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -4265,9 +4265,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.61" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if 1.0.0", @@ -4286,7 +4286,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4306,9 +4306,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -4571,7 +4571,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4679,12 +4679,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4779,9 +4779,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] @@ -4882,7 +4882,7 @@ dependencies = [ "prost 0.12.3", "prost-types 0.12.3", "regex", - "syn 2.0.42", + "syn 2.0.46", "tempfile", "which", ] @@ -4923,7 +4923,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -5014,9 +5014,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -5479,7 +5479,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.20", + "semver 1.0.21", ] [[package]] @@ -5640,11 +5640,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5838,9 +5838,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" dependencies = [ "serde", ] @@ -5862,31 +5862,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -5902,9 +5902,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" dependencies = [ "itoa", "ryu", @@ -5935,13 +5935,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" +checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -5985,9 +5985,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.28" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9269cfafc7e0257ee4a42f3f68a307f458c63d9e7c8ba4b58c5d15f1b7d7e8d3" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap 2.1.0", "itoa", @@ -6023,7 +6023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92761393ee4dc3ff8f4af487bd58f4307c9329bbedea02cac0089ad9c411e153" dependencies = [ "dashmap", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.12.1", @@ -6037,7 +6037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "538c30747ae860d6fb88330addbbd3e0ddbe46d662d032855596d8a8ca260611" dependencies = [ "dashmap", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.12.1", @@ -6051,7 +6051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ "dashmap", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.12.1", @@ -6109,7 +6109,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6376,7 +6376,7 @@ checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.1", "bytes 1.5.0", - "futures 0.3.29", + "futures 0.3.30", "httparse", "log", "rand 0.8.5", @@ -6438,13 +6438,13 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", "atoi", "byteorder", "bytes 1.5.0", "chrono", "crc", - "crossbeam-queue 0.3.9", + "crossbeam-queue 0.3.10", "dotenvy", "either", "event-listener", @@ -6692,6 +6692,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89a286a7e3b5720b9a477b23253bc50debac207c8d21505f8e70b36792f11b5" + [[package]] name = "strum" version = "0.24.1" @@ -6707,6 +6713,18 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +[[package]] +name = "strum_macros" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e61bb0be289045cb80bfce000512e32d09f8337e54c186725da381377ad1f8d5" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "strum_macros" version = "0.24.3" @@ -6730,7 +6748,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6752,9 +6770,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.42" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ "proc-macro2", "quote", @@ -6770,7 +6788,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6845,15 +6863,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall 0.4.1", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -6928,7 +6946,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6952,7 +6970,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "test-case-core", ] @@ -6963,7 +6981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "055831a02a4f5aa28fede67f2902014273eb8c21b958ac5ebbd59b71ef30dbc3" dependencies = [ "async-trait", - "futures 0.3.29", + "futures 0.3.30", "test-context-macros", ] @@ -6988,22 +7006,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -7103,7 +7121,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -7265,7 +7283,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -7652,7 +7670,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "wasm-bindgen-shared", ] @@ -7686,7 +7704,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7719,7 +7737,7 @@ dependencies = [ "derive_more", "ethabi", "ethereum-types 0.14.1", - "futures 0.3.29", + "futures 0.3.30", "futures-timer", "headers", "hex", @@ -7848,11 +7866,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -8055,9 +8073,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.30" +version = "0.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" +checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" dependencies = [ "memchr", ] @@ -8099,9 +8117,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" dependencies = [ "libc", "linux-raw-sys", @@ -8129,7 +8147,7 @@ dependencies = [ "diesel", "diesel_migrations", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "libsqlite3-sys", @@ -8145,7 +8163,7 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-net", "ya-persistence", @@ -8165,24 +8183,24 @@ dependencies = [ "regex", "serde", "serde_json", - "serde_yaml 0.9.28", + "serde_yaml 0.9.30", "shlex 1.2.0", "tempdir", "thiserror", - "ya-client-model", + "ya-client-model 0.6.0", ] [[package]] name = "ya-client" -version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=afdc1daf1f17a5128d911f8fff9d55c4f213e9af#afdc1daf1f17a5128d911f8fff9d55c4f213e9af" +version = "0.8.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" dependencies = [ "actix-codec", "awc", "bytes 1.5.0", "chrono", "envy", - "futures 0.3.29", + "futures 0.3.30", "heck 0.4.1", "hex", "log", @@ -8193,13 +8211,30 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.6.0", ] [[package]] name = "ya-client-model" version = "0.5.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=afdc1daf1f17a5128d911f8fff9d55c4f213e9af#afdc1daf1f17a5128d911f8fff9d55c4f213e9af" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a4a2f46bc12b1c747495c7911a844e7dc2c879826993426d568b9b594c26a1" +dependencies = [ + "bigdecimal 0.2.2", + "chrono", + "derive_more", + "rand 0.8.5", + "serde", + "serde_json", + "strum 0.19.5", + "strum_macros 0.19.4", + "thiserror", +] + +[[package]] +name = "ya-client-model" +version = "0.6.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -8242,7 +8277,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.6.0", "ya-service-bus", ] @@ -8264,13 +8299,13 @@ dependencies = [ "anyhow", "bigdecimal 0.2.2", "chrono", - "futures 0.3.29", + "futures 0.3.30", "log", "maplit", "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.5.0", "ya-core-model", "ya-payment-driver", "ya-persistence", @@ -8295,7 +8330,7 @@ dependencies = [ "ethabi", "ethereum-tx-sign", "ethereum-types 0.14.1", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "log", @@ -8316,7 +8351,7 @@ dependencies = [ "url", "uuid 0.8.2", "web3", - "ya-client-model", + "ya-client-model 0.6.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8341,7 +8376,7 @@ dependencies = [ "ethereum-tx-sign", "ethereum-types 0.14.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "log", @@ -8360,7 +8395,7 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model", + "ya-client-model 0.6.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8384,7 +8419,7 @@ dependencies = [ "dotenv", "env_logger 0.7.1", "flexi_logger 0.22.6", - "futures 0.3.29", + "futures 0.3.30", "graphene-sgx", "hex", "ipnet", @@ -8414,7 +8449,7 @@ dependencies = [ "url", "winapi 0.3.9", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.6.0", "ya-compile-time-utils", "ya-core-model", "ya-manifest-utils", @@ -8465,7 +8500,7 @@ dependencies = [ "bytes 1.5.0", "crossterm 0.26.1", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "gftp", "globset", "h2", @@ -8487,7 +8522,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-service-bus", "ya-utils-futures", @@ -8520,7 +8555,7 @@ dependencies = [ "ctor", "env_logger 0.10.1", "flexbuffers", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "serde", @@ -8530,7 +8565,7 @@ dependencies = [ "thiserror", "tokio", "uuid 1.6.1", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-persistence", "ya-service-api", @@ -8557,7 +8592,7 @@ dependencies = [ "dotenv", "env_logger 0.7.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "log", "promptly", "r2d2", @@ -8570,7 +8605,7 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-persistence", "ya-sb-router", @@ -8608,10 +8643,10 @@ dependencies = [ "pretty_assertions", "regex", "schemars", - "semver 1.0.20", + "semver 1.0.21", "serde", "serde_json", - "serde_yaml 0.9.28", + "serde_yaml 0.9.30", "serial_test 2.0.0", "shlex 1.2.0", "snailquote", @@ -8623,7 +8658,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.6.0", "ya-manifest-test-utils", "ya-utils-path", ] @@ -8647,7 +8682,7 @@ dependencies = [ "diesel_migrations", "digest 0.8.1", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "humantime 2.1.0", "lazy_static", "libsqlite3-sys", @@ -8709,7 +8744,7 @@ dependencies = [ "anyhow", "awc", "bigdecimal 0.2.2", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "metrics 0.16.0", @@ -8736,7 +8771,7 @@ dependencies = [ "chrono", "env_logger 0.7.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "humantime 2.1.0", "lazy_static", "log", @@ -8753,7 +8788,7 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", @@ -8803,7 +8838,7 @@ dependencies = [ "env_logger 0.7.1", "erc20_payment_lib", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "humantime 2.1.0", "lazy_static", @@ -8823,7 +8858,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -8851,7 +8886,7 @@ dependencies = [ "diesel_migrations", "ethereum-types 0.14.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "log", "num-bigint 0.3.3", @@ -8861,7 +8896,7 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-persistence", "ya-service-bus", @@ -8885,7 +8920,7 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-service-api", "ya-service-api-interfaces", @@ -8910,7 +8945,7 @@ dependencies = [ "dialoguer", "directories", "dotenv", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "golem-certificate", "hex", @@ -8950,7 +8985,7 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-compile-time-utils", "ya-core-model", "ya-file-logging", @@ -8975,7 +9010,7 @@ dependencies = [ "chrono", "derive_more", "educe", - "futures 0.3.29", + "futures 0.3.30", "hex", "humantime 2.1.0", "log", @@ -9004,7 +9039,7 @@ dependencies = [ "derive_more", "digest 0.9.0", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "log", @@ -9019,7 +9054,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.5.0", "ya-relay-proto", "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", ] @@ -9032,7 +9067,7 @@ dependencies = [ "anyhow", "bytes 1.5.0", "derive_more", - "futures 0.3.29", + "futures 0.3.30", "hex", "prost 0.12.3", "prost-build 0.12.3", @@ -9050,7 +9085,7 @@ version = "0.5.0" source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" dependencies = [ "derive_more", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "managed", @@ -9070,7 +9105,7 @@ version = "0.5.0" source = "git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5#c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" dependencies = [ "derive_more", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "managed", @@ -9110,7 +9145,7 @@ dependencies = [ "anyhow", "bytes 1.5.0", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "log", "prost 0.10.4", "prost-build 0.10.4", @@ -9149,7 +9184,7 @@ dependencies = [ "anyhow", "bitflags 1.3.2", "chrono", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.11.2", @@ -9172,7 +9207,7 @@ source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=44c942c72f6 dependencies = [ "actix", "bitflags 1.3.2", - "futures 0.3.29", + "futures 0.3.30", "pin-project 0.4.30", ] @@ -9213,7 +9248,7 @@ version = "0.2.0" dependencies = [ "actix-web", "anyhow", - "futures 0.3.29", + "futures 0.3.30", ] [[package]] @@ -9228,7 +9263,7 @@ dependencies = [ "anyhow", "awc", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "log", "serde", "structopt", @@ -9251,7 +9286,7 @@ source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=44c942c72f6 dependencies = [ "actix", "flexbuffers", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "miniz_oxide 0.5.4", @@ -9274,7 +9309,7 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-service-bus", ] @@ -9335,7 +9370,7 @@ dependencies = [ "bytes 1.5.0", "crossterm 0.26.1", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "gftp", "globset", "h2", @@ -9360,7 +9395,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-exe-unit", "ya-framework-basic", @@ -9379,7 +9414,7 @@ dependencies = [ "actix-rt", "anyhow", "chrono", - "futures 0.3.29", + "futures 0.3.30", "log", "tokio", ] @@ -9393,14 +9428,14 @@ dependencies = [ "prettytable-rs", "serde", "serde_json", - "serde_yaml 0.9.28", + "serde_yaml 0.9.30", ] [[package]] name = "ya-utils-futures" version = "0.3.0" dependencies = [ - "futures 0.3.29", + "futures 0.3.30", "tokio", ] @@ -9409,7 +9444,7 @@ name = "ya-utils-networking" version = "0.2.0" dependencies = [ "anyhow", - "futures 0.3.29", + "futures 0.3.30", "ipnet", "lazy_static", "log", @@ -9437,7 +9472,7 @@ dependencies = [ "anyhow", "derive_more", "fs2", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "libc", "nix 0.22.3", @@ -9482,7 +9517,7 @@ dependencies = [ "anyhow", "bytes 1.5.0", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "hex", "ipnet", "lazy_static", @@ -9499,7 +9534,7 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", @@ -9523,7 +9558,7 @@ dependencies = [ "chrono", "directories", "dotenv", - "futures 0.3.29", + "futures 0.3.30", "gftp", "lazy_static", "libsqlite3-sys", @@ -9541,7 +9576,7 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-compile-time-utils", "ya-core-model", "ya-dummy-driver", @@ -9592,22 +9627,22 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -9627,7 +9662,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -9654,26 +9689,25 @@ checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "byteorder", "crc32fast", - "crossbeam-utils 0.8.17", + "crossbeam-utils 0.8.18", "flate2", ] [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 9f7b923261..fa48057add 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,8 +60,8 @@ ya-utils-networking = "0.2" ya-fd-metrics = { path = "utils/fd-metrics" } ya-version = "0.2" ya-vpn = "0.2" -ya-client = "0.7" -ya-client-model = "0.5" +ya-client = "0.8" +ya-client-model = "0.6" gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "afdc1daf1f17a5128d911f8fff9d55c4f213e9af" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "afdc1daf1f17a5128d911f8fff9d55c4f213e9af" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } ## RELAY and networking stack diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index df7415bac8..78c15149a2 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -15,8 +15,8 @@ path = "src/main.rs" [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client = { version = "0.7", features = ['cli'] } -ya-client-model = "0.5" +ya-client = { version = "0.8", features = ['cli'] } +ya-client-model = "0.6" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ['activity', 'payment'] } ya-file-logging = "0.1" diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index 9f1633af05..c64fc428ea 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] ya-core-model = { version = "0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.5", features = ["sgx"] } +ya-client-model = { version = "0.6", features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/gsb-api/Cargo.toml b/core/gsb-api/Cargo.toml index 8d1db889b9..df32d0c860 100644 --- a/core/gsb-api/Cargo.toml +++ b/core/gsb-api/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index c78db1c600..afd523c56b 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = { version = "0.5", features = ["with-diesel"] } +ya-client-model = { version = "0.6", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = ["identity", "appkey"] } ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index 5eb13843d6..9b7cd586dc 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -11,7 +11,7 @@ bcast-singleton = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client = "0.7" +ya-client = "0.8" ya-core-model = { version = "^0.9", features = ["market", "net"] } ya-diesel-utils = { version = "0.1" } ya-market-resolver = "0.2" diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index 8d6f99ad92..0b026e02d2 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -35,7 +35,7 @@ sgx = ['graphene-sgx'] version = [] [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-service-bus = "0.6.1" bigdecimal = { version = "0.2", features = ["serde"], optional = true } diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index 0d51a93bab..e4f3824923 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -15,7 +15,7 @@ packet-trace-enable = [ ] [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = ["net", "identity"] } ya-relay-client = { workspace = true } diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 25fcf53c2f..ed80dd6da0 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -29,7 +29,7 @@ thiserror = "1.0" tokio = { version = "1", features = ["macros"] } ## yagna dependencies -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = [ "driver", "identity", diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index 55a54b5749..089e1de6a8 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -43,7 +43,7 @@ url = { workspace = true } trust-dns-resolver = { workspace = true } ## yagna dependencies ya-payment-driver = "0.3" -ya-client-model = "0.5" +ya-client-model = "0.6" ya-service-api-interfaces = "0.2" ya-utils-futures = "0.3" ya-utils-networking = "0.2" diff --git a/core/payment-driver/erc20next/Cargo.toml b/core/payment-driver/erc20next/Cargo.toml index 0a7f7da89b..b4aa656d15 100644 --- a/core/payment-driver/erc20next/Cargo.toml +++ b/core/payment-driver/erc20next/Cargo.toml @@ -43,7 +43,7 @@ rust_decimal = "1" ## yagna dependencies ya-payment-driver = "0.3" -ya-client-model = "0.5" +ya-client-model = "0.6" ya-service-api-interfaces = "0.2" ya-utils-futures = "0.3" ya-utils-networking = "0.2" diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index b021dbbd61..766ade21ef 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -9,7 +9,7 @@ default = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = { version = "0.5", features = ["with-diesel"] } +ya-client-model = { version = "0.6", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = [ "activity", "driver", @@ -59,7 +59,7 @@ humantime = "2.0.1" erc20_payment_lib = { workspace = true } [dev-dependencies] -ya-client = "0.7" +ya-client = "0.8" ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-erc20next-driver = "0.4" diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index 21cf823a7e..6013ec618e 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -9,7 +9,7 @@ default = [] service = ["ya-service-api", "ya-service-api-interfaces", "ya-utils-process", "structopt"] [dependencies] -ya-client-model = { version = "0.5", features = [ "with-diesel" ] } +ya-client-model = { version = "0.6", features = [ "with-diesel" ] } ya-core-model = { version = "0.9" } ya-service-api = { version = "0.1", optional = true } ya-service-api-interfaces = { version = "0.2", optional = true } diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index 1148c348e2..2a1424e2ca 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.7" +ya-client = "0.8" ya-core-model = { version = "^0.9", features = ["appkey"] } ya-service-api = "0.1" ya-service-bus = "0.6.1" diff --git a/core/sgx/Cargo.toml b/core/sgx/Cargo.toml index 79f64cf9cd..4fc30157ea 100644 --- a/core/sgx/Cargo.toml +++ b/core/sgx/Cargo.toml @@ -7,6 +7,6 @@ edition = "2018" [dependencies] anyhow = "1.0" graphene-sgx = { version = "0.3.3", features = [ "ias" ] } -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = ["sgx"] } ya-service-bus = "0.6.1" diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 7e4653fc4f..b177d4cc31 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.7" +ya-client = "0.8" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ["version"] } ya-persistence = "0.3" diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index e53f46653e..5cf5676f9f 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] ya-core-model = { version = "^0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.5", features = ["sgx"] } +ya-client-model = { version = "0.6", features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" @@ -42,7 +42,7 @@ packet-trace-enable = ["ya-packet-trace/enable"] default = [] [dev-dependencies] -ya-client = "0.7" +ya-client = "0.8" actix-rt = "2.7" sha3 = "0.8.2" structopt = "0.3" diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index e5ff92faf1..af1f4357ae 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -39,7 +39,7 @@ winapi = { version = "0.3.8", features = ["jobapi2", "processthreadsapi"] } [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client-model = "0.5" +ya-client-model = "0.6" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index d735528f07..e980e0ccc4 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9" } ya-service-bus = "0.6.1" ya-utils-path = { version ="0.1", path = "../../../utils/path" } diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index b30befac1c..1ab485cc81 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -26,22 +26,20 @@ use futures::prelude::*; use futures::task::{Context, Poll}; use url::Url; -use crate::error::Error; - pub use crate::archive::{archive, extract, ArchiveFormat}; pub use crate::container::ContainerTransferProvider; +use crate::error::Error; pub use crate::file::{DirTransferProvider, FileTransferProvider}; pub use crate::gftp::GftpTransferProvider; +use crate::hash::with_hash_stream; pub use crate::http::HttpTransferProvider; pub use crate::location::{TransferUrl, UrlExt}; +use crate::progress::progress_report_channel; pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_progress_reporting}; pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; -use crate::hash::with_hash_stream; -use crate::progress::progress_report_channel; -use crate::transfer::Progress; -use ya_client_model::activity::TransferArgs; +use ya_client_model::activity::{CommandProgress, TransferArgs}; /// Transfers data from `stream` to a `TransferSink` pub async fn transfer(stream: S, mut sink: TransferSink) -> Result<(), Error> @@ -299,7 +297,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, - pub report: Arc>>>, + pub report: Arc>>>, } impl TransferContext { @@ -315,11 +313,11 @@ impl TransferContext { } } - pub fn register_reporter(&self, report: Option>) { + pub fn register_reporter(&self, report: Option>) { *self.report.lock().unwrap() = report; } - pub fn take_reporter(&self) -> Option> { + pub fn take_reporter(&self) -> Option> { self.report.lock().unwrap().take() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 5c0ba866f5..2ba4c9bd9c 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -1,10 +1,12 @@ use crate::error::Error; -use crate::transfer::Progress; use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; + use futures::{SinkExt, StreamExt, TryFutureExt}; use tokio::task::spawn_local; +use ya_client_model::activity::CommandProgress; + type Stream = TransferStream; /// Wraps a stream to report progress. @@ -53,7 +55,7 @@ pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { let report = ctx.take_reporter(); wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { if let Some(report) = &report { - report.send(Progress { progress, size }).ok(); + report.send(CommandProgress { progress, size }).ok(); } }) } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 615dd17bf1..7629125a15 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,7 +5,6 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; -use serde::{Deserialize, Serialize}; use url::Url; use crate::cache::{Cache, CachePath}; @@ -16,6 +15,7 @@ use crate::{ HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; use ya_runtime_api::deploy::ContainerVolume; use ya_utils_futures::abort::Abort; @@ -36,12 +36,6 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct Progress { - pub progress: u64, - pub size: Option, -} - #[derive(Debug, Message)] #[rtype(result = "Result<()>")] pub struct TransferResource { @@ -51,7 +45,7 @@ pub struct TransferResource { /// Channel for watching for transfer progress. `None` means that there /// will be no progress updates. - pub progress: Option>, + pub progress: Option>, } #[derive(Message)] @@ -70,7 +64,7 @@ pub struct DeployImage { pub task_package: Option, /// Channel for watching for deploy progress. `None` means that there /// will be no progress updates. - pub progress: Option>, + pub progress: Option>, } #[derive(Clone, Debug, Message)] @@ -196,7 +190,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, - progress: Option>, + progress: Option>, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 323bb318a6..ab7c0e8bcd 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -10,7 +10,7 @@ use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; use ya_transfer::transfer::{ - AbortTransfers, DeployImage, Progress, TransferService, TransferServiceContext, + AbortTransfers, CommandProgress, DeployImage, TransferService, TransferServiceContext, }; /// When re-deploying image, `TransferService` should uses partially downloaded image. @@ -126,7 +126,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let addr = TransferService::new(exe_ctx).start(); log::info!("[>>] Deployment with hash verification"); - let (tx, mut rx) = tokio::sync::watch::channel::(Progress::default()); + let (tx, mut rx) = tokio::sync::watch::channel(CommandProgress::default()); tokio::task::spawn_local(async move { let _result = addr diff --git a/exe-unit/src/state.rs b/exe-unit/src/state.rs index b4e957fc66..7b98b8b72b 100644 --- a/exe-unit/src/state.rs +++ b/exe-unit/src/state.rs @@ -164,6 +164,7 @@ impl Batch { ..event }) } + RuntimeEventKind::Progress(_) => Some(event), }; if let Some(evt) = stream_event { diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index 1fde23a1b2..63c9525804 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = { version = "0.7", features = ['cli'] } +ya-client = { version = "0.8", features = ['cli'] } ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features=["payment", "version"] } ya-provider = "0.3" diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index e3039d162b..09eceeca36 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9" } ya-service-bus = "0.6.1" ya-utils-path = { version ="0.1", path = "../../../utils/path" } diff --git a/utils/agreement-utils/Cargo.toml b/utils/agreement-utils/Cargo.toml index 43e0aadabc..bf677d70c9 100644 --- a/utils/agreement-utils/Cargo.toml +++ b/utils/agreement-utils/Cargo.toml @@ -13,7 +13,7 @@ keywords=["golem", "yagna"] default = [] [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" chrono = "0.4" regex = "1.5.4" diff --git a/utils/manifest-utils/Cargo.toml b/utils/manifest-utils/Cargo.toml index c5d584cd5e..4b08b3c87e 100644 --- a/utils/manifest-utils/Cargo.toml +++ b/utils/manifest-utils/Cargo.toml @@ -22,7 +22,7 @@ schema = ["schemars"] [dependencies] ya-agreement-utils = { workspace = true } ya-utils-path = "0.1" -ya-client-model = "0.5" +ya-client-model = "0.6" golem-certificate = "0.1.1" regex = "1.5" From 7008049a9933a58b2a78e9943693ce0f435bb25b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 4 Jan 2024 13:17:12 +0100 Subject: [PATCH 07/52] Disable spamming ExeUnit logs during transfer --- exe-unit/src/logger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe-unit/src/logger.rs b/exe-unit/src/logger.rs index d9cd42565d..9d36cd3473 100644 --- a/exe-unit/src/logger.rs +++ b/exe-unit/src/logger.rs @@ -6,7 +6,7 @@ const ENV_VAR_LOG_DIR: &str = "EXE_UNIT_LOG_DIR"; const ENV_VAR_FILE_LOG_LEVEL: &str = "EXE_UNIT_FILE_LOG_LEVEL"; const DEFAULT_LOG_LEVEL: &str = "info"; -const DEFAULT_FILE_LOG_LEVEL: &str = "debug"; +const DEFAULT_FILE_LOG_LEVEL: &str = "debug,tokio_util=off,h2=info"; const DEFAULT_LOG_DIR: &str = "logs"; const DEFAULT_LOG_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.3f%z"; From 1ffaf5a7e078a261f47981dbe6b47cc3f7704d5c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 5 Jan 2024 18:52:57 +0100 Subject: [PATCH 08/52] Use new progress model from ya-client --- Cargo.lock | 97 +++++++++++--------- Cargo.toml | 4 +- exe-unit/components/transfer/Cargo.toml | 1 + exe-unit/components/transfer/src/lib.rs | 34 ++++++- exe-unit/components/transfer/src/progress.rs | 6 +- exe-unit/components/transfer/src/transfer.rs | 53 +++++++++-- exe-unit/src/lib.rs | 18 +++- exe-unit/src/message.rs | 15 ++- 8 files changed, 163 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd96f033fa..91f0543b9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,7 +154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -336,7 +336,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -603,7 +603,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -941,7 +941,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "syn_derive", ] @@ -1270,9 +1270,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -1612,7 +1612,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1816,6 +1816,15 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "duration-string" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcc1d9ae294a15ed05aeae8e11ee5f2b3fe971c077d45a42fb20825fba6ee13" +dependencies = [ + "serde", +] + [[package]] name = "dyn-clone" version = "1.0.16" @@ -1914,7 +1923,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1927,7 +1936,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -2573,7 +2582,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -2711,7 +2720,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -3735,7 +3744,7 @@ checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4286,7 +4295,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4505,9 +4514,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" dependencies = [ "memchr", "thiserror", @@ -4571,7 +4580,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4684,7 +4693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4779,9 +4788,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.74" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" dependencies = [ "unicode-ident", ] @@ -4882,7 +4891,7 @@ dependencies = [ "prost 0.12.3", "prost-types 0.12.3", "regex", - "syn 2.0.46", + "syn 2.0.48", "tempfile", "which", ] @@ -4923,7 +4932,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -5886,7 +5895,7 @@ checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -5902,9 +5911,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -5941,7 +5950,7 @@ checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6109,7 +6118,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6748,7 +6757,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6770,9 +6779,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.46" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -6788,7 +6797,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6946,7 +6955,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6970,7 +6979,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "test-case-core", ] @@ -7021,7 +7030,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -7121,7 +7130,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -7283,7 +7292,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -7670,7 +7679,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -7704,7 +7713,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8073,9 +8082,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.31" +version = "0.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" +checksum = "8434aeec7b290e8da5c3f0d628cb0eac6cabcb31d14bb74f779a08109a5914d6" dependencies = [ "memchr", ] @@ -8193,7 +8202,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" +source = "git+https://github.com/golemfactory/ya-client.git?rev=b313c3f79a5160c22bab3deea903f1e8ae009164#b313c3f79a5160c22bab3deea903f1e8ae009164" dependencies = [ "actix-codec", "awc", @@ -8234,12 +8243,13 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" +source = "git+https://github.com/golemfactory/ya-client.git?rev=b313c3f79a5160c22bab3deea903f1e8ae009164#b313c3f79a5160c22bab3deea903f1e8ae009164" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", + "duration-string", "hex", "openssl", "rand 0.8.5", @@ -9391,6 +9401,7 @@ dependencies = [ "test-context", "thiserror", "tokio", + "tokio-stream", "tokio-tar", "tokio-util", "url", @@ -9642,7 +9653,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -9662,7 +9673,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fa48057add..c4fd1b3ec7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "b313c3f79a5160c22bab3deea903f1e8ae009164" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "b313c3f79a5160c22bab3deea903f1e8ae009164" } ## RELAY and networking stack diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index e980e0ccc4..9e9de1e517 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -42,6 +42,7 @@ tokio-util = { version = "0.7", features = ["io"] } url = "2.1.1" walkdir = "2.3.1" async-trait = "0.1.74" +tokio-stream = "0.1.14" [target.'cfg(target_family = "unix")'.dependencies] awc = { version = "3", features = ["openssl"] } diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 1ab485cc81..ec227bff97 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -66,6 +66,8 @@ where let src = src.as_ref(); let dst = dst.as_ref(); + ctx.report(CommandProgress::TransferStarted); + loop { let fut = async { dst.prepare_destination(&dst_url.url, ctx).await?; @@ -81,10 +83,15 @@ where }; match fut.await { - Ok(val) => return Ok(val), + Ok(val) => { + ctx.report(CommandProgress::TransferFinished); + return Ok(val); + } Err(err) => match ctx.state.delay(&err) { Some(delay) => { log::warn!("Retrying in {}s because: {}", delay.as_secs_f32(), err); + + ctx.report_retry(err.to_string(), delay); tokio::time::sleep(delay).await; } None => return Err(err), @@ -297,7 +304,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, - pub report: Arc>>>, + pub report: Arc>>>, } impl TransferContext { @@ -313,12 +320,29 @@ impl TransferContext { } } - pub fn register_reporter(&self, report: Option>) { + pub fn register_reporter( + &self, + report: Option>, + ) { *self.report.lock().unwrap() = report; } - pub fn take_reporter(&self) -> Option> { - self.report.lock().unwrap().take() + pub fn reporter(&self) -> Option> { + self.report.lock().unwrap().clone() + } + + pub fn report_retry(&self, message: String, delay: Duration) { + self.report(CommandProgress::Retry(message, delay)) + } + + pub fn report_fetching_cached(&self) { + self.report(CommandProgress::FetchingFromCache) + } + + pub fn report(&self, status: CommandProgress) { + if let Some(reporter) = self.reporter() { + reporter.send(status).ok(); + } } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 2ba4c9bd9c..e4133a3087 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -52,10 +52,12 @@ where type Sink = TransferSink; pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { - let report = ctx.take_reporter(); + let report = ctx.reporter(); wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { if let Some(report) = &report { - report.send(CommandProgress { progress, size }).ok(); + report + .send(CommandProgress::TransferProgress(progress, size)) + .ok(); } }) } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 7629125a15..8b5ccb718b 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,6 +5,7 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; +use futures::{Sink, StreamExt, TryStreamExt}; use url::Url; use crate::cache::{Cache, CachePath}; @@ -15,6 +16,7 @@ use crate::{ HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +use ya_client_model::activity::exe_script_command::ProgressArgs; pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; use ya_runtime_api::deploy::ContainerVolume; @@ -36,16 +38,49 @@ macro_rules! actor_try { }; } -#[derive(Debug, Message)] +#[derive(Debug, Clone)] +pub struct ProgressConfig { + /// Channel for watching for transfer progress. + pub progress: tokio::sync::broadcast::Sender, + pub progress_args: ProgressArgs, +} + +#[derive(Debug, Message, Default)] #[rtype(result = "Result<()>")] pub struct TransferResource { pub from: String, pub to: String, pub args: TransferArgs, + /// Progress reporting configuration. `None` means that there will be no progress updates. + pub progress_args: Option, +} - /// Channel for watching for transfer progress. `None` means that there - /// will be no progress updates. - pub progress: Option>, +impl TransferResource { + pub fn forward_progress( + &mut self, + args: &ProgressArgs, + sender: impl Sink + 'static, + ) { + let rx = match &self.progress_args { + None => { + let (tx, rx) = tokio::sync::broadcast::channel(50); + self.progress_args = Some(ProgressConfig { + progress: tx, + progress_args: args.clone(), + }); + rx + } + Some(args) => args.progress.subscribe(), + }; + + tokio::task::spawn_local(async move { + tokio_stream::wrappers::BroadcastStream::new(rx) + .map_err(|e| Error::Other(e.to_string())) + .forward(sender) + .await + .ok() + }); + } } #[derive(Message)] @@ -62,9 +97,10 @@ impl AddVolumes { #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, + pub progress_args: ProgressArgs, /// Channel for watching for deploy progress. `None` means that there /// will be no progress updates. - pub progress: Option>, + pub progress: Option>, } #[derive(Clone, Debug, Message)] @@ -168,7 +204,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, - progress: Option>, + progress: Option>, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -190,7 +226,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, - progress: Option>, + progress: Option>, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -222,6 +258,7 @@ impl TransferService { let fut = async move { if path.exists() { log::info!("Deploying cached image: {:?}", path); + ctx.report_fetching_cached(); return Ok(Some(path)); } @@ -299,7 +336,7 @@ impl Handler for TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); - ctx.register_reporter(msg.progress); + ctx.register_reporter(msg.progress_args.map(|args| args.progress)); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 56a70f27ac..e3983044ca 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -303,20 +303,30 @@ impl RuntimeRef { transfer_service: &Addr, ) -> Result<()> { match &runtime_cmd.command { - ExeScriptCommand::Transfer { from, to, args } => { - let msg = TransferResource { + ExeScriptCommand::Transfer { + from, + to, + args, + progress, + } => { + let mut msg = TransferResource { from: from.clone(), to: to.clone(), args: args.clone(), - progress: None, + progress_args: None, }; + + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } transfer_service.send(msg).await??; } - ExeScriptCommand::Deploy { net, hosts } => { + ExeScriptCommand::Deploy { net, hosts, .. } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. let task_package = transfer_service .send(DeployImage { task_package: None, + progress_args: Default::default(), progress: None, }) .await??; diff --git a/exe-unit/src/message.rs b/exe-unit/src/message.rs index 27be13d80b..8c19e9c3ee 100644 --- a/exe-unit/src/message.rs +++ b/exe-unit/src/message.rs @@ -5,6 +5,7 @@ use crate::Result; use actix::prelude::*; use futures::channel::mpsc; +use futures::{Sink, SinkExt}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -12,7 +13,9 @@ use std::path::PathBuf; use ya_client_model::activity; use ya_client_model::activity::activity_state::{State, StatePair}; use ya_client_model::activity::exe_script_command::Network; -use ya_client_model::activity::{CommandOutput, ExeScriptCommand, ExeScriptCommandResult}; +use ya_client_model::activity::{ + CommandOutput, CommandProgress, ExeScriptCommand, ExeScriptCommandResult, +}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "Result>")] @@ -110,6 +113,16 @@ impl ExecuteCommand { }, ) } + + pub fn progress_sink(&self) -> impl Sink { + let CommandContext { batch_id, idx, .. } = self.clone().split().1; + self.tx.clone().with(move |item| { + let batch_id = batch_id.clone(); + futures::future::ok(RuntimeEvent::Process(activity::RuntimeEvent::progress( + batch_id, idx, item, + ))) + }) + } } #[derive(Clone, Debug)] From 1675c21565084bd9fc88a70aa5aeb5d48327bcc6 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 5 Jan 2024 19:01:13 +0100 Subject: [PATCH 09/52] Fix ya-client dependencies versions problems (still not compiling) --- Cargo.lock | 38 +++++++++++++++++++------------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91f0543b9e..09af9f2c59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9011,8 +9011,8 @@ dependencies = [ [[package]] name = "ya-relay-client" -version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +version = "0.6.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "anyhow", "async-trait", @@ -9036,13 +9036,13 @@ dependencies = [ "ya-packet-trace 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ya-relay-core", "ya-relay-proto", - "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-stack 0.5.1", ] [[package]] name = "ya-relay-core" -version = "0.4.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +version = "0.4.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "anyhow", "chrono", @@ -9064,15 +9064,15 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.5.0", + "ya-client-model 0.6.0", "ya-relay-proto", - "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-stack 0.5.1", ] [[package]] name = "ya-relay-proto" -version = "0.4.2" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +version = "0.4.3" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "anyhow", "bytes 1.5.0", @@ -9086,13 +9086,13 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a)", ] [[package]] name = "ya-relay-stack" version = "0.5.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5#c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" dependencies = [ "derive_more", "futures 0.3.30", @@ -9102,17 +9102,17 @@ dependencies = [ "num-derive", "num-traits", "rand 0.8.5", - "smoltcp", "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ya-smoltcp", ] [[package]] name = "ya-relay-stack" -version = "0.5.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5#c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" +version = "0.5.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "derive_more", "futures 0.3.30", @@ -9122,11 +9122,11 @@ dependencies = [ "num-derive", "num-traits", "rand 0.8.5", + "smoltcp", "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ya-smoltcp", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a)", ] [[package]] @@ -9142,7 +9142,7 @@ dependencies = [ [[package]] name = "ya-relay-util" version = "0.1.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "bytes 1.5.0", "derive_more", @@ -9463,7 +9463,7 @@ dependencies = [ "thiserror", "trust-dns-resolver 0.22.0", "url", - "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5)", + "ya-relay-stack 0.5.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c4fd1b3ec7..b15202f204 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -241,7 +241,7 @@ trust-dns-resolver = "0.22" derive_more = "0.99.11" ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "785c9c4271f514d47fe18018efb9438007ae611a" } +ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "8d544dd766e14479519437d0bf4f2879e6a6de5a" } gftp = { version = "0.4.0", path = "core/gftp" } ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } From 49f23ae0cc118a646f3f3454f35cad5ff1f91e47 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 8 Jan 2024 18:34:32 +0100 Subject: [PATCH 10/52] Use new ya-client Progress structs; Move towards streaming implementation --- Cargo.lock | 14 ++- Cargo.toml | 4 +- exe-unit/components/transfer/src/lib.rs | 35 ++----- exe-unit/components/transfer/src/progress.rs | 93 ++++++++++++++++++- exe-unit/components/transfer/src/transfer.rs | 13 +-- .../components/transfer/tests/test_deploy.rs | 26 +++--- .../transfer/tests/test_transfer_abort.rs | 2 +- .../transfer/tests/test_transfer_resume.rs | 2 +- .../transfer/tests/test_transfer_service.rs | 8 +- exe-unit/src/manifest.rs | 5 + 10 files changed, 138 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6377cc411..c1d6749a48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1805,6 +1805,15 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "duration-string" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcc1d9ae294a15ed05aeae8e11ee5f2b3fe971c077d45a42fb20825fba6ee13" +dependencies = [ + "serde", +] + [[package]] name = "dyn-clone" version = "1.0.16" @@ -8169,7 +8178,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db#a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" +source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" dependencies = [ "actix-codec", "awc", @@ -8193,12 +8202,13 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db#a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" +source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", + "duration-string", "hex", "openssl", "rand 0.8.5", diff --git a/Cargo.toml b/Cargo.toml index 21e586e210..e9ffd15bd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } ## RELAY and networking stack diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index ec227bff97..4f05b41396 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -15,7 +15,6 @@ mod traverse; use std::cell::RefCell; use std::pin::Pin; use std::rc::Rc; -use std::sync::Arc; use std::time::Duration; use bytes::Bytes; @@ -34,7 +33,7 @@ pub use crate::gftp::GftpTransferProvider; use crate::hash::with_hash_stream; pub use crate::http::HttpTransferProvider; pub use crate::location::{TransferUrl, UrlExt}; -use crate::progress::progress_report_channel; +use crate::progress::{progress_report_channel, ProgressReporter}; pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_progress_reporting}; pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; @@ -66,8 +65,6 @@ where let src = src.as_ref(); let dst = dst.as_ref(); - ctx.report(CommandProgress::TransferStarted); - loop { let fut = async { dst.prepare_destination(&dst_url.url, ctx).await?; @@ -84,14 +81,14 @@ where match fut.await { Ok(val) => { - ctx.report(CommandProgress::TransferFinished); return Ok(val); } Err(err) => match ctx.state.delay(&err) { Some(delay) => { - log::warn!("Retrying in {}s because: {}", delay.as_secs_f32(), err); + let msg = format!("Retry in {}s because of error: {err}", delay.as_secs_f32()); + log::warn!("{}", msg); - ctx.report_retry(err.to_string(), delay); + ctx.progress.report_message(msg); tokio::time::sleep(delay).await; } None => return Err(err), @@ -304,7 +301,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, - pub report: Arc>>>, + pub progress: ProgressReporter, } impl TransferContext { @@ -316,7 +313,7 @@ impl TransferContext { Self { args, state, - report: Arc::new(std::sync::Mutex::new(None)), + progress: ProgressReporter::default(), } } @@ -324,25 +321,11 @@ impl TransferContext { &self, report: Option>, ) { - *self.report.lock().unwrap() = report; - } - - pub fn reporter(&self) -> Option> { - self.report.lock().unwrap().clone() - } - - pub fn report_retry(&self, message: String, delay: Duration) { - self.report(CommandProgress::Retry(message, delay)) - } - - pub fn report_fetching_cached(&self) { - self.report(CommandProgress::FetchingFromCache) + self.progress.register_reporter(report); } - pub fn report(&self, status: CommandProgress) { - if let Some(reporter) = self.reporter() { - reporter.send(status).ok(); - } + pub fn reporter(&self) -> ProgressReporter { + self.progress.clone() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index e4133a3087..bed4fd7b47 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -3,12 +3,99 @@ use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; use futures::{SinkExt, StreamExt, TryFutureExt}; +use std::sync::Arc; use tokio::task::spawn_local; +use tokio::time::Instant; +use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::CommandProgress; type Stream = TransferStream; +#[derive(Debug, Clone)] +pub struct ProgressConfig { + /// Channel for watching for transfer progress. + pub progress: tokio::sync::broadcast::Sender, + pub progress_args: ProgressArgs, +} + +#[derive(Default, Clone)] +pub struct ProgressReporter { + config: ProgressArgs, + inner: Arc>>, +} + +struct ProgressImpl { + pub report: tokio::sync::broadcast::Sender, + pub last: CommandProgress, + pub last_update: Instant, +} + +impl ProgressReporter { + pub fn next_step(&self) { + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last.step.0 += 1; + inner.last.progress = (0, None); + inner.last_update = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok() + }); + } + + /// TODO: implement `update_interval` and `step` + pub fn report_progress(&self, progress: u64, size: Option) { + let _update_interval = self.config.update_interval; + let _step = self.config.step; + + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last.progress = (progress, size); + inner.last_update = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok() + }); + } + + pub fn report_message(&self, message: String) { + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last_update = Instant::now(); + inner + .report + .send(CommandProgress { + message: Some(message), + ..inner.last.clone() + }) + .ok() + }); + } + + pub fn report_fetching_cached(&self) { + self.report_message("Fetched file from cache".to_string()) + } + + pub fn register_reporter( + &self, + report: Option>, + ) { + if let Some(report) = report { + self.inner + .lock() + .unwrap() + .as_mut() + .map(|inner| inner.report = report); + } + } +} + /// Wraps a stream to report progress. /// The `report` function is called with the current offset and the total size. /// The total size is 0 if the size is unknown. (For example, when the source is a directory.) @@ -54,11 +141,7 @@ type Sink = TransferSink; pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { let report = ctx.reporter(); wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { - if let Some(report) = &report { - report - .send(CommandProgress::TransferProgress(progress, size)) - .ok(); - } + report.report_progress(progress, size) }) } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 8b5ccb718b..335c82fd74 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -16,6 +16,7 @@ use crate::{ HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +use crate::progress::ProgressConfig; use ya_client_model::activity::exe_script_command::ProgressArgs; pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; @@ -38,13 +39,6 @@ macro_rules! actor_try { }; } -#[derive(Debug, Clone)] -pub struct ProgressConfig { - /// Channel for watching for transfer progress. - pub progress: tokio::sync::broadcast::Sender, - pub progress_args: ProgressArgs, -} - #[derive(Debug, Message, Default)] #[rtype(result = "Result<()>")] pub struct TransferResource { @@ -93,7 +87,7 @@ impl AddVolumes { } } -#[derive(Debug, Message)] +#[derive(Debug, Message, Default)] #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, @@ -258,7 +252,8 @@ impl TransferService { let fut = async move { if path.exists() { log::info!("Deploying cached image: {:?}", path); - ctx.report_fetching_cached(); + ctx.reporter() + .report_message("Deployed image from cache".to_string()); return Ok(Some(path)); } diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index ab7c0e8bcd..57bf9656d4 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -3,15 +3,14 @@ use std::env; use std::time::Duration; use test_context::test_context; use tokio::time::sleep; +use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{ - AbortTransfers, CommandProgress, DeployImage, TransferService, TransferServiceContext, -}; +use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -66,7 +65,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re let result = addr .send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await?; log::info!("Deployment stopped"); @@ -76,7 +75,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re log::info!("Re-deploying the same image"); addr.send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await??; @@ -126,13 +125,14 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let addr = TransferService::new(exe_ctx).start(); log::info!("[>>] Deployment with hash verification"); - let (tx, mut rx) = tokio::sync::watch::channel(CommandProgress::default()); + let (tx, mut rx) = tokio::sync::broadcast::channel(15); tokio::task::spawn_local(async move { let _result = addr .send(DeployImage { task_package: task_package.clone(), progress: Some(tx), + progress_args: ProgressArgs::default(), }) .await??; log::info!("Deployment stopped"); @@ -140,18 +140,16 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< }); let mut last_progress = 0u64; - while let Ok(_) = rx.changed().await { - let progress = rx.borrow_and_update(); + while let Ok(progress) = rx.recv().await { + assert_eq!(progress.progress.1.unwrap(), file_size); + assert!(progress.progress.0 >= last_progress); - assert_eq!(progress.size.unwrap(), file_size); - assert!(progress.progress >= last_progress); - - last_progress = progress.progress; + last_progress = progress.progress.0; log::info!( "Progress: {}/{}", - progress.progress, - progress.size.unwrap_or(0) + progress.progress.0, + progress.progress.1.unwrap_or(0) ); } diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 63f167d3a2..a41ad24269 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,7 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), - progress: None, + progress_args: None, }) .await?; diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 9b561d2ef2..044e29384b 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -113,7 +113,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress: None, + progress_args: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index e92d5809b3..f339fecb4e 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -31,7 +31,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress: None, + progress_args: None, }) .await??; @@ -99,7 +99,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment with hash verification"); addr.send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await??; log::warn!("Deployment complete"); @@ -108,7 +108,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment from cache"); addr.send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await??; log::warn!("Deployment from cache complete"); @@ -157,7 +157,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu } #[ignore] -#[cfg_attr(not(feature = "framework-test"), ignore)] +//#[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/src/manifest.rs b/exe-unit/src/manifest.rs index 15aa919e49..21602bdc79 100644 --- a/exe-unit/src/manifest.rs +++ b/exe-unit/src/manifest.rs @@ -503,6 +503,10 @@ mod tests { ExeScriptCommand::Deploy { net: Default::default(), hosts: Default::default(), + hostname: None, + volumes: vec![], + env: Default::default(), + progress: None, }, ExeScriptCommand::Start { args: Default::default(), @@ -657,6 +661,7 @@ mod tests { from: "/src/0x0add".to_string(), to: "/dst/0x0add".to_string(), args: Default::default(), + progress: None, }]; let validator: ScriptValidator = r#"{ From b40c4e6ed9ea2b1b4d257707317cc78478701d20 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 8 Jan 2024 19:10:29 +0100 Subject: [PATCH 11/52] Use ProgressReporter in Deploy and Transfer flow --- exe-unit/components/transfer/src/lib.rs | 9 +- exe-unit/components/transfer/src/progress.rs | 25 +++-- exe-unit/components/transfer/src/transfer.rs | 101 +++++++++++++----- .../components/transfer/tests/test_deploy.rs | 10 +- .../transfer/tests/test_transfer_abort.rs | 2 +- .../transfer/tests/test_transfer_resume.rs | 2 +- .../transfer/tests/test_transfer_service.rs | 2 +- exe-unit/src/lib.rs | 28 +++-- 8 files changed, 115 insertions(+), 64 deletions(-) diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 4f05b41396..4cca1ff186 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -38,7 +38,7 @@ pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_pr pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; -use ya_client_model::activity::{CommandProgress, TransferArgs}; +use ya_client_model::activity::TransferArgs; /// Transfers data from `stream` to a `TransferSink` pub async fn transfer(stream: S, mut sink: TransferSink) -> Result<(), Error> @@ -317,13 +317,6 @@ impl TransferContext { } } - pub fn register_reporter( - &self, - report: Option>, - ) { - self.progress.register_reporter(report); - } - pub fn reporter(&self) -> ProgressReporter { self.progress.clone() } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index bed4fd7b47..bbddbe9de8 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -78,20 +78,23 @@ impl ProgressReporter { }); } - pub fn report_fetching_cached(&self) { - self.report_message("Fetched file from cache".to_string()) - } - pub fn register_reporter( &self, - report: Option>, + args: Option, + steps: usize, + unit: Option, ) { - if let Some(report) = report { - self.inner - .lock() - .unwrap() - .as_mut() - .map(|inner| inner.report = report); + if let Some(args) = args { + *(self.inner.lock().unwrap()) = Some(ProgressImpl { + report: args.progress, + last: CommandProgress { + step: (0, steps), + message: None, + progress: (0, None), + unit, + }, + last_update: Instant::now(), + }); } } } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 335c82fd74..55022a73d2 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -11,12 +11,12 @@ use url::Url; use crate::cache::{Cache, CachePath}; use crate::error::Error; use crate::error::Error as TransferError; +pub use crate::progress::ProgressConfig; use crate::{ transfer_with, ContainerTransferProvider, FileTransferProvider, GftpTransferProvider, HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; -use crate::progress::ProgressConfig; use ya_client_model::activity::exe_script_command::ProgressArgs; pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; @@ -46,19 +46,40 @@ pub struct TransferResource { pub to: String, pub args: TransferArgs, /// Progress reporting configuration. `None` means that there will be no progress updates. - pub progress_args: Option, + pub progress_config: Option, } -impl TransferResource { - pub fn forward_progress( +#[derive(Message)] +#[rtype(result = "Result<()>")] +pub struct AddVolumes(Vec); + +impl AddVolumes { + pub fn new(vols: Vec) -> Self { + AddVolumes(vols) + } +} + +#[derive(Debug, Message, Default)] +#[rtype(result = "Result>")] +pub struct DeployImage { + pub task_package: Option, + /// Progress reporting configuration. `None` means that there will be no progress updates. + pub progress_config: Option, +} + +pub trait ForwardProgressToSink { + fn progress_config_mut(&mut self) -> &mut Option; + + fn forward_progress( &mut self, args: &ProgressArgs, sender: impl Sink + 'static, ) { - let rx = match &self.progress_args { + let progress_args = self.progress_config_mut(); + let rx = match progress_args { None => { let (tx, rx) = tokio::sync::broadcast::channel(50); - self.progress_args = Some(ProgressConfig { + *progress_args = Some(ProgressConfig { progress: tx, progress_args: args.clone(), }); @@ -77,24 +98,44 @@ impl TransferResource { } } -#[derive(Message)] -#[rtype(result = "Result<()>")] -pub struct AddVolumes(Vec); +impl ForwardProgressToSink for DeployImage { + fn progress_config_mut(&mut self) -> &mut Option { + &mut self.progress_config + } +} -impl AddVolumes { - pub fn new(vols: Vec) -> Self { - AddVolumes(vols) +impl ForwardProgressToSink for TransferResource { + fn progress_config_mut(&mut self) -> &mut Option { + &mut self.progress_config } } -#[derive(Debug, Message, Default)] -#[rtype(result = "Result>")] -pub struct DeployImage { - pub task_package: Option, - pub progress_args: ProgressArgs, - /// Channel for watching for deploy progress. `None` means that there - /// will be no progress updates. - pub progress: Option>, +impl DeployImage { + pub fn forward_progress( + &mut self, + args: &ProgressArgs, + sender: impl Sink + 'static, + ) { + let rx = match &self.progress_config { + None => { + let (tx, rx) = tokio::sync::broadcast::channel(50); + self.progress_config = Some(ProgressConfig { + progress: tx, + progress_args: args.clone(), + }); + rx + } + Some(args) => args.progress.subscribe(), + }; + + tokio::task::spawn_local(async move { + tokio_stream::wrappers::BroadcastStream::new(rx) + .map_err(|e| Error::Other(e.to_string())) + .forward(sender) + .await + .ok() + }); + } } #[derive(Clone, Debug, Message)] @@ -198,7 +239,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, - progress: Option>, + ctx: TransferContext, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -220,7 +261,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, - progress: Option>, + ctx: TransferContext, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -231,10 +272,6 @@ impl TransferService { hash: None, }; - let ctx = TransferContext::default(); - ctx.state.retry_with(self.deploy_retry.clone()); - ctx.register_reporter(progress); - // Using partially downloaded image from previous executions could speed up deploy // process, but it comes with the cost: If image under URL changed, Requestor will get // error on the end. This can result with Provider being perceived as unreliable. @@ -312,11 +349,16 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); + let ctx = TransferContext::default(); + ctx.state.retry_with(self.deploy_retry.clone()); + ctx.progress + .register_reporter(deploy.progress_config, 1, Some("Bytes".to_string())); + #[cfg(not(feature = "sgx"))] - return self.deploy_no_sgx(src_url, src_name, path, deploy.progress); + return self.deploy_no_sgx(src_url, src_name, path, ctx); #[cfg(feature = "sgx")] - return self.deploy_sgx(src_url, src_name, path, deploy.progress); + return self.deploy_sgx(src_url, src_name, path, ctx); } } @@ -331,7 +373,8 @@ impl Handler for TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); - ctx.register_reporter(msg.progress_args.map(|args| args.progress)); + ctx.progress + .register_reporter(msg.progress_config, 1, Some("Bytes".to_string())); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 57bf9656d4..509c8fc0d3 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -10,7 +10,9 @@ use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; +use ya_transfer::transfer::{ + AbortTransfers, DeployImage, ProgressConfig, TransferService, TransferServiceContext, +}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -131,8 +133,10 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let _result = addr .send(DeployImage { task_package: task_package.clone(), - progress: Some(tx), - progress_args: ProgressArgs::default(), + progress_config: Some(ProgressConfig { + progress: tx, + progress_args: ProgressArgs::default(), + }), }) .await??; log::info!("Deployment stopped"); diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index a41ad24269..3b8e7e7ec6 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,7 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), - progress_args: None, + progress_config: None, }) .await?; diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 044e29384b..7253453199 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -113,7 +113,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress_args: None, + progress_config: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index f339fecb4e..fe2cf29d8e 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -31,7 +31,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress_args: None, + progress_config: None, }) .await??; diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index e3983044ca..a2cfb911a6 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -18,7 +18,8 @@ use ya_core_model::activity::local::Credentials; use ya_runtime_api::deploy; use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; use ya_transfer::transfer::{ - AddVolumes, DeployImage, TransferResource, TransferService, TransferServiceContext, + AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, + TransferServiceContext, }; use crate::acl::Acl; @@ -313,7 +314,7 @@ impl RuntimeRef { from: from.clone(), to: to.clone(), args: args.clone(), - progress_args: None, + progress_config: None, }; if let Some(args) = progress { @@ -321,15 +322,22 @@ impl RuntimeRef { } transfer_service.send(msg).await??; } - ExeScriptCommand::Deploy { net, hosts, .. } => { + ExeScriptCommand::Deploy { + net, + hosts, + progress, + .. + } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. - let task_package = transfer_service - .send(DeployImage { - task_package: None, - progress_args: Default::default(), - progress: None, - }) - .await??; + let mut msg = DeployImage { + task_package: None, + progress_config: None, + }; + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + + let task_package = transfer_service.send(msg).await??; runtime .send(UpdateDeployment { task_package, From 1916012f6b239cc93587850e034c4aba07bd1adb Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 8 Jan 2024 19:34:08 +0100 Subject: [PATCH 12/52] More user friendly Deploy api in TransferService --- exe-unit/components/transfer/src/transfer.rs | 9 ++++ .../components/transfer/tests/test_deploy.rs | 49 ++++++++----------- .../transfer/tests/test_transfer_service.rs | 18 +++---- exe-unit/src/lib.rs | 5 +- 4 files changed, 36 insertions(+), 45 deletions(-) diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 55022a73d2..251f87f930 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -67,6 +67,15 @@ pub struct DeployImage { pub progress_config: Option, } +impl DeployImage { + pub fn with_package(task_package: &str) -> DeployImage { + DeployImage { + task_package: Some(task_package.to_string()), + progress_config: None, + } + } +} + pub trait ForwardProgressToSink { fn progress_config_mut(&mut self) -> &mut Option; diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 509c8fc0d3..ae32972974 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -1,18 +1,20 @@ use actix::Actor; +use futures::channel::mpsc; +use futures::SinkExt; use std::env; use std::time::Duration; use test_context::test_context; use tokio::time::sleep; +use tokio_stream::StreamExt; use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::CommandProgress; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{ - AbortTransfers, DeployImage, ProgressConfig, TransferService, TransferServiceContext, -}; +use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -42,10 +44,10 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -64,22 +66,14 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re }); log::info!("[>>] Deployment with hash verification"); - let result = addr - .send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await?; + let result = addr.send(DeployImage::with_package(&task_package)).await?; log::info!("Deployment stopped"); assert!(result.is_err()); log::info!("Re-deploying the same image"); - addr.send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; Ok(()) } @@ -113,10 +107,10 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -127,24 +121,21 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let addr = TransferService::new(exe_ctx).start(); log::info!("[>>] Deployment with hash verification"); - let (tx, mut rx) = tokio::sync::broadcast::channel(15); + let (tx, mut rx) = mpsc::channel::(15); + let mut msg = DeployImage::with_package(&task_package); + msg.forward_progress( + &ProgressArgs::default(), + tx.sink_map_err(|e| ya_transfer::error::Error::Other(e.to_string())), + ); tokio::task::spawn_local(async move { - let _result = addr - .send(DeployImage { - task_package: task_package.clone(), - progress_config: Some(ProgressConfig { - progress: tx, - progress_args: ProgressArgs::default(), - }), - }) - .await??; + let _result = addr.send(msg).await??; log::info!("Deployment stopped"); anyhow::Ok(()) }); let mut last_progress = 0u64; - while let Ok(progress) = rx.recv().await { + while let Some(progress) = rx.next().await { assert_eq!(progress.progress.1.unwrap(), file_size); assert!(progress.progress.0 >= last_progress); diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index fe2cf29d8e..80efccc800 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -79,10 +79,10 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -97,20 +97,14 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu println!(); log::warn!("[>>] Deployment with hash verification"); - addr.send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; log::warn!("Deployment complete"); println!(); log::warn!("[>>] Deployment from cache"); - addr.send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; log::warn!("Deployment from cache complete"); println!(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index a2cfb911a6..c2d28d6908 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -329,10 +329,7 @@ impl RuntimeRef { .. } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. - let mut msg = DeployImage { - task_package: None, - progress_config: None, - }; + let mut msg = DeployImage::default(); if let Some(args) = progress { msg.forward_progress(args, runtime_cmd.progress_sink()) } From f07e678bfcd1414dd7e9ee62a0da53dedabf48a5 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 9 Jan 2024 17:09:41 +0100 Subject: [PATCH 13/52] Update ya-client --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- exe-unit/components/transfer/src/progress.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1d6749a48..ffd3f7cb13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8178,7 +8178,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" +source = "git+https://github.com/golemfactory/ya-client.git?rev=f2fb8e5392c5fdee0f965309625a561c330a8eae#f2fb8e5392c5fdee0f965309625a561c330a8eae" dependencies = [ "actix-codec", "awc", @@ -8202,7 +8202,7 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" +source = "git+https://github.com/golemfactory/ya-client.git?rev=f2fb8e5392c5fdee0f965309625a561c330a8eae#f2fb8e5392c5fdee0f965309625a561c330a8eae" dependencies = [ "bigdecimal 0.2.2", "chrono", diff --git a/Cargo.toml b/Cargo.toml index e9ffd15bd3..63d9331e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "f2fb8e5392c5fdee0f965309625a561c330a8eae" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "f2fb8e5392c5fdee0f965309625a561c330a8eae" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } ## RELAY and networking stack diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index bbddbe9de8..d29cd8de71 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -50,7 +50,7 @@ impl ProgressReporter { /// TODO: implement `update_interval` and `step` pub fn report_progress(&self, progress: u64, size: Option) { let _update_interval = self.config.update_interval; - let _step = self.config.step; + let _update_step = self.config.update_step; self.inner.lock().unwrap().as_mut().map(|inner| { inner.last.progress = (progress, size); From 10e62dbb18a4d0579f669a3cd08c45279bde2c55 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 11 Jan 2024 18:41:29 +0100 Subject: [PATCH 14/52] Limit speed of progress updates to avoid unresponsive exe-unit --- exe-unit/components/transfer/src/progress.rs | 33 ++++++++++++-------- exe-unit/src/lib.rs | 1 + 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index d29cd8de71..91932d8e1c 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -4,6 +4,7 @@ use crate::{TransferContext, TransferData}; use futures::{SinkExt, StreamExt, TryFutureExt}; use std::sync::Arc; +use std::time::Duration; use tokio::task::spawn_local; use tokio::time::Instant; @@ -28,7 +29,7 @@ pub struct ProgressReporter { struct ProgressImpl { pub report: tokio::sync::broadcast::Sender, pub last: CommandProgress, - pub last_update: Instant, + pub last_send: Instant, } impl ProgressReporter { @@ -36,7 +37,7 @@ impl ProgressReporter { self.inner.lock().unwrap().as_mut().map(|inner| { inner.last.step.0 += 1; inner.last.progress = (0, None); - inner.last_update = Instant::now(); + inner.last_send = Instant::now(); inner .report .send(CommandProgress { @@ -49,25 +50,31 @@ impl ProgressReporter { /// TODO: implement `update_interval` and `step` pub fn report_progress(&self, progress: u64, size: Option) { - let _update_interval = self.config.update_interval; + let update_interval: Duration = self + .config + .update_interval + .map(Into::into) + .unwrap_or(Duration::from_secs(1)); let _update_step = self.config.update_step; self.inner.lock().unwrap().as_mut().map(|inner| { inner.last.progress = (progress, size); - inner.last_update = Instant::now(); - inner - .report - .send(CommandProgress { - message: None, - ..inner.last.clone() - }) - .ok() + if inner.last_send + update_interval <= Instant::now() { + inner.last_send = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok(); + } }); } pub fn report_message(&self, message: String) { self.inner.lock().unwrap().as_mut().map(|inner| { - inner.last_update = Instant::now(); + inner.last_send = Instant::now(); inner .report .send(CommandProgress { @@ -93,7 +100,7 @@ impl ProgressReporter { progress: (0, None), unit, }, - last_update: Instant::now(), + last_send: Instant::now(), }); } } diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index c2d28d6908..3bd6322d36 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -98,6 +98,7 @@ impl ExeUnit { let supervisor_template = OfferTemplate::new(serde_json::json!({ "golem.com.usage.vector": MetricsService::usage_vector(), "golem.activity.caps.transfer.protocol": TransferService::schemes(), + "golem.activity.caps.transfer.report-progress": true, })); Ok(supervisor_template.patch(runtime_template)) From d9f9bbc39221cbd8b2e7b85b3e9fd0fe3fa9eb3e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jan 2024 16:43:14 +0100 Subject: [PATCH 15/52] ProgressReporter update interval test --- Cargo.lock | 1 + exe-unit/components/transfer/Cargo.toml | 1 + exe-unit/components/transfer/src/progress.rs | 78 +++++++++++++++++++- exe-unit/components/transfer/src/transfer.rs | 4 +- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffd3f7cb13..1bcd56c82c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9337,6 +9337,7 @@ dependencies = [ "awc", "bytes 1.5.0", "crossterm 0.26.1", + "duration-string", "env_logger 0.7.1", "futures 0.3.30", "gftp", diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 9e9de1e517..e9e67ddae0 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -72,6 +72,7 @@ actix-files = "0.6" actix-web = "4" anyhow = "1.0" crossterm = "0.26.1" +duration-string = "0.3" env_logger = "0.7" secp256k1 = { version = "0.27.0" } serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 91932d8e1c..e9814d6d43 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -48,7 +48,7 @@ impl ProgressReporter { }); } - /// TODO: implement `update_interval` and `step` + /// TODO: implement `update_step` pub fn report_progress(&self, progress: u64, size: Option) { let update_interval: Duration = self .config @@ -86,12 +86,13 @@ impl ProgressReporter { } pub fn register_reporter( - &self, + &mut self, args: Option, steps: usize, unit: Option, ) { if let Some(args) = args { + self.config = args.progress_args; *(self.inner.lock().unwrap()) = Some(ProgressImpl { report: args.progress, last: CommandProgress { @@ -198,3 +199,76 @@ where sink } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Instant; + + use duration_string::DurationString; + use tokio::time::Duration; + + #[actix_rt::test] + async fn test_progress_reporter_interval() { + let mut report = ProgressReporter::default(); + let (tx, mut rx) = tokio::sync::broadcast::channel(10); + report.register_reporter( + Some(ProgressConfig { + progress: tx, + progress_args: ProgressArgs { + update_interval: Some("500ms".parse::().unwrap()), + update_step: None, + }, + }), + 2, + Some("Bytes".to_string()), + ); + + let size = 200; + let mut before = Instant::now(); + tokio::task::spawn_local(async move { + for _step in 0..2 { + tokio::time::sleep(Duration::from_millis(25)).await; + + for i in 0..=size { + report.report_progress(i, Some(size)); + tokio::time::sleep(Duration::from_millis(50)).await; + } + report.next_step(); + } + }); + + let mut counter = 0; + let mut step = 0; + while let Ok(progress) = rx.recv().await { + //println!("{progress:?}"); + + counter += 1; + let update = Instant::now().duration_since(before); + before = Instant::now(); + let diff = if update > Duration::from_millis(525) { + update - Duration::from_millis(525) + } else { + Duration::from_millis(525) - update + }; + + assert!(diff <= Duration::from_millis(20)); + + // `ProgressReporter` should ignore 10 messages in each loop. + assert_eq!(progress.progress.0, counter * 10); + assert_eq!(progress.progress.1, Some(size)); + assert_eq!(progress.step, (step, 2)); + assert_eq!(progress.unit, Some("Bytes".to_string())); + assert_eq!(progress.message, None); + + if counter == 20 { + counter = 0; + step += 1; + + // Skip step change event + rx.recv().await.unwrap(); + before = Instant::now(); + } + } + } +} diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 251f87f930..b9e9f55739 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -358,7 +358,7 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); - let ctx = TransferContext::default(); + let mut ctx = TransferContext::default(); ctx.state.retry_with(self.deploy_retry.clone()); ctx.progress .register_reporter(deploy.progress_config, 1, Some("Bytes".to_string())); @@ -380,7 +380,7 @@ impl Handler for TransferService { let src = actor_try!(self.provider(&src_url)); let dst = actor_try!(self.provider(&dst_url)); - let ctx = TransferContext::default(); + let mut ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); ctx.progress .register_reporter(msg.progress_config, 1, Some("Bytes".to_string())); From 7ad5482dd438a37c4a5ed34f55fe089633d28ad3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jan 2024 16:55:30 +0100 Subject: [PATCH 16/52] Test sending progress message --- exe-unit/components/transfer/src/progress.rs | 33 ++++++++++++++------ 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index e9814d6d43..289ed049b2 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -227,21 +227,24 @@ mod tests { let size = 200; let mut before = Instant::now(); tokio::task::spawn_local(async move { - for _step in 0..2 { + for step in 0..2 { tokio::time::sleep(Duration::from_millis(25)).await; for i in 0..=size { report.report_progress(i, Some(size)); tokio::time::sleep(Duration::from_millis(50)).await; } - report.next_step(); + if step == 0 { + report.next_step(); + } } + report.report_message("Finished".to_string()); }); let mut counter = 0; let mut step = 0; - while let Ok(progress) = rx.recv().await { - //println!("{progress:?}"); + while let Ok(event) = rx.recv().await { + //println!("{event:?}"); counter += 1; let update = Instant::now().duration_since(before); @@ -255,13 +258,17 @@ mod tests { assert!(diff <= Duration::from_millis(20)); // `ProgressReporter` should ignore 10 messages in each loop. - assert_eq!(progress.progress.0, counter * 10); - assert_eq!(progress.progress.1, Some(size)); - assert_eq!(progress.step, (step, 2)); - assert_eq!(progress.unit, Some("Bytes".to_string())); - assert_eq!(progress.message, None); + assert_eq!(event.progress.0, counter * 10); + assert_eq!(event.progress.1, Some(size)); + assert_eq!(event.step, (step, 2)); + assert_eq!(event.unit, Some("Bytes".to_string())); + assert_eq!(event.message, None); if counter == 20 { + if step == 1 { + break; + } + counter = 0; step += 1; @@ -270,5 +277,13 @@ mod tests { before = Instant::now(); } } + + // Reporting message will result in event containing progress adn step from previous event. + let last = rx.recv().await.unwrap(); + //println!("{last:?}"); + assert_eq!(last.message, Some("Finished".to_string())); + assert_eq!(last.progress.0, size); + assert_eq!(last.progress.1, Some(size)); + assert_eq!(last.step, (1, 2)); } } From f3b76b1e8226cc5400b3e6434c3d8010ac55bd0d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 17 Jan 2024 13:53:48 +0100 Subject: [PATCH 17/52] Make ExeUnit instantiable in test --- exe-unit/components/transfer/src/transfer.rs | 2 +- exe-unit/src/bin.rs | 337 +------- exe-unit/src/exe_unit.rs | 579 +++++++++++++ exe-unit/src/handlers/local.rs | 6 +- exe-unit/src/lib.rs | 816 +++++++------------ 5 files changed, 875 insertions(+), 865 deletions(-) create mode 100644 exe-unit/src/exe_unit.rs diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index b9e9f55739..afca8cb6e0 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -248,7 +248,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, - ctx: TransferContext, + _ctx: TransferContext, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) diff --git a/exe-unit/src/bin.rs b/exe-unit/src/bin.rs index 53b1e37077..4dc9e0b7bc 100644 --- a/exe-unit/src/bin.rs +++ b/exe-unit/src/bin.rs @@ -1,191 +1,7 @@ -use actix::{Actor, Addr}; -use anyhow::{bail, Context}; -use futures::channel::oneshot; -use std::convert::TryFrom; -use std::path::PathBuf; -use structopt::{clap, StructOpt}; +use structopt::StructOpt; -use ya_client_model::activity::ExeScriptCommand; -use ya_service_bus::RpcEnvelope; - -use ya_core_model::activity; -use ya_exe_unit::agreement::Agreement; use ya_exe_unit::logger::*; -use ya_exe_unit::manifest::ManifestContext; -use ya_exe_unit::message::{GetState, GetStateResponse, Register}; -use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::service::metrics::MetricsService; -use ya_exe_unit::service::signal::SignalMonitor; -use ya_exe_unit::state::Supervision; -use ya_exe_unit::{ExeUnit, ExeUnitContext}; -use ya_transfer::transfer::TransferService; -use ya_utils_path::normalize_path; - -#[derive(structopt::StructOpt, Debug)] -#[structopt(global_setting = clap::AppSettings::ColoredHelp)] -#[structopt(version = ya_compile_time_utils::version_describe!())] -struct Cli { - /// Runtime binary path - #[structopt(long, short)] - binary: PathBuf, - #[structopt(flatten)] - supervise: SuperviseCli, - /// Additional runtime arguments - #[structopt( - long, - short, - set = clap::ArgSettings::Global, - number_of_values = 1, - )] - runtime_arg: Vec, - /// Enclave secret key used in secure communication - #[structopt( - long, - env = "EXE_UNIT_SEC_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, - )] - #[allow(dead_code)] - sec_key: Option, - /// Requestor public key used in secure communication - #[structopt( - long, - env = "EXE_UNIT_REQUESTOR_PUB_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, - )] - #[allow(dead_code)] - requestor_pub_key: Option, - #[structopt(subcommand)] - command: Command, -} - -#[derive(structopt::StructOpt, Debug)] -struct SuperviseCli { - /// Hardware resources are handled by the runtime - #[structopt( - long = "runtime-managed-hardware", - alias = "cap-handoff", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, - )] - hardware: bool, - /// Images are handled by the runtime - #[structopt( - long = "runtime-managed-image", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, - )] - image: bool, -} - -#[derive(structopt::StructOpt, Debug)] -#[structopt(global_setting = clap::AppSettings::DeriveDisplayOrder)] -enum Command { - /// Execute commands from file - FromFile { - /// ExeUnit daemon GSB URL - #[structopt(long)] - report_url: Option, - /// ExeUnit service ID - #[structopt(long)] - service_id: Option, - /// Command file path - input: PathBuf, - #[structopt(flatten)] - args: RunArgs, - }, - /// Bind to Service Bus - ServiceBus { - /// ExeUnit service ID - service_id: String, - /// ExeUnit daemon GSB URL - report_url: String, - #[structopt(flatten)] - args: RunArgs, - }, - /// Print an offer template in JSON format - OfferTemplate, - /// Run runtime's test command - Test, -} - -#[derive(structopt::StructOpt, Debug)] -struct RunArgs { - /// Agreement file path - #[structopt(long, short)] - agreement: PathBuf, - /// Working directory - #[structopt(long, short)] - work_dir: PathBuf, - /// Common cache directory - #[structopt(long, short)] - cache_dir: PathBuf, -} - -fn create_path(path: &PathBuf) -> anyhow::Result { - if let Err(error) = std::fs::create_dir_all(path) { - match &error.kind() { - std::io::ErrorKind::AlreadyExists => (), - _ => bail!("Can't create directory: {}, {}", path.display(), error), - } - } - Ok(normalize_path(path)?) -} - -#[cfg(feature = "sgx")] -fn init_crypto( - sec_key: Option, - req_key: Option, -) -> anyhow::Result { - use ya_exe_unit::crypto::Crypto; - - let req_key = req_key.ok_or_else(|| anyhow::anyhow!("Missing requestor public key"))?; - match sec_key { - Some(key) => Ok(Crypto::try_with_keys(key, req_key)?), - None => { - log::info!("Generating a new key pair..."); - Ok(Crypto::try_new(req_key)?) - } - } -} - -async fn send_script( - exe_unit: Addr>, - activity_id: Option, - exe_script: Vec, -) { - use std::time::Duration; - use ya_exe_unit::state::{State, StatePair}; - - let delay = Duration::from_secs_f32(0.5); - loop { - match exe_unit.send(GetState).await { - Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, - Ok(GetStateResponse(StatePair(State::Terminated, _))) - | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) - | Err(_) => { - return log::error!("ExeUnit has terminated"); - } - _ => tokio::time::sleep(delay).await, - } - } - - log::debug!("Executing commands: {:?}", exe_script); - - let msg = activity::Exec { - activity_id: activity_id.unwrap_or_default(), - batch_id: hex::encode(rand::random::<[u8; 16]>()), - exe_script, - timeout: None, - }; - if let Err(e) = exe_unit - .send(RpcEnvelope::with_caller(String::new(), msg)) - .await - { - log::error!("Unable to execute exe script: {:?}", e); - } -} +use ya_exe_unit::{run, Cli}; #[cfg(feature = "packet-trace-enable")] fn init_packet_trace() -> anyhow::Result<()> { @@ -197,147 +13,6 @@ fn init_packet_trace() -> anyhow::Result<()> { Ok(()) } -async fn run() -> anyhow::Result<()> { - dotenv::dotenv().ok(); - - #[cfg(feature = "packet-trace-enable")] - init_packet_trace()?; - - #[allow(unused_mut)] - let mut cli: Cli = Cli::from_args(); - if !cli.binary.exists() { - bail!("Runtime binary does not exist: {}", cli.binary.display()); - } - - let mut commands = None; - let ctx_activity_id; - let ctx_report_url; - - let args = match &cli.command { - Command::FromFile { - args, - service_id, - report_url, - input, - } => { - let contents = std::fs::read_to_string(input).map_err(|e| { - anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) - })?; - let contents = serde_json::from_str(&contents).map_err(|e| { - anyhow::anyhow!( - "Cannot deserialize commands from file {}: {e}", - input.display(), - ) - })?; - ctx_activity_id = service_id.clone(); - ctx_report_url = report_url.clone(); - commands = Some(contents); - args - } - Command::ServiceBus { - args, - service_id, - report_url, - } => { - ctx_activity_id = Some(service_id.clone()); - ctx_report_url = Some(report_url.clone()); - args - } - Command::OfferTemplate => { - let args = cli.runtime_arg.clone(); - let offer_template = ExeUnit::::offer_template(cli.binary, args)?; - println!("{}", serde_json::to_string(&offer_template)?); - return Ok(()); - } - Command::Test => { - let args = cli.runtime_arg.clone(); - let output = ExeUnit::::test(cli.binary, args)?; - println!("{}", String::from_utf8_lossy(&output.stdout)); - eprintln!("{}", String::from_utf8_lossy(&output.stderr)); - if !output.status.success() { - bail!("Test failed"); - } - return Ok(()); - } - }; - - if !args.agreement.exists() { - bail!( - "Agreement file does not exist: {}", - args.agreement.display() - ); - } - let work_dir = create_path(&args.work_dir).map_err(|e| { - anyhow::anyhow!( - "Cannot create the working directory {}: {e}", - args.work_dir.display(), - ) - })?; - let cache_dir = create_path(&args.cache_dir).map_err(|e| { - anyhow::anyhow!( - "Cannot create the cache directory {}: {e}", - args.work_dir.display(), - ) - })?; - let mut agreement = Agreement::try_from(&args.agreement).map_err(|e| { - anyhow::anyhow!( - "Error parsing the agreement from {}: {e}", - args.agreement.display(), - ) - })?; - - log::info!("Attempting to read app manifest .."); - - let manifest_ctx = - ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; - agreement.task_package = manifest_ctx - .payload() - .or_else(|| agreement.task_package.take()); - - log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); - log::info!("User-provided payload: {:?}", agreement.task_package); - - let ctx = ExeUnitContext { - supervise: Supervision { - hardware: cli.supervise.hardware, - image: cli.supervise.image, - manifest: manifest_ctx, - }, - activity_id: ctx_activity_id.clone(), - report_url: ctx_report_url, - agreement, - work_dir, - cache_dir, - runtime_args: cli.runtime_arg.clone(), - acl: Default::default(), - credentials: None, - #[cfg(feature = "sgx")] - crypto: init_crypto( - cli.sec_key.replace("".into()), - cli.requestor_pub_key.clone(), - )?, - }; - - log::debug!("CLI args: {:?}", cli); - log::debug!("ExeUnitContext args: {:?}", ctx); - - let (tx, rx) = oneshot::channel(); - - let metrics = MetricsService::try_new(&ctx, Some(10000), ctx.supervise.hardware)?.start(); - let transfers = TransferService::new((&ctx).into()).start(); - let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); - let exe_unit = ExeUnit::new(tx, ctx, metrics, transfers, runtime).start(); - let signals = SignalMonitor::new(exe_unit.clone()).start(); - exe_unit.send(Register(signals)).await?; - - if let Some(exe_script) = commands { - tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); - } - - rx.await??; - Ok(()) -} - #[actix_rt::main] async fn main() { let panic_hook = std::panic::take_hook(); @@ -351,7 +26,13 @@ async fn main() { log::warn!("Using fallback logging due to an error: {:?}", error); }; - std::process::exit(match run().await { + dotenv::dotenv().ok(); + #[cfg(feature = "packet-trace-enable")] + init_packet_trace()?; + + let cli: Cli = Cli::from_args(); + + std::process::exit(match run(cli).await { Ok(_) => 0, Err(error) => { log::error!("{}", error); diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs new file mode 100644 index 0000000000..b666fdd259 --- /dev/null +++ b/exe-unit/src/exe_unit.rs @@ -0,0 +1,579 @@ +use actix::dev::IntervalFunc; +use actix::{ + Actor, ActorFutureExt, ActorStreamExt, Addr, AsyncContext, Context, ContextFutureSpawner, + Handler, ResponseFuture, Running, StreamHandler, WrapFuture, +}; +use chrono::Utc; +use futures::channel::{mpsc, oneshot}; +use futures::{FutureExt, SinkExt}; +use std::path::PathBuf; +use std::time::Duration; + +use ya_agreement_utils::OfferTemplate; +use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; +use ya_core_model::activity; +use ya_core_model::activity::local::Credentials; +use ya_runtime_api::deploy; +use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; +use ya_transfer::transfer::{ + AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, + TransferServiceContext, +}; + +use crate::acl::Acl; +use crate::agreement::Agreement; +use crate::error::Error; +use crate::message::{ + ExecuteCommand, GetMetrics, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, + ShutdownReason, SignExeScript, Stop, UpdateDeployment, +}; +use crate::runtime::{Runtime, RuntimeMode}; +use crate::service::metrics::MetricsService; +use crate::service::{ServiceAddr, ServiceControl}; +use crate::state::{ExeUnitState, StateError, Supervision}; + +lazy_static::lazy_static! { + static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); +} + +pub struct ExeUnit { + pub(crate) ctx: ExeUnitContext, + pub(crate) state: ExeUnitState, + pub(crate) events: Channel, + pub(crate) runtime: Addr, + pub(crate) metrics: Addr, + pub(crate) transfers: Addr, + pub(crate) services: Vec>, + pub(crate) shutdown_tx: Option>>, +} + +impl ExeUnit { + pub fn new( + shutdown_tx: oneshot::Sender>, + ctx: ExeUnitContext, + metrics: Addr, + transfers: Addr, + runtime: Addr, + ) -> Self { + ExeUnit { + ctx, + state: ExeUnitState::default(), + events: Channel::default(), + runtime: runtime.clone(), + metrics: metrics.clone(), + transfers: transfers.clone(), + services: vec![ + Box::new(ServiceAddr::new(metrics)), + Box::new(ServiceAddr::new(transfers)), + Box::new(ServiceAddr::new(runtime)), + ], + shutdown_tx: Some(shutdown_tx), + } + } + + pub fn offer_template(binary: PathBuf, args: Vec) -> crate::Result { + use crate::runtime::process::RuntimeProcess; + + let runtime_template = RuntimeProcess::offer_template(binary, args)?; + let supervisor_template = OfferTemplate::new(serde_json::json!({ + "golem.com.usage.vector": MetricsService::usage_vector(), + "golem.activity.caps.transfer.protocol": TransferService::schemes(), + "golem.activity.caps.transfer.report-progress": true, + })); + + Ok(supervisor_template.patch(runtime_template)) + } + + pub fn test(binary: PathBuf, args: Vec) -> crate::Result { + use crate::runtime::process::RuntimeProcess; + RuntimeProcess::test(binary, args) + } + + fn report_usage(&mut self, context: &mut Context) { + if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { + return; + } + let fut = report_usage( + self.ctx.report_url.clone().unwrap(), + self.ctx.activity_id.clone().unwrap(), + context.address(), + self.metrics.clone(), + ); + context.spawn(fut.into_actor(self)); + } + + pub(crate) async fn stop_runtime(runtime: Addr, reason: ShutdownReason) { + if let Err(e) = runtime + .send(Shutdown(reason)) + .timeout(Duration::from_secs(5u64)) + .await + { + log::warn!("Unable to stop the runtime: {:?}", e); + } + } +} + +#[derive(Clone)] +pub struct RuntimeRef(Addr>); + +impl RuntimeRef { + pub fn from_ctx(ctx: &Context>) -> Self { + RuntimeRef(ctx.address()) + } +} + +impl std::ops::Deref for RuntimeRef { + type Target = Addr>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl RuntimeRef { + pub async fn exec( + self, + exec: activity::Exec, + runtime: Addr, + transfers: Addr, + mut events: mpsc::Sender, + mut control: oneshot::Receiver<()>, + ) { + let batch_id = exec.batch_id.clone(); + for (idx, command) in exec.exe_script.into_iter().enumerate() { + if let Ok(Some(_)) = control.try_recv() { + log::warn!("Batch {} execution aborted", batch_id); + break; + } + + let runtime_cmd = ExecuteCommand { + batch_id: batch_id.clone(), + command: command.clone(), + tx: events.clone(), + idx, + }; + + let evt = RuntimeEvent::started(batch_id.clone(), idx, command.clone()); + if let Err(e) = events.send(evt).await { + log::error!("Unable to report event: {:?}", e); + } + + let (return_code, message) = match { + if runtime_cmd.stateless() { + self.exec_stateless(&runtime_cmd).await + } else { + self.exec_stateful(runtime_cmd, &runtime, &transfers).await + } + } { + Ok(_) => (0, None), + Err(ref err) => match err { + Error::CommandExitCodeError(c) => (*c, Some(err.to_string())), + _ => (-1, Some(err.to_string())), + }, + }; + + let evt = RuntimeEvent::finished(batch_id.clone(), idx, return_code, message.clone()); + if let Err(e) = events.send(evt).await { + log::error!("Unable to report event: {:?}", e); + } + + if return_code != 0 { + let message = message.unwrap_or_else(|| "reason unspecified".into()); + log::warn!("Batch {} execution interrupted: {}", batch_id, message); + break; + } + } + } + + async fn exec_stateless(&self, runtime_cmd: &ExecuteCommand) -> crate::Result<()> { + match runtime_cmd.command { + ExeScriptCommand::Sign {} => { + let batch_id = runtime_cmd.batch_id.clone(); + let signature = self.send(SignExeScript { batch_id }).await??; + let stdout = serde_json::to_string(&signature)?; + + runtime_cmd + .tx + .clone() + .send(RuntimeEvent::stdout( + runtime_cmd.batch_id.clone(), + runtime_cmd.idx, + CommandOutput::Bin(stdout.into_bytes()), + )) + .await + .map_err(|e| Error::runtime(format!("Unable to send stdout event: {:?}", e)))?; + } + ExeScriptCommand::Terminate {} => { + log::debug!("Terminating running ExeScripts"); + let exclude_batches = vec![runtime_cmd.batch_id.clone()]; + self.send(Stop { exclude_batches }).await??; + self.send(SetState::from(State::Initialized)).await?; + } + _ => (), + } + Ok(()) + } + + async fn exec_stateful( + &self, + runtime_cmd: ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + let state = self.send(crate::message::GetState {}).await?.0; + let state_pre = match (&state.0, &state.1) { + (_, Some(_)) => { + return Err(StateError::Busy(state).into()); + } + (State::New, _) | (State::Terminated, _) => { + return Err(StateError::InvalidState(state).into()); + } + (State::Initialized, _) => match &runtime_cmd.command { + ExeScriptCommand::Deploy { .. } => { + StatePair(State::Initialized, Some(State::Deployed)) + } + _ => return Err(StateError::InvalidState(state).into()), + }, + (State::Deployed, _) => match &runtime_cmd.command { + ExeScriptCommand::Start { .. } => StatePair(State::Deployed, Some(State::Ready)), + _ => return Err(StateError::InvalidState(state).into()), + }, + (s, _) => match &runtime_cmd.command { + ExeScriptCommand::Deploy { .. } | ExeScriptCommand::Start { .. } => { + return Err(StateError::InvalidState(state).into()); + } + _ => StatePair(*s, Some(*s)), + }, + }; + self.send(SetState::from(state_pre)).await?; + + log::info!("Executing command: {:?}", runtime_cmd.command); + + let result = async { + self.pre_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + let exit_code = runtime.send(runtime_cmd.clone()).await??; + if exit_code != 0 { + return Err(Error::CommandExitCodeError(exit_code)); + } + + self.post_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + Ok(()) + } + .await; + + let state_cur = self.send(crate::message::GetState {}).await?.0; + if state_cur != state_pre { + return Err(StateError::UnexpectedState { + current: state_cur, + expected: state_pre, + } + .into()); + } + + self.send(SetState::from(state_pre.1.unwrap())).await?; + result + } + + async fn pre_runtime( + &self, + runtime_cmd: &ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + match &runtime_cmd.command { + ExeScriptCommand::Transfer { + from, + to, + args, + progress, + } => { + let mut msg = TransferResource { + from: from.clone(), + to: to.clone(), + args: args.clone(), + progress_config: None, + }; + + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + transfer_service.send(msg).await??; + } + ExeScriptCommand::Deploy { + net, + hosts, + progress, + .. + } => { + // TODO: We should pass `task_package` here not in `TransferService` initialization. + let mut msg = DeployImage::default(); + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + + let task_package = transfer_service.send(msg).await??; + runtime + .send(UpdateDeployment { + task_package, + networks: Some(net.clone()), + hosts: Some(hosts.clone()), + ..Default::default() + }) + .await??; + } + _ => (), + } + Ok(()) + } + + async fn post_runtime( + &self, + runtime_cmd: &ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + if let ExeScriptCommand::Deploy { .. } = &runtime_cmd.command { + let mut runtime_mode = RuntimeMode::ProcessPerCommand; + let stdout = self + .send(GetStdOut { + batch_id: runtime_cmd.batch_id.clone(), + idx: runtime_cmd.idx, + }) + .await?; + + if let Some(output) = stdout { + let deployment = deploy::DeployResult::from_bytes(output).map_err(|e| { + log::error!("Deployment failed: {}", e); + Error::CommandError(e.to_string()) + })?; + transfer_service + .send(AddVolumes::new(deployment.vols)) + .await??; + runtime_mode = deployment.start_mode.into(); + } + runtime + .send(UpdateDeployment { + runtime_mode: Some(runtime_mode), + ..Default::default() + }) + .await??; + } + Ok(()) + } +} + +impl Actor for ExeUnit { + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + let rx = self.events.rx.take().unwrap(); + Self::add_stream(rx, ctx); + + let addr = ctx.address(); + if let Some(activity_id) = &self.ctx.activity_id { + let srv_id = activity::exeunit::bus_id(activity_id); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + + #[cfg(feature = "sgx")] + { + actix_rpc::bind::( + &srv_id, + addr.clone().recipient(), + ); + } + #[cfg(not(feature = "sgx"))] + { + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::binds::( + &srv_id, + addr.clone().recipient(), + ); + } + } + + IntervalFunc::new(*DEFAULT_REPORT_INTERVAL, Self::report_usage) + .finish() + .spawn(ctx); + + log::info!("Initializing manifests"); + self.ctx + .supervise + .manifest + .build_validators() + .into_actor(self) + .map(|result, this, ctx| match result { + Ok(validators) => { + this.ctx.supervise.manifest.add_validators(validators); + log::info!("Manifest initialization complete"); + } + Err(e) => { + let err = Error::Other(format!("manifest initialization error: {}", e)); + log::error!("Supervisor is shutting down due to {}", err); + ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); + } + }) + .wait(ctx); + + let addr_ = addr.clone(); + async move { + addr.send(Initialize).await?.map_err(Error::from)?; + addr.send(SetState::from(State::Initialized)).await?; + Ok::<_, Error>(()) + } + .then(|result| async move { + match result { + Ok(_) => log::info!("Supervisor initialized"), + Err(e) => { + let err = Error::Other(format!("initialization error: {}", e)); + log::error!("Supervisor is shutting down due to {}", err); + let _ = addr_.send(Shutdown(ShutdownReason::Error(err))).await; + } + } + }) + .into_actor(self) + .spawn(ctx); + } + + fn stopping(&mut self, _: &mut Self::Context) -> Running { + if self.state.inner.0 == State::Terminated { + return Running::Stop; + } + Running::Continue + } + + fn stopped(&mut self, _: &mut Self::Context) { + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(Ok(())); + } + } +} + +#[derive(derivative::Derivative)] +#[derivative(Debug)] +pub struct ExeUnitContext { + pub supervise: Supervision, + pub activity_id: Option, + pub report_url: Option, + pub agreement: Agreement, + pub work_dir: PathBuf, + pub cache_dir: PathBuf, + pub runtime_args: Vec, + pub acl: Acl, + pub credentials: Option, + #[cfg(feature = "sgx")] + #[derivative(Debug = "ignore")] + pub crypto: crate::crypto::Crypto, +} + +impl ExeUnitContext { + pub fn verify_activity_id(&self, activity_id: &str) -> crate::Result<()> { + match &self.activity_id { + Some(act_id) => match act_id == activity_id { + true => Ok(()), + false => Err(Error::RemoteServiceError(format!( + "Forbidden! Invalid activity id: {}", + activity_id + ))), + }, + None => Ok(()), + } + } +} + +impl From<&ExeUnitContext> for TransferServiceContext { + fn from(val: &ExeUnitContext) -> Self { + TransferServiceContext { + task_package: val.agreement.task_package.clone(), + deploy_retry: None, + cache_dir: val.cache_dir.clone(), + work_dir: val.work_dir.clone(), + transfer_retry: None, + } + } +} + +pub struct Channel { + pub(crate) tx: mpsc::Sender, + rx: Option>, +} + +impl Default for Channel { + fn default() -> Self { + let (tx, rx) = mpsc::channel(8); + Channel { tx, rx: Some(rx) } + } +} + +pub async fn report(url: S, msg: M) -> bool +where + M: RpcMessage + Unpin + 'static, + S: AsRef, +{ + let url = url.as_ref(); + match ya_service_bus::typed::service(url).send(msg).await { + Err(ya_service_bus::Error::Timeout(msg)) => { + log::warn!("Timed out reporting to {}: {}", url, msg); + true + } + Err(e) => { + log::error!("Error reporting to {}: {:?}", url, e); + false + } + Ok(Err(e)) => { + log::error!("Error response while reporting to {}: {:?}", url, e); + false + } + Ok(Ok(_)) => true, + } +} + +async fn report_usage( + report_url: String, + activity_id: String, + exe_unit: Addr>, + metrics: Addr, +) { + match metrics.send(GetMetrics).await { + Ok(resp) => match resp { + Ok(data) => { + let msg = activity::local::SetUsage { + activity_id, + usage: ActivityUsage { + current_usage: Some(data), + timestamp: Utc::now().timestamp(), + }, + timeout: None, + }; + if !report(&report_url, msg).await { + exe_unit.do_send(Shutdown(ShutdownReason::Error(Error::RuntimeError( + format!("Reporting endpoint '{}' is not available", report_url), + )))); + } + } + Err(err) => match err { + Error::UsageLimitExceeded(info) => { + log::warn!("Usage limit exceeded: {}", info); + exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); + } + error => log::warn!("Unable to retrieve metrics: {:?}", error), + }, + }, + Err(e) => log::warn!("Unable to report activity usage: {:?}", e), + } +} + +impl Handler for TransferService { + type Result = ResponseFuture>; + + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + let addr = ctx.address(); + async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() + } +} diff --git a/exe-unit/src/handlers/local.rs b/exe-unit/src/handlers/local.rs index 2a02222ae4..3236695ecf 100644 --- a/exe-unit/src/handlers/local.rs +++ b/exe-unit/src/handlers/local.rs @@ -1,11 +1,13 @@ +use actix::prelude::*; +use futures::FutureExt; + use crate::error::Error; use crate::message::*; use crate::runtime::Runtime; use crate::service::ServiceAddr; use crate::state::State; use crate::{report, ExeUnit}; -use actix::prelude::*; -use futures::FutureExt; + use ya_client_model::activity; use ya_core_model::activity::local::SetState as SetActivityState; diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 3bd6322d36..388d81f06a 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -1,35 +1,27 @@ #[macro_use] extern crate derive_more; -use std::path::PathBuf; -use std::time::Duration; - use actix::prelude::*; -use chrono::Utc; -use futures::channel::{mpsc, oneshot}; -use futures::{FutureExt, SinkExt}; +use anyhow::{bail, Context}; +use futures::channel::oneshot; +use std::convert::TryFrom; +use std::path::PathBuf; +use structopt::clap; -use ya_agreement_utils::agreement::OfferTemplate; -use ya_client_model::activity::{ - activity_state::StatePair, ActivityUsage, CommandOutput, ExeScriptCommand, State, -}; +use ya_client_model::activity::ExeScriptCommand; use ya_core_model::activity; -use ya_core_model::activity::local::Credentials; -use ya_runtime_api::deploy; -use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; -use ya_transfer::transfer::{ - AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, - TransferServiceContext, -}; +use ya_service_bus::RpcEnvelope; +use ya_transfer::transfer::TransferService; +use ya_utils_path::normalize_path; -use crate::acl::Acl; use crate::agreement::Agreement; use crate::error::Error; -use crate::message::*; -use crate::runtime::*; +use crate::manifest::ManifestContext; +use crate::message::{GetState, GetStateResponse, Register}; +use crate::runtime::process::RuntimeProcess; use crate::service::metrics::MetricsService; -use crate::service::{ServiceAddr, ServiceControl}; -use crate::state::{ExeUnitState, StateError, Supervision}; +use crate::service::signal::SignalMonitor; +use crate::state::Supervision; mod acl; pub mod agreement; @@ -50,550 +42,306 @@ pub mod service; pub mod state; mod dns; -pub type Result = std::result::Result; - -lazy_static::lazy_static! { - static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); -} - -pub struct ExeUnit { - ctx: ExeUnitContext, - state: ExeUnitState, - events: Channel, - runtime: Addr, - metrics: Addr, - transfers: Addr, - services: Vec>, - shutdown_tx: Option>>, -} - -impl ExeUnit { - pub fn new( - shutdown_tx: oneshot::Sender>, - ctx: ExeUnitContext, - metrics: Addr, - transfers: Addr, - runtime: Addr, - ) -> Self { - ExeUnit { - ctx, - state: ExeUnitState::default(), - events: Channel::default(), - runtime: runtime.clone(), - metrics: metrics.clone(), - transfers: transfers.clone(), - services: vec![ - Box::new(ServiceAddr::new(metrics)), - Box::new(ServiceAddr::new(transfers)), - Box::new(ServiceAddr::new(runtime)), - ], - shutdown_tx: Some(shutdown_tx), - } - } - - pub fn offer_template(binary: PathBuf, args: Vec) -> Result { - use crate::runtime::process::RuntimeProcess; - - let runtime_template = RuntimeProcess::offer_template(binary, args)?; - let supervisor_template = OfferTemplate::new(serde_json::json!({ - "golem.com.usage.vector": MetricsService::usage_vector(), - "golem.activity.caps.transfer.protocol": TransferService::schemes(), - "golem.activity.caps.transfer.report-progress": true, - })); +mod exe_unit; - Ok(supervisor_template.patch(runtime_template)) - } - - pub fn test(binary: PathBuf, args: Vec) -> Result { - use crate::runtime::process::RuntimeProcess; - RuntimeProcess::test(binary, args) - } +pub use exe_unit::{report, ExeUnit, ExeUnitContext, RuntimeRef}; - fn report_usage(&mut self, context: &mut Context) { - if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { - return; - } - let fut = report_usage( - self.ctx.report_url.clone().unwrap(), - self.ctx.activity_id.clone().unwrap(), - context.address(), - self.metrics.clone(), - ); - context.spawn(fut.into_actor(self)); - } +pub type Result = std::result::Result; - async fn stop_runtime(runtime: Addr, reason: ShutdownReason) { - if let Err(e) = runtime - .send(Shutdown(reason)) - .timeout(Duration::from_secs(5u64)) - .await - { - log::warn!("Unable to stop the runtime: {:?}", e); - } - } +#[derive(structopt::StructOpt, Debug)] +#[structopt(global_setting = clap::AppSettings::ColoredHelp)] +#[structopt(version = ya_compile_time_utils::version_describe!())] +pub struct Cli { + /// Runtime binary path + #[structopt(long, short)] + binary: PathBuf, + #[structopt(flatten)] + supervise: SuperviseCli, + /// Additional runtime arguments + #[structopt( + long, + short, + set = clap::ArgSettings::Global, + number_of_values = 1, + )] + runtime_arg: Vec, + /// Enclave secret key used in secure communication + #[structopt( + long, + env = "EXE_UNIT_SEC_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, + )] + #[allow(dead_code)] + sec_key: Option, + /// Requestor public key used in secure communication + #[structopt( + long, + env = "EXE_UNIT_REQUESTOR_PUB_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, + )] + #[allow(dead_code)] + requestor_pub_key: Option, + #[structopt(subcommand)] + command: Command, } -#[derive(Clone)] -struct RuntimeRef(Addr>); - -impl RuntimeRef { - fn from_ctx(ctx: &Context>) -> Self { - RuntimeRef(ctx.address()) - } +#[derive(structopt::StructOpt, Debug)] +pub struct SuperviseCli { + /// Hardware resources are handled by the runtime + #[structopt( + long = "runtime-managed-hardware", + alias = "cap-handoff", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, + )] + hardware: bool, + /// Images are handled by the runtime + #[structopt( + long = "runtime-managed-image", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, + )] + image: bool, } -impl std::ops::Deref for RuntimeRef { - type Target = Addr>; - - fn deref(&self) -> &Self::Target { - &self.0 - } +#[derive(structopt::StructOpt, Debug)] +#[structopt(global_setting = clap::AppSettings::DeriveDisplayOrder)] +pub enum Command { + /// Execute commands from file + FromFile { + /// ExeUnit daemon GSB URL + #[structopt(long)] + report_url: Option, + /// ExeUnit service ID + #[structopt(long)] + service_id: Option, + /// Command file path + input: PathBuf, + #[structopt(flatten)] + args: RunArgs, + }, + /// Bind to Service Bus + ServiceBus { + /// ExeUnit service ID + service_id: String, + /// ExeUnit daemon GSB URL + report_url: String, + #[structopt(flatten)] + args: RunArgs, + }, + /// Print an offer template in JSON format + OfferTemplate, + /// Run runtime's test command + Test, } -impl RuntimeRef { - async fn exec( - self, - exec: activity::Exec, - runtime: Addr, - transfers: Addr, - mut events: mpsc::Sender, - mut control: oneshot::Receiver<()>, - ) { - let batch_id = exec.batch_id.clone(); - for (idx, command) in exec.exe_script.into_iter().enumerate() { - if let Ok(Some(_)) = control.try_recv() { - log::warn!("Batch {} execution aborted", batch_id); - break; - } - - let runtime_cmd = ExecuteCommand { - batch_id: batch_id.clone(), - command: command.clone(), - tx: events.clone(), - idx, - }; - - let evt = RuntimeEvent::started(batch_id.clone(), idx, command.clone()); - if let Err(e) = events.send(evt).await { - log::error!("Unable to report event: {:?}", e); - } - - let (return_code, message) = match { - if runtime_cmd.stateless() { - self.exec_stateless(&runtime_cmd).await - } else { - self.exec_stateful(runtime_cmd, &runtime, &transfers).await - } - } { - Ok(_) => (0, None), - Err(ref err) => match err { - Error::CommandExitCodeError(c) => (*c, Some(err.to_string())), - _ => (-1, Some(err.to_string())), - }, - }; - - let evt = RuntimeEvent::finished(batch_id.clone(), idx, return_code, message.clone()); - if let Err(e) = events.send(evt).await { - log::error!("Unable to report event: {:?}", e); - } - - if return_code != 0 { - let message = message.unwrap_or_else(|| "reason unspecified".into()); - log::warn!("Batch {} execution interrupted: {}", batch_id, message); - break; - } - } - } - - async fn exec_stateless(&self, runtime_cmd: &ExecuteCommand) -> Result<()> { - match runtime_cmd.command { - ExeScriptCommand::Sign {} => { - let batch_id = runtime_cmd.batch_id.clone(); - let signature = self.send(SignExeScript { batch_id }).await??; - let stdout = serde_json::to_string(&signature)?; - - runtime_cmd - .tx - .clone() - .send(RuntimeEvent::stdout( - runtime_cmd.batch_id.clone(), - runtime_cmd.idx, - CommandOutput::Bin(stdout.into_bytes()), - )) - .await - .map_err(|e| Error::runtime(format!("Unable to send stdout event: {:?}", e)))?; - } - ExeScriptCommand::Terminate {} => { - log::debug!("Terminating running ExeScripts"); - let exclude_batches = vec![runtime_cmd.batch_id.clone()]; - self.send(Stop { exclude_batches }).await??; - self.send(SetState::from(State::Initialized)).await?; - } - _ => (), - } - Ok(()) - } - - async fn exec_stateful( - &self, - runtime_cmd: ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - let state = self.send(GetState {}).await?.0; - let state_pre = match (&state.0, &state.1) { - (_, Some(_)) => { - return Err(StateError::Busy(state).into()); - } - (State::New, _) | (State::Terminated, _) => { - return Err(StateError::InvalidState(state).into()); - } - (State::Initialized, _) => match &runtime_cmd.command { - ExeScriptCommand::Deploy { .. } => { - StatePair(State::Initialized, Some(State::Deployed)) - } - _ => return Err(StateError::InvalidState(state).into()), - }, - (State::Deployed, _) => match &runtime_cmd.command { - ExeScriptCommand::Start { .. } => StatePair(State::Deployed, Some(State::Ready)), - _ => return Err(StateError::InvalidState(state).into()), - }, - (s, _) => match &runtime_cmd.command { - ExeScriptCommand::Deploy { .. } | ExeScriptCommand::Start { .. } => { - return Err(StateError::InvalidState(state).into()); - } - _ => StatePair(*s, Some(*s)), - }, - }; - self.send(SetState::from(state_pre)).await?; - - log::info!("Executing command: {:?}", runtime_cmd.command); - - let result = async { - self.pre_runtime(&runtime_cmd, runtime, transfer_service) - .await?; - - let exit_code = runtime.send(runtime_cmd.clone()).await??; - if exit_code != 0 { - return Err(Error::CommandExitCodeError(exit_code)); - } - - self.post_runtime(&runtime_cmd, runtime, transfer_service) - .await?; - - Ok(()) - } - .await; - - let state_cur = self.send(GetState {}).await?.0; - if state_cur != state_pre { - return Err(StateError::UnexpectedState { - current: state_cur, - expected: state_pre, - } - .into()); - } - - self.send(SetState::from(state_pre.1.unwrap())).await?; - result - } - - async fn pre_runtime( - &self, - runtime_cmd: &ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - match &runtime_cmd.command { - ExeScriptCommand::Transfer { - from, - to, - args, - progress, - } => { - let mut msg = TransferResource { - from: from.clone(), - to: to.clone(), - args: args.clone(), - progress_config: None, - }; - - if let Some(args) = progress { - msg.forward_progress(args, runtime_cmd.progress_sink()) - } - transfer_service.send(msg).await??; - } - ExeScriptCommand::Deploy { - net, - hosts, - progress, - .. - } => { - // TODO: We should pass `task_package` here not in `TransferService` initialization. - let mut msg = DeployImage::default(); - if let Some(args) = progress { - msg.forward_progress(args, runtime_cmd.progress_sink()) - } +#[derive(structopt::StructOpt, Debug)] +pub struct RunArgs { + /// Agreement file path + #[structopt(long, short)] + agreement: PathBuf, + /// Working directory + #[structopt(long, short)] + work_dir: PathBuf, + /// Common cache directory + #[structopt(long, short)] + cache_dir: PathBuf, +} - let task_package = transfer_service.send(msg).await??; - runtime - .send(UpdateDeployment { - task_package, - networks: Some(net.clone()), - hosts: Some(hosts.clone()), - ..Default::default() - }) - .await??; - } - _ => (), +fn create_path(path: &PathBuf) -> anyhow::Result { + if let Err(error) = std::fs::create_dir_all(path) { + match &error.kind() { + std::io::ErrorKind::AlreadyExists => (), + _ => bail!("Can't create directory: {}, {}", path.display(), error), } - Ok(()) } + Ok(normalize_path(path)?) +} - async fn post_runtime( - &self, - runtime_cmd: &ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - if let ExeScriptCommand::Deploy { .. } = &runtime_cmd.command { - let mut runtime_mode = RuntimeMode::ProcessPerCommand; - let stdout = self - .send(GetStdOut { - batch_id: runtime_cmd.batch_id.clone(), - idx: runtime_cmd.idx, - }) - .await?; - - if let Some(output) = stdout { - let deployment = deploy::DeployResult::from_bytes(output).map_err(|e| { - log::error!("Deployment failed: {}", e); - Error::CommandError(e.to_string()) - })?; - transfer_service - .send(AddVolumes::new(deployment.vols)) - .await??; - runtime_mode = deployment.start_mode.into(); - } - runtime - .send(UpdateDeployment { - runtime_mode: Some(runtime_mode), - ..Default::default() - }) - .await??; +#[cfg(feature = "sgx")] +fn init_crypto( + sec_key: Option, + req_key: Option, +) -> anyhow::Result { + let req_key = req_key.ok_or_else(|| anyhow::anyhow!("Missing requestor public key"))?; + match sec_key { + Some(key) => Ok(crate::crypto::Crypto::try_with_keys(key, req_key)?), + None => { + log::info!("Generating a new key pair..."); + Ok(crate::crypto::Crypto::try_new(req_key)?) } - Ok(()) } } -impl Actor for ExeUnit { - type Context = Context; - - fn started(&mut self, ctx: &mut Self::Context) { - let rx = self.events.rx.take().unwrap(); - Self::add_stream(rx, ctx); - - let addr = ctx.address(); - if let Some(activity_id) = &self.ctx.activity_id { - let srv_id = activity::exeunit::bus_id(activity_id); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - - #[cfg(feature = "sgx")] - { - actix_rpc::bind::( - &srv_id, - addr.clone().recipient(), - ); - } - #[cfg(not(feature = "sgx"))] - { - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::binds::( - &srv_id, - addr.clone().recipient(), - ); +async fn send_script( + exe_unit: Addr>, + activity_id: Option, + exe_script: Vec, +) { + use crate::state::{State, StatePair}; + use std::time::Duration; + + let delay = Duration::from_secs_f32(0.5); + loop { + match exe_unit.send(GetState).await { + Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, + Ok(GetStateResponse(StatePair(State::Terminated, _))) + | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) + | Err(_) => { + return log::error!("ExeUnit has terminated"); } + _ => tokio::time::sleep(delay).await, } - - IntervalFunc::new(*DEFAULT_REPORT_INTERVAL, Self::report_usage) - .finish() - .spawn(ctx); - - log::info!("Initializing manifests"); - self.ctx - .supervise - .manifest - .build_validators() - .into_actor(self) - .map(|result, this, ctx| match result { - Ok(validators) => { - this.ctx.supervise.manifest.add_validators(validators); - log::info!("Manifest initialization complete"); - } - Err(e) => { - let err = Error::Other(format!("manifest initialization error: {}", e)); - log::error!("Supervisor is shutting down due to {}", err); - ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); - } - }) - .wait(ctx); - - let addr_ = addr.clone(); - async move { - addr.send(Initialize).await?.map_err(Error::from)?; - addr.send(SetState::from(State::Initialized)).await?; - Ok::<_, Error>(()) - } - .then(|result| async move { - match result { - Ok(_) => log::info!("Supervisor initialized"), - Err(e) => { - let err = Error::Other(format!("initialization error: {}", e)); - log::error!("Supervisor is shutting down due to {}", err); - let _ = addr_.send(Shutdown(ShutdownReason::Error(err))).await; - } - } - }) - .into_actor(self) - .spawn(ctx); } - fn stopping(&mut self, _: &mut Self::Context) -> Running { - if self.state.inner.0 == State::Terminated { - return Running::Stop; - } - Running::Continue - } - - fn stopped(&mut self, _: &mut Self::Context) { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(Ok(())); - } + log::debug!("Executing commands: {:?}", exe_script); + + let msg = activity::Exec { + activity_id: activity_id.unwrap_or_default(), + batch_id: hex::encode(rand::random::<[u8; 16]>()), + exe_script, + timeout: None, + }; + if let Err(e) = exe_unit + .send(RpcEnvelope::with_caller(String::new(), msg)) + .await + { + log::error!("Unable to execute exe script: {:?}", e); } } -#[derive(derivative::Derivative)] -#[derivative(Debug)] -pub struct ExeUnitContext { - pub supervise: Supervision, - pub activity_id: Option, - pub report_url: Option, - pub agreement: Agreement, - pub work_dir: PathBuf, - pub cache_dir: PathBuf, - pub runtime_args: Vec, - pub acl: Acl, - pub credentials: Option, - #[cfg(feature = "sgx")] - #[derivative(Debug = "ignore")] - pub crypto: crypto::Crypto, -} - -impl ExeUnitContext { - pub fn verify_activity_id(&self, activity_id: &str) -> Result<()> { - match &self.activity_id { - Some(act_id) => match act_id == activity_id { - true => Ok(()), - false => Err(Error::RemoteServiceError(format!( - "Forbidden! Invalid activity id: {}", - activity_id - ))), - }, - None => Ok(()), - } +pub async fn run(mut cli: Cli) -> anyhow::Result<()> { + if !cli.binary.exists() { + bail!("Runtime binary does not exist: {}", cli.binary.display()); } -} -impl From<&ExeUnitContext> for TransferServiceContext { - fn from(val: &ExeUnitContext) -> Self { - TransferServiceContext { - task_package: val.agreement.task_package.clone(), - deploy_retry: None, - cache_dir: val.cache_dir.clone(), - work_dir: val.work_dir.clone(), - transfer_retry: None, + let mut commands = None; + let ctx_activity_id; + let ctx_report_url; + + let args = match &cli.command { + Command::FromFile { + args, + service_id, + report_url, + input, + } => { + let contents = std::fs::read_to_string(input).map_err(|e| { + anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) + })?; + let contents = serde_json::from_str(&contents).map_err(|e| { + anyhow::anyhow!( + "Cannot deserialize commands from file {}: {e}", + input.display(), + ) + })?; + ctx_activity_id = service_id.clone(); + ctx_report_url = report_url.clone(); + commands = Some(contents); + args } - } -} - -struct Channel { - tx: mpsc::Sender, - rx: Option>, -} - -impl Default for Channel { - fn default() -> Self { - let (tx, rx) = mpsc::channel(8); - Channel { tx, rx: Some(rx) } - } -} - -pub(crate) async fn report(url: S, msg: M) -> bool -where - M: RpcMessage + Unpin + 'static, - S: AsRef, -{ - let url = url.as_ref(); - match ya_service_bus::typed::service(url).send(msg).await { - Err(ya_service_bus::Error::Timeout(msg)) => { - log::warn!("Timed out reporting to {}: {}", url, msg); - true + Command::ServiceBus { + args, + service_id, + report_url, + } => { + ctx_activity_id = Some(service_id.clone()); + ctx_report_url = Some(report_url.clone()); + args } - Err(e) => { - log::error!("Error reporting to {}: {:?}", url, e); - false + Command::OfferTemplate => { + let args = cli.runtime_arg.clone(); + let offer_template = ExeUnit::::offer_template(cli.binary, args)?; + println!("{}", serde_json::to_string(&offer_template)?); + return Ok(()); } - Ok(Err(e)) => { - log::error!("Error response while reporting to {}: {:?}", url, e); - false + Command::Test => { + let args = cli.runtime_arg.clone(); + let output = ExeUnit::::test(cli.binary, args)?; + println!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + if !output.status.success() { + bail!("Test failed"); + } + return Ok(()); } - Ok(Ok(_)) => true, - } -} + }; -async fn report_usage( - report_url: String, - activity_id: String, - exe_unit: Addr>, - metrics: Addr, -) { - match metrics.send(GetMetrics).await { - Ok(resp) => match resp { - Ok(data) => { - let msg = activity::local::SetUsage { - activity_id, - usage: ActivityUsage { - current_usage: Some(data), - timestamp: Utc::now().timestamp(), - }, - timeout: None, - }; - if !report(&report_url, msg).await { - exe_unit.do_send(Shutdown(ShutdownReason::Error(Error::RuntimeError( - format!("Reporting endpoint '{}' is not available", report_url), - )))); - } - } - Err(err) => match err { - Error::UsageLimitExceeded(info) => { - log::warn!("Usage limit exceeded: {}", info); - exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); - } - error => log::warn!("Unable to retrieve metrics: {:?}", error), - }, + if !args.agreement.exists() { + bail!( + "Agreement file does not exist: {}", + args.agreement.display() + ); + } + let work_dir = create_path(&args.work_dir).map_err(|e| { + anyhow::anyhow!( + "Cannot create the working directory {}: {e}", + args.work_dir.display(), + ) + })?; + let cache_dir = create_path(&args.cache_dir).map_err(|e| { + anyhow::anyhow!( + "Cannot create the cache directory {}: {e}", + args.work_dir.display(), + ) + })?; + let mut agreement = Agreement::try_from(&args.agreement).map_err(|e| { + anyhow::anyhow!( + "Error parsing the agreement from {}: {e}", + args.agreement.display(), + ) + })?; + + log::info!("Attempting to read app manifest .."); + + let manifest_ctx = + ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; + agreement.task_package = manifest_ctx + .payload() + .or_else(|| agreement.task_package.take()); + + log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); + log::info!("User-provided payload: {:?}", agreement.task_package); + + let ctx = ExeUnitContext { + supervise: Supervision { + hardware: cli.supervise.hardware, + image: cli.supervise.image, + manifest: manifest_ctx, }, - Err(e) => log::warn!("Unable to report activity usage: {:?}", e), + activity_id: ctx_activity_id.clone(), + report_url: ctx_report_url, + agreement, + work_dir, + cache_dir, + runtime_args: cli.runtime_arg.clone(), + acl: Default::default(), + credentials: None, + #[cfg(feature = "sgx")] + crypto: init_crypto( + cli.sec_key.replace("".into()), + cli.requestor_pub_key.clone(), + )?, + }; + + log::debug!("CLI args: {:?}", cli); + log::debug!("ExeUnitContext args: {:?}", ctx); + + let (tx, rx) = oneshot::channel(); + + let metrics = MetricsService::try_new(&ctx, Some(10000), ctx.supervise.hardware)?.start(); + let transfers = TransferService::new((&ctx).into()).start(); + let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); + let exe_unit = ExeUnit::new(tx, ctx, metrics, transfers, runtime).start(); + let signals = SignalMonitor::new(exe_unit.clone()).start(); + exe_unit.send(Register(signals)).await?; + + if let Some(exe_script) = commands { + tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); } -} - -impl Handler for TransferService { - type Result = ResponseFuture>; - fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { - let addr = ctx.address(); - async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() - } + rx.await??; + Ok(()) } From 7e6d099e849de2ea70a9fbc0ea5c23a3ac4a1a1c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 17 Jan 2024 13:55:02 +0100 Subject: [PATCH 18/52] Fix mut cli warning for non sgx build --- exe-unit/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 388d81f06a..edda6509ca 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -212,6 +212,8 @@ async fn send_script( } } +// We need this mut for conditional compilation for sgx +#[allow(unused_mut)] pub async fn run(mut cli: Cli) -> anyhow::Result<()> { if !cli.binary.exists() { bail!("Runtime binary does not exist: {}", cli.binary.display()); From 5be9840f73bb81c139e006d016b739b6871ea552 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 18 Jan 2024 17:57:22 +0100 Subject: [PATCH 19/52] Add Exeunit system test using mock runtime (not passing yet) --- Cargo.lock | 45 +++++- Cargo.toml | 1 + exe-unit/Cargo.toml | 5 + exe-unit/components/mock-runtime/Cargo.toml | 30 ++++ .../resources/mock-runtime-descriptor.json | 9 ++ .../mock-runtime/src/lib.rs} | 57 +------ exe-unit/components/mock-runtime/src/main.rs | 42 ++++++ exe-unit/runtime-api/Cargo.toml | 5 - exe-unit/src/exe_unit.rs | 29 +++- exe-unit/src/lib.rs | 140 +++++++++++------- exe-unit/tests/resources/agreement.json | 80 ++++++++++ exe-unit/tests/test_exe_unit_basic.rs | 90 +++++++++++ .../test-framework/framework-basic/src/lib.rs | 2 +- .../framework-basic/src/temp.rs | 23 --- .../framework-basic/src/test_dirs.rs | 90 +++++++++++ 15 files changed, 504 insertions(+), 144 deletions(-) create mode 100644 exe-unit/components/mock-runtime/Cargo.toml create mode 100644 exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json rename exe-unit/{runtime-api/examples/runtime-server-mock.rs => components/mock-runtime/src/lib.rs} (58%) create mode 100644 exe-unit/components/mock-runtime/src/main.rs create mode 100644 exe-unit/tests/resources/agreement.json create mode 100644 exe-unit/tests/test_exe_unit_basic.rs delete mode 100644 test-utils/test-framework/framework-basic/src/temp.rs create mode 100644 test-utils/test-framework/framework-basic/src/test_dirs.rs diff --git a/Cargo.lock b/Cargo.lock index 1bcd56c82c..70158d2cf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8404,12 +8404,14 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "sha3 0.8.2", "shell-words", "signal-hook", "socket2 0.4.10", "structopt", "tempdir", + "test-context", "thiserror", "tokio", "tokio-stream", @@ -8421,9 +8423,11 @@ dependencies = [ "ya-client-model", "ya-compile-time-utils", "ya-core-model", + "ya-framework-basic", "ya-manifest-utils", + "ya-mock-runtime", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-sb-router", "ya-service-bus", "ya-std-utils", @@ -8729,6 +8733,24 @@ dependencies = [ "ya-service-bus", ] +[[package]] +name = "ya-mock-runtime" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes 1.5.0", + "env_logger 0.10.1", + "futures 0.3.30", + "log", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "url", + "ya-runtime-api 0.7.0", +] + [[package]] name = "ya-net" version = "0.3.0" @@ -9106,6 +9128,25 @@ dependencies = [ "derive_more", ] +[[package]] +name = "ya-runtime-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0db25811f107d62be6c6ac7444d9c6c3e39714b6f76d72798b66ecce47506f" +dependencies = [ + "anyhow", + "bytes 1.5.0", + "futures 0.3.30", + "log", + "prost 0.10.4", + "prost-build 0.10.4", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "ya-runtime-api" version = "0.7.1" @@ -9369,7 +9410,7 @@ dependencies = [ "ya-core-model", "ya-exe-unit", "ya-framework-basic", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-service-bus", "ya-utils-futures", "ya-utils-path", diff --git a/Cargo.toml b/Cargo.toml index 63d9331e2f..99643039c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,6 +206,7 @@ members = [ "exe-unit/runtime-api", "exe-unit/tokio-process-ns", "exe-unit/components/transfer", + "exe-unit/components/mock-runtime", "golem_cli", "utils/actix_utils", "utils/agreement-utils", diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index af1f4357ae..27ceca7cf7 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -109,6 +109,11 @@ actix-files = "0.6" actix-web = "4" env_logger = "0.7" rustyline = "7.0.0" +serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} sha3 = "0.8.2" shell-words = "1.0.0" tempdir = "0.3.7" +test-context = "0.1.4" + +ya-framework-basic = { version = "0.1" } +ya-mock-runtime = { path = "components/mock-runtime"} diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml new file mode 100644 index 0000000000..9d6b190227 --- /dev/null +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "ya-mock-runtime" +version = "0.1.0" +authors = ["Golem Factory "] +edition = "2021" +description = "Mock runtime for testing purposes and set of libraries for testing ExeUnits in tests." + + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "ya-mock-runtime" +path = "src/main.rs" + + +[dependencies] +anyhow = "1.0.31" +bytes = "1.0" +futures = { version = "0.3" } +log = "0.4" +env_logger = "0.10" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "1.0" +tokio = { version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"] } +tokio-util = { version = "0.7", features = ["codec"] } +url = "2.3" + +ya-runtime-api = "0.7" diff --git a/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json b/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json new file mode 100644 index 0000000000..93f43f2d54 --- /dev/null +++ b/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json @@ -0,0 +1,9 @@ +[ + { + "name": "ya-mock-runtime", + "version": "0.1.0", + "supervisor-path": "exe-unit", + "runtime-path": "ya-mock-runtime", + "description": "Mock runtime for testing purposes" + } +] \ No newline at end of file diff --git a/exe-unit/runtime-api/examples/runtime-server-mock.rs b/exe-unit/components/mock-runtime/src/lib.rs similarity index 58% rename from exe-unit/runtime-api/examples/runtime-server-mock.rs rename to exe-unit/components/mock-runtime/src/lib.rs index 496b741a17..c4e3b352c4 100644 --- a/exe-unit/runtime-api/examples/runtime-server-mock.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -2,19 +2,16 @@ use futures::future::BoxFuture; use futures::prelude::*; use futures::FutureExt; use std::clone::Clone; -use std::env; use std::sync::{Arc, Mutex}; use std::time::Duration; use ya_runtime_api::server::*; -// server - -struct RuntimeMock +pub struct RuntimeMock where H: RuntimeHandler, { - handler: H, + pub handler: H, } impl RuntimeService for RuntimeMock { @@ -61,14 +58,15 @@ impl RuntimeService for RuntimeMock { // client // holds last received status -struct EventMock(Arc>); +#[derive(Clone)] +pub struct EventMock(Arc>); impl EventMock { - fn new() -> Self { + pub fn new() -> Self { Self(Arc::new(Mutex::new(Default::default()))) } - fn get_last_status(&self) -> ProcessStatus { + pub fn get_last_status(&self) -> ProcessStatus { self.0.lock().unwrap().clone() } } @@ -84,46 +82,3 @@ impl RuntimeHandler for EventMock { future::ready(()).boxed() } } - -impl Clone for EventMock { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - if env::var("RUST_LOG").is_err() { - env::set_var("RUST_LOG", "debug") - } - env_logger::init(); - if env::var("X_SERVER").is_ok() { - run(|event_emitter| RuntimeMock { - handler: event_emitter, - }) - .await - } else { - use tokio::process::Command; - let exe = env::current_exe().unwrap(); - - let mut cmd = Command::new(exe); - cmd.env("X_SERVER", "1"); - let events = EventMock::new(); - let c = spawn(cmd, events.clone()).await?; - log::debug!("hello_result={:?}", c.hello("0.0.0x").await); - let run = RunProcess { - bin: "sleep".to_owned(), - args: vec!["10".to_owned()], - ..Default::default() - }; - let sleep_1 = c.run_process(run.clone()); - let sleep_2 = c.run_process(run.clone()); - let sleep_3 = c.run_process(run); - log::info!("start sleep1"); - log::info!("sleep1={:?}", sleep_1.await); - log::info!("start sleep2 sleep3"); - log::info!("sleep23={:?}", future::join(sleep_2, sleep_3).await); - log::info!("last status: {:?}", events.get_last_status()); - } - Ok(()) -} diff --git a/exe-unit/components/mock-runtime/src/main.rs b/exe-unit/components/mock-runtime/src/main.rs new file mode 100644 index 0000000000..509263c203 --- /dev/null +++ b/exe-unit/components/mock-runtime/src/main.rs @@ -0,0 +1,42 @@ +use futures::future; +use std::env; + +use ya_mock_runtime::{EventMock, RuntimeMock}; +use ya_runtime_api::server::{run, spawn, RunProcess, RuntimeService}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "debug") + } + env_logger::init(); + if env::var("X_SERVER").is_ok() { + run(|event_emitter| RuntimeMock { + handler: event_emitter, + }) + .await + } else { + use tokio::process::Command; + let exe = env::current_exe().unwrap(); + + let mut cmd = Command::new(exe); + cmd.env("X_SERVER", "1"); + let events = EventMock::new(); + let c = spawn(cmd, events.clone()).await?; + log::debug!("hello_result={:?}", c.hello("0.0.0x").await); + let run = RunProcess { + bin: "sleep".to_owned(), + args: vec!["10".to_owned()], + ..Default::default() + }; + let sleep_1 = c.run_process(run.clone()); + let sleep_2 = c.run_process(run.clone()); + let sleep_3 = c.run_process(run); + log::info!("start sleep1"); + log::info!("sleep1={:?}", sleep_1.await); + log::info!("start sleep2 sleep3"); + log::info!("sleep23={:?}", future::join(sleep_2, sleep_3).await); + log::info!("last status: {:?}", events.get_last_status()); + } + Ok(()) +} diff --git a/exe-unit/runtime-api/Cargo.toml b/exe-unit/runtime-api/Cargo.toml index 9de7a43bfc..316a74f2fb 100644 --- a/exe-unit/runtime-api/Cargo.toml +++ b/exe-unit/runtime-api/Cargo.toml @@ -8,12 +8,7 @@ license = "GPL-3.0" homepage = "https://github.com/golemfactory/yagna/tree/master/exe-unit/runtime-api" repository = "https://github.com/golemfactory/yagna" -[[example]] -name = "runtime-server-mock" -required-features = ["server"] - [features] -default = ['server'] codec = [] server = ['prost', 'futures', 'tokio', 'tokio-util'] diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index b666fdd259..edaa86a8e7 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -1,13 +1,14 @@ use actix::dev::IntervalFunc; use actix::{ Actor, ActorFutureExt, ActorStreamExt, Addr, AsyncContext, Context, ContextFutureSpawner, - Handler, ResponseFuture, Running, StreamHandler, WrapFuture, + Handler, Message, ResponseFuture, Running, StreamHandler, WrapFuture, }; use chrono::Utc; use futures::channel::{mpsc, oneshot}; use futures::{FutureExt, SinkExt}; use std::path::PathBuf; use std::time::Duration; +use tokio::sync::broadcast; use ya_agreement_utils::OfferTemplate; use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; @@ -36,6 +37,10 @@ lazy_static::lazy_static! { static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); } +#[derive(Clone, Debug, Default, Message)] +#[rtype(result = "()")] +pub struct AwaitFinish {} + pub struct ExeUnit { pub(crate) ctx: ExeUnitContext, pub(crate) state: ExeUnitState, @@ -44,17 +49,17 @@ pub struct ExeUnit { pub(crate) metrics: Addr, pub(crate) transfers: Addr, pub(crate) services: Vec>, - pub(crate) shutdown_tx: Option>>, + pub(crate) shutdown_tx: broadcast::Sender<()>, } impl ExeUnit { pub fn new( - shutdown_tx: oneshot::Sender>, ctx: ExeUnitContext, metrics: Addr, transfers: Addr, runtime: Addr, ) -> Self { + let (shutdown_tx, _) = broadcast::channel(1); ExeUnit { ctx, state: ExeUnitState::default(), @@ -67,7 +72,7 @@ impl ExeUnit { Box::new(ServiceAddr::new(transfers)), Box::new(ServiceAddr::new(runtime)), ], - shutdown_tx: Some(shutdown_tx), + shutdown_tx, } } @@ -449,9 +454,7 @@ impl Actor for ExeUnit { } fn stopped(&mut self, _: &mut Self::Context) { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(Ok(())); - } + self.shutdown_tx.send(()).ok(); } } @@ -569,6 +572,18 @@ async fn report_usage( } } +impl Handler for ExeUnit { + type Result = ResponseFuture<()>; + + fn handle(&mut self, _msg: AwaitFinish, _: &mut Self::Context) -> Self::Result { + let mut rx = self.shutdown_tx.subscribe(); + async move { + rx.recv().await.ok(); + } + .boxed_local() + } +} + impl Handler for TransferService { type Result = ResponseFuture>; diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index edda6509ca..f98231aaac 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -3,7 +3,6 @@ extern crate derive_more; use actix::prelude::*; use anyhow::{bail, Context}; -use futures::channel::oneshot; use std::convert::TryFrom; use std::path::PathBuf; use structopt::clap; @@ -44,7 +43,7 @@ pub mod state; mod dns; mod exe_unit; -pub use exe_unit::{report, ExeUnit, ExeUnitContext, RuntimeRef}; +pub use exe_unit::{report, AwaitFinish, ExeUnit, ExeUnitContext, RuntimeRef}; pub type Result = std::result::Result; @@ -54,56 +53,56 @@ pub type Result = std::result::Result; pub struct Cli { /// Runtime binary path #[structopt(long, short)] - binary: PathBuf, + pub binary: PathBuf, #[structopt(flatten)] - supervise: SuperviseCli, + pub supervise: SuperviseCli, /// Additional runtime arguments #[structopt( - long, - short, - set = clap::ArgSettings::Global, - number_of_values = 1, + long, + short, + set = clap::ArgSettings::Global, + number_of_values = 1, )] - runtime_arg: Vec, + pub runtime_arg: Vec, /// Enclave secret key used in secure communication #[structopt( - long, - env = "EXE_UNIT_SEC_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, + long, + env = "EXE_UNIT_SEC_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, )] #[allow(dead_code)] - sec_key: Option, + pub sec_key: Option, /// Requestor public key used in secure communication #[structopt( - long, - env = "EXE_UNIT_REQUESTOR_PUB_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, + long, + env = "EXE_UNIT_REQUESTOR_PUB_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, )] #[allow(dead_code)] - requestor_pub_key: Option, + pub requestor_pub_key: Option, #[structopt(subcommand)] - command: Command, + pub command: Command, } -#[derive(structopt::StructOpt, Debug)] +#[derive(structopt::StructOpt, Debug, Clone)] pub struct SuperviseCli { /// Hardware resources are handled by the runtime #[structopt( - long = "runtime-managed-hardware", - alias = "cap-handoff", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, + long = "runtime-managed-hardware", + alias = "cap-handoff", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, )] - hardware: bool, + pub hardware: bool, /// Images are handled by the runtime #[structopt( - long = "runtime-managed-image", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, + long = "runtime-managed-image", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, )] - image: bool, + pub image: bool, } #[derive(structopt::StructOpt, Debug)] @@ -137,17 +136,17 @@ pub enum Command { Test, } -#[derive(structopt::StructOpt, Debug)] +#[derive(structopt::StructOpt, Debug, Clone)] pub struct RunArgs { /// Agreement file path #[structopt(long, short)] - agreement: PathBuf, + pub agreement: PathBuf, /// Working directory #[structopt(long, short)] - work_dir: PathBuf, + pub work_dir: PathBuf, /// Common cache directory #[structopt(long, short)] - cache_dir: PathBuf, + pub cache_dir: PathBuf, } fn create_path(path: &PathBuf) -> anyhow::Result { @@ -175,7 +174,7 @@ fn init_crypto( } } -async fn send_script( +pub async fn send_script( exe_unit: Addr>, activity_id: Option, exe_script: Vec, @@ -215,6 +214,8 @@ async fn send_script( // We need this mut for conditional compilation for sgx #[allow(unused_mut)] pub async fn run(mut cli: Cli) -> anyhow::Result<()> { + log::debug!("CLI args: {:?}", cli); + if !cli.binary.exists() { bail!("Runtime binary does not exist: {}", cli.binary.display()); } @@ -223,14 +224,14 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { let ctx_activity_id; let ctx_report_url; - let args = match &cli.command { + let args = match cli.command { Command::FromFile { args, service_id, report_url, input, } => { - let contents = std::fs::read_to_string(input).map_err(|e| { + let contents = std::fs::read_to_string(&input).map_err(|e| { anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) })?; let contents = serde_json::from_str(&contents).map_err(|e| { @@ -271,6 +272,43 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { } }; + let exe_unit = exe_unit(ExeUnitConfig { + report_url: ctx_report_url, + service_id: ctx_activity_id.clone(), + runtime_args: cli.runtime_arg, + binary: cli.binary, + supervise: cli.supervise, + sec_key: cli.sec_key, + args, + requestor_pub_key: cli.requestor_pub_key, + }) + .await?; + + if let Some(exe_script) = commands { + tokio::task::spawn(send_script(exe_unit.clone(), ctx_activity_id, exe_script)); + } + + exe_unit.send(AwaitFinish {}).await?; + Ok(()) +} + +#[derive(Debug, Clone)] +pub struct ExeUnitConfig { + pub args: RunArgs, + pub binary: PathBuf, + pub runtime_args: Vec, + pub service_id: Option, + pub report_url: Option, + pub supervise: SuperviseCli, + + #[allow(dead_code)] + pub sec_key: Option, + #[allow(dead_code)] + pub requestor_pub_key: Option, +} + +pub async fn exe_unit(config: ExeUnitConfig) -> anyhow::Result>> { + let args = config.args; if !args.agreement.exists() { bail!( "Agreement file does not exist: {}", @@ -309,41 +347,33 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { let ctx = ExeUnitContext { supervise: Supervision { - hardware: cli.supervise.hardware, - image: cli.supervise.image, + hardware: config.supervise.hardware, + image: config.supervise.image, manifest: manifest_ctx, }, - activity_id: ctx_activity_id.clone(), - report_url: ctx_report_url, + activity_id: config.service_id.clone(), + report_url: config.report_url, agreement, work_dir, cache_dir, - runtime_args: cli.runtime_arg.clone(), + runtime_args: config.runtime_args, acl: Default::default(), credentials: None, #[cfg(feature = "sgx")] crypto: init_crypto( - cli.sec_key.replace("".into()), - cli.requestor_pub_key.clone(), + config.sec_key.replace("".into()), + config.requestor_pub_key.clone(), )?, }; - log::debug!("CLI args: {:?}", cli); log::debug!("ExeUnitContext args: {:?}", ctx); - let (tx, rx) = oneshot::channel(); - let metrics = MetricsService::try_new(&ctx, Some(10000), ctx.supervise.hardware)?.start(); let transfers = TransferService::new((&ctx).into()).start(); - let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); - let exe_unit = ExeUnit::new(tx, ctx, metrics, transfers, runtime).start(); + let runtime = RuntimeProcess::new(&ctx, config.binary).start(); + let exe_unit = ExeUnit::new(ctx, metrics, transfers, runtime).start(); let signals = SignalMonitor::new(exe_unit.clone()).start(); exe_unit.send(Register(signals)).await?; - if let Some(exe_script) = commands { - tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); - } - - rx.await??; - Ok(()) + Ok(exe_unit) } diff --git a/exe-unit/tests/resources/agreement.json b/exe-unit/tests/resources/agreement.json new file mode 100644 index 0000000000..9dbbccfea4 --- /dev/null +++ b/exe-unit/tests/resources/agreement.json @@ -0,0 +1,80 @@ +{ + "agreementId": "0ec929f5acc8f98a47ab72d61a2c2f343d45d8438d3aa4ccdc84e717c219e185", + "proposedSignature": "NoSignature", + "state": "Pending", + "timestamp": "2022-05-22T10:41:42.564784259Z", + "validTo": "2022-05-22T11:41:42.562457Z", + + "offer": { + "properties": { + "golem.activity.caps.transfer.protocol": [ + "gftp", + "https", + "http" + ], + "golem.activity.caps.transfer.report-progress": true, + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.payment.platform.erc20-goerli-tglm.address": "0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a", + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [ + 0.001388888888888889, + 0.0002777777777777778, + 0.0 + ], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": [ + "golem.usage.cpu_sec", + "golem.usage.duration_sec" + ], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", + "golem.inf.cpu.capabilities": [], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 11, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 28.0, + "golem.inf.storage.gib": 10.188121795654297, + "golem.node.id.name": "mock-provider", + "golem.node.net.is-public": true, + "golem.runtime.capabilities": [], + "golem.runtime.name": "ya-mock-runtime", + "golem.runtime.version": "0.1.0", + "golem.srv.caps.multi-activity": true, + "golem.srv.caps.payload-manifest": true + }, + "constraints": "(&\n (golem.srv.comp.expiration>1705586871777)\n)", + "offerId": "afce49b1ea5b45db91bdd6e5481479f9-9095fca9dea0a91ce95cf994125b33cdd838fcc963a1106f2be9e4b5b65a52f0", + "providerId": "0x86a269498fb5270f20bdc6fdcf6039122b0d3b23", + "timestamp": "2022-05-22T10:41:42.564784259Z" + }, + + "demand": { + "constraints": "(&(golem.com.payment.platform.erc20-goerli-tglm.address=*)\n\t(golem.com.pricing.model=linear)\n\t(&(golem.inf.mem.gib>=0.5)\n\t(golem.inf.storage.gib>=2.0)\n\t(golem.inf.cpu.threads>=1)\n\t(golem.runtime.name=ya-mock-runtime)))", + "demandId": "773035fc685c46da8e61473ac2a2568e-3f3eb86d6ef9a01708d0f57d0b19cc69fd74422150c120e33cc1b5f4a1a12b96", + "properties": { + "golem": { + "com.payment": { + "chosen-platform": "erc20-goerli-tglm", + "debit-notes.accept-timeout?": 240, + "platform.erc20-goerli-tglm.address": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7" + }, + "node": {}, + "srv": { + "caps.multi-activity": true, + "comp": { + "expiration": 1653216996555, + "task_package": "http://127.0.0.1:8001/rnd", + "vm": { + "package_format": "gvmkit-squash" + } + } + } + } + }, + "requestorId": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7", + "timestamp": "2022-05-22T10:41:42.564784259Z" + } +} diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs new file mode 100644 index 0000000000..bd2f4d57fd --- /dev/null +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -0,0 +1,90 @@ +use actix::Addr; +use test_context::test_context; +use ya_client_model::activity::ExeScriptCommand; +use ya_exe_unit::message::{Shutdown, ShutdownReason}; +use ya_exe_unit::runtime::process::RuntimeProcess; +use ya_exe_unit::{ + exe_unit, send_script, AwaitFinish, ExeUnit, ExeUnitConfig, RunArgs, SuperviseCli, +}; +use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; +use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::{resource, temp_dir}; + +#[derive(Debug, Clone)] +pub struct ExeUnitHandle(pub Addr>); + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); + self.0.send(AwaitFinish {}).await.ok(); + } +} + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("exe-unit-start-terminate")?; + let temp_dir = dir.path(); + + generate_file_with_hash(temp_dir, "rnd", 4096_usize, 10); + start_http(ctx, temp_dir.to_path_buf()) + .await + .expect("unable to start http servers"); + + let config = ExeUnitConfig { + args: RunArgs { + agreement: resource!("agreement.json"), + cache_dir: temp_dir.join("cache"), + work_dir: temp_dir.join("work"), + }, + binary: cargo_binary("ya-mock-runtime").unwrap(), + runtime_args: vec![], + supervise: SuperviseCli { + hardware: false, + image: false, + }, + sec_key: None, + requestor_pub_key: None, + service_id: None, + report_url: None, + }; + + let exe = exe_unit(config).await.unwrap(); + ctx.register(ExeUnitHandle(exe.clone())); + + log::info!("Sending [deploy, start] batch for execution."); + + send_script( + exe.clone(), + None, + vec![ + ExeScriptCommand::Deploy { + net: vec![], + progress: None, + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }, + ExeScriptCommand::Start { args: vec![] }, + ], + ) + .await; + + log::info!("Sending shutdown request."); + + exe.send(Shutdown(ShutdownReason::Finished)) + .await + .unwrap() + .unwrap(); + exe.send(AwaitFinish {}).await?; + + Ok(()) +} diff --git a/test-utils/test-framework/framework-basic/src/lib.rs b/test-utils/test-framework/framework-basic/src/lib.rs index 40350bad24..b3bb3916d9 100644 --- a/test-utils/test-framework/framework-basic/src/lib.rs +++ b/test-utils/test-framework/framework-basic/src/lib.rs @@ -3,4 +3,4 @@ pub mod file; pub mod hash; pub mod log; pub mod server_external; -pub mod temp; +pub mod test_dirs; diff --git a/test-utils/test-framework/framework-basic/src/temp.rs b/test-utils/test-framework/framework-basic/src/temp.rs deleted file mode 100644 index aea081f7ef..0000000000 --- a/test-utils/test-framework/framework-basic/src/temp.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::fs; -use tempdir::TempDir; - -pub mod macros { - #[macro_export] - macro_rules! temp_dir { - ($prefix:literal) => { - // CARGO_TARGET_TMPDIR is available in compile time only in binary modules, so we can't - // use it in this library. Thanks to using macro it will be resolved in final code not here - // and it will work. - ya_framework_basic::temp::temp_dir_(env!("CARGO_TARGET_TMPDIR"), $prefix) - }; - } -} - -pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { - fs::create_dir_all(base_dir)?; - let dir = TempDir::new_in(base_dir, prefix)?; - let temp_dir = dir.path(); - fs::create_dir_all(temp_dir)?; - - Ok(dir) -} diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs new file mode 100644 index 0000000000..bf0e1c892b --- /dev/null +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -0,0 +1,90 @@ +use anyhow::{anyhow, bail}; +use std::fs; +use std::path::PathBuf; +use tempdir::TempDir; + +pub mod macros { + /// Creates temporary directory in cargo target directory. + #[macro_export] + macro_rules! temp_dir { + ($prefix:literal) => { + // CARGO_TARGET_TMPDIR is available in compile time only in binary modules, so we can't + // use it in this library. Thanks to using macro it will be resolved in final code not here + // and it will work. + ya_framework_basic::test_dirs::temp_dir_(env!("CARGO_TARGET_TMPDIR"), $prefix) + }; + } + + /// Returns resource from `resources` directory in tests folder. + #[macro_export] + macro_rules! resource { + ($name:literal) => { + // CARGO_MANIFEST_DIR is available in compile time only in binary modules, so we can't + // use it in this library. Thanks to using macro it will be resolved in final code not here + // and it will work. + ya_framework_basic::test_dirs::resource_(env!("CARGO_MANIFEST_DIR"), $name) + }; + } +} + +pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { + fs::create_dir_all(base_dir)?; + let dir = TempDir::new_in(base_dir, prefix)?; + let temp_dir = dir.path(); + fs::create_dir_all(temp_dir)?; + + Ok(dir) +} + +#[cfg(debug_assertions)] +pub fn is_debug() -> bool { + true +} + +#[cfg(not(debug_assertions))] +pub fn is_debug() -> bool { + false +} + +#[cfg(target_family = "windows")] +pub fn extension() -> String { + ".exe".to_string() +} + +#[cfg(not(target_family = "windows"))] +pub fn extension() -> String { + "".to_string() +} + +/// Returns absolute path to cargo project binary. +pub fn cargo_binary(bin_name: &str) -> anyhow::Result { + let current = std::env::current_exe() + .map_err(|e| anyhow!("Failed to get path to current binary. {e}"))? + .parent() + .and_then(|path| path.parent()) + .ok_or(anyhow!("No parent dir for current binary."))? + .to_path_buf(); + let bin_name = format!("{bin_name}{}", extension()); + let bin_path = current.join(&bin_name); + if !bin_path.exists() { + bail!( + "Path doesn't exist: {}, when looking for binary: {}", + bin_path.display(), + bin_name + ); + } + + if !bin_path.is_file() { + bail!("Expected {} to be binary file.", bin_path.display()); + } + + Ok(bin_path) +} + +/// Returns resource from `resources` directory in tests. +pub fn resource_(base_dir: &str, name: &str) -> PathBuf { + PathBuf::from(base_dir) + .join("tests") + .join("resources") + .join(name) +} From fbba58008981af4beb5316c9c2eebb06b72af357 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 19 Jan 2024 13:28:21 +0100 Subject: [PATCH 20/52] Better mechanism to wait for ExeUnit finish --- exe-unit/src/exe_unit.rs | 17 +++++++---------- exe-unit/src/lib.rs | 4 ++-- exe-unit/tests/test_exe_unit_basic.rs | 11 ++++++++--- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index edaa86a8e7..aa4a426038 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -32,14 +32,15 @@ use crate::runtime::{Runtime, RuntimeMode}; use crate::service::metrics::MetricsService; use crate::service::{ServiceAddr, ServiceControl}; use crate::state::{ExeUnitState, StateError, Supervision}; +use crate::Result; lazy_static::lazy_static! { static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); } #[derive(Clone, Debug, Default, Message)] -#[rtype(result = "()")] -pub struct AwaitFinish {} +#[rtype(result = "Result>")] +pub struct FinishNotifier {} pub struct ExeUnit { pub(crate) ctx: ExeUnitContext, @@ -572,15 +573,11 @@ async fn report_usage( } } -impl Handler for ExeUnit { - type Result = ResponseFuture<()>; +impl Handler for ExeUnit { + type Result = Result>; - fn handle(&mut self, _msg: AwaitFinish, _: &mut Self::Context) -> Self::Result { - let mut rx = self.shutdown_tx.subscribe(); - async move { - rx.recv().await.ok(); - } - .boxed_local() + fn handle(&mut self, _msg: FinishNotifier, _: &mut Self::Context) -> Self::Result { + Ok(self.shutdown_tx.subscribe()) } } diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index f98231aaac..8b310d7d21 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -43,7 +43,7 @@ pub mod state; mod dns; mod exe_unit; -pub use exe_unit::{report, AwaitFinish, ExeUnit, ExeUnitContext, RuntimeRef}; +pub use exe_unit::{report, ExeUnit, ExeUnitContext, FinishNotifier, RuntimeRef}; pub type Result = std::result::Result; @@ -288,7 +288,7 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { tokio::task::spawn(send_script(exe_unit.clone(), ctx_activity_id, exe_script)); } - exe_unit.send(AwaitFinish {}).await?; + exe_unit.send(FinishNotifier {}).await??.recv().await?; Ok(()) } diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index bd2f4d57fd..93cec3cccb 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -4,7 +4,7 @@ use ya_client_model::activity::ExeScriptCommand; use ya_exe_unit::message::{Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; use ya_exe_unit::{ - exe_unit, send_script, AwaitFinish, ExeUnit, ExeUnitConfig, RunArgs, SuperviseCli, + exe_unit, send_script, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli, }; use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; use ya_framework_basic::file::generate_file_with_hash; @@ -19,8 +19,11 @@ pub struct ExeUnitHandle(pub Addr>); #[async_trait::async_trait] impl AsyncDroppable for ExeUnitHandle { async fn async_drop(&self) { + let finish = self.0.send(FinishNotifier {}).await; self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); - self.0.send(AwaitFinish {}).await.ok(); + if let Ok(Ok(mut finish)) = finish { + finish.recv().await.ok(); + } } } @@ -57,6 +60,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow }; let exe = exe_unit(config).await.unwrap(); + let mut finish = exe.send(FinishNotifier {}).await??; ctx.register(ExeUnitHandle(exe.clone())); log::info!("Sending [deploy, start] batch for execution."); @@ -84,7 +88,8 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow .await .unwrap() .unwrap(); - exe.send(AwaitFinish {}).await?; + + finish.recv().await.unwrap(); Ok(()) } From f82c3edb00bf3e0f6555ea0c9ac6b3940b51a062 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 19 Jan 2024 17:31:53 +0100 Subject: [PATCH 21/52] Move ExeUnit test utils to mock-runtime --- Cargo.lock | 4 + exe-unit/components/mock-runtime/Cargo.toml | 7 ++ exe-unit/components/mock-runtime/src/lib.rs | 2 + .../components/mock-runtime/src/testing.rs | 3 + .../mock-runtime/src/testing/exe_unit_ext.rs | 75 +++++++++++++++++++ .../components/transfer/tests/test_deploy.rs | 6 +- .../transfer/tests/test_transfer_resume.rs | 4 +- .../transfer/tests/test_transfer_service.rs | 6 +- exe-unit/src/lib.rs | 9 ++- exe-unit/tests/resources/agreement.json | 2 +- exe-unit/tests/test_exe_unit_basic.rs | 59 ++++----------- .../framework-basic/src/file.rs | 57 ++++++++++++-- 12 files changed, 172 insertions(+), 62 deletions(-) create mode 100644 exe-unit/components/mock-runtime/src/testing.rs create mode 100644 exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs diff --git a/Cargo.lock b/Cargo.lock index 70158d2cf1..516887adfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8737,7 +8737,9 @@ dependencies = [ name = "ya-mock-runtime" version = "0.1.0" dependencies = [ + "actix", "anyhow", + "async-trait", "bytes 1.5.0", "env_logger 0.10.1", "futures 0.3.30", @@ -8748,6 +8750,8 @@ dependencies = [ "tokio", "tokio-util", "url", + "ya-exe-unit", + "ya-framework-basic", "ya-runtime-api 0.7.0", ] diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 9d6b190227..c73f95478f 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -28,3 +28,10 @@ tokio-util = { version = "0.7", features = ["codec"] } url = "2.3" ya-runtime-api = "0.7" + +# Dependancies for ExeUnit testing utils +ya-exe-unit = "0.4" +ya-framework-basic = { version = "0.1" } + +async-trait = "0.1.77" +actix = { version = "0.13", default-features = false } diff --git a/exe-unit/components/mock-runtime/src/lib.rs b/exe-unit/components/mock-runtime/src/lib.rs index c4e3b352c4..c818b73da4 100644 --- a/exe-unit/components/mock-runtime/src/lib.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -1,3 +1,5 @@ +pub mod testing; + use futures::future::BoxFuture; use futures::prelude::*; use futures::FutureExt; diff --git a/exe-unit/components/mock-runtime/src/testing.rs b/exe-unit/components/mock-runtime/src/testing.rs new file mode 100644 index 0000000000..cb349f200a --- /dev/null +++ b/exe-unit/components/mock-runtime/src/testing.rs @@ -0,0 +1,3 @@ +mod exe_unit_ext; + +pub use exe_unit_ext::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs new file mode 100644 index 0000000000..6dee0f450c --- /dev/null +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -0,0 +1,75 @@ +use actix::Addr; +use anyhow::bail; +use std::path::Path; +use std::time::Duration; + +use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; +use ya_exe_unit::runtime::process::RuntimeProcess; +use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; +use ya_framework_basic::async_drop::AsyncDroppable; +use ya_framework_basic::test_dirs::cargo_binary; + +#[async_trait::async_trait] +pub trait ExeUnitExt { + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; +} + +#[derive(Debug, Clone)] +pub struct ExeUnitHandle(pub Addr>); + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + let finish = self.0.send(FinishNotifier {}).await; + self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); + if let Ok(Ok(mut finish)) = finish { + finish.recv().await.ok(); + } + } +} + +pub fn exe_unit_config(temp_dir: &Path, agreement_path: &Path, binary: &str) -> ExeUnitConfig { + ExeUnitConfig { + args: RunArgs { + agreement: agreement_path.to_path_buf(), + cache_dir: temp_dir.join("cache"), + work_dir: temp_dir.join("work"), + }, + binary: cargo_binary(binary).unwrap(), + runtime_args: vec![], + supervise: SuperviseCli { + hardware: false, + image: false, + }, + sec_key: None, + requestor_pub_key: None, + service_id: None, + report_url: None, + } +} + +#[async_trait::async_trait] +impl ExeUnitExt for Addr> { + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { + let delay = Duration::from_secs_f32(0.5); + loop { + match self + .send(GetBatchResults { + batch_id: batch_id.to_string(), + idx: None, + }) + .await + { + Ok(results) => { + if let Some(last) = results.0.last() { + if last.is_batch_finished { + return Ok(()); + } + } + } + Err(e) => bail!("Waiting for batch: {batch_id}. Error: {e}"), + } + tokio::time::sleep(delay).await; + } + } +} diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index ae32972974..2f84fb4d24 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -10,7 +10,7 @@ use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::CommandProgress; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; @@ -36,7 +36,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re std::fs::create_dir_all(dir)?; } - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); @@ -99,7 +99,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let chunk_size = 4096_usize; let chunk_count = 1024 * 1; let file_size = (chunk_size * chunk_count) as u64; - let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + let hash = generate_random_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 7253453199..03920246b6 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -9,7 +9,7 @@ use url::Url; use ya_client_model::activity::TransferArgs; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::hash::verify_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; @@ -162,7 +162,7 @@ async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result< }]; addr.send(AddVolumes::new(volumes)).await??; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 3 * 1024); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 3 * 1024); log::debug!("Starting HTTP servers"); start_http(ctx, temp_dir.to_path_buf()) diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index 80efccc800..500b21ecf8 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -5,7 +5,7 @@ use test_context::test_context; use ya_client_model::activity::TransferArgs; use ya_exe_unit::error::Error; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::hash::verify_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; @@ -70,7 +70,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu }, // Uncomment to enable logs ]; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); log::debug!("Starting HTTP servers"); @@ -182,7 +182,7 @@ async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Resul path: "/extract".into(), }, ]; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); log::debug!("Starting HTTP servers"); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 8b310d7d21..38608f5561 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -178,7 +178,7 @@ pub async fn send_script( exe_unit: Addr>, activity_id: Option, exe_script: Vec, -) { +) -> anyhow::Result { use crate::state::{State, StatePair}; use std::time::Duration; @@ -189,7 +189,8 @@ pub async fn send_script( Ok(GetStateResponse(StatePair(State::Terminated, _))) | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) | Err(_) => { - return log::error!("ExeUnit has terminated"); + log::error!("ExeUnit has terminated"); + bail!("ExeUnit has terminated"); } _ => tokio::time::sleep(delay).await, } @@ -197,9 +198,10 @@ pub async fn send_script( log::debug!("Executing commands: {:?}", exe_script); + let batch_id = hex::encode(rand::random::<[u8; 16]>()); let msg = activity::Exec { activity_id: activity_id.unwrap_or_default(), - batch_id: hex::encode(rand::random::<[u8; 16]>()), + batch_id: batch_id.clone(), exe_script, timeout: None, }; @@ -209,6 +211,7 @@ pub async fn send_script( { log::error!("Unable to execute exe script: {:?}", e); } + Ok(batch_id) } // We need this mut for conditional compilation for sgx diff --git a/exe-unit/tests/resources/agreement.json b/exe-unit/tests/resources/agreement.json index 9dbbccfea4..427dbee17f 100644 --- a/exe-unit/tests/resources/agreement.json +++ b/exe-unit/tests/resources/agreement.json @@ -66,7 +66,7 @@ "caps.multi-activity": true, "comp": { "expiration": 1653216996555, - "task_package": "http://127.0.0.1:8001/rnd", + "task_package": "hash://sha3:22e08b990b6c6685a7e80ecd9a1adb52561a7d9fe9e69b915269da229be6c1ad69dea4ff8a77dc2c4973558da9150909a2be4121b1cbe1ddb04630c1f75aad4f:http://127.0.0.1:8001/image-1", "vm": { "package_format": "gvmkit-squash" } diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 93cec3cccb..4fc29b002c 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -1,63 +1,31 @@ -use actix::Addr; use test_context::test_context; + use ya_client_model::activity::ExeScriptCommand; use ya_exe_unit::message::{Shutdown, ShutdownReason}; -use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::{ - exe_unit, send_script, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli, -}; -use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; -use ya_framework_basic::file::generate_file_with_hash; +use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; -use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; - -#[derive(Debug, Clone)] -pub struct ExeUnitHandle(pub Addr>); - -#[async_trait::async_trait] -impl AsyncDroppable for ExeUnitHandle { - async fn async_drop(&self) { - let finish = self.0.send(FinishNotifier {}).await; - self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); - if let Ok(Ok(mut finish)) = finish { - finish.recv().await.ok(); - } - } -} +use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("exe-unit-start-terminate")?; let temp_dir = dir.path(); + let image_repo = temp_dir.join("images"); - generate_file_with_hash(temp_dir, "rnd", 4096_usize, 10); - start_http(ctx, temp_dir.to_path_buf()) + generate_image(&image_repo, "image-1", 4096_usize, 10); + start_http(ctx, image_repo) .await .expect("unable to start http servers"); - let config = ExeUnitConfig { - args: RunArgs { - agreement: resource!("agreement.json"), - cache_dir: temp_dir.join("cache"), - work_dir: temp_dir.join("work"), - }, - binary: cargo_binary("ya-mock-runtime").unwrap(), - runtime_args: vec![], - supervise: SuperviseCli { - hardware: false, - image: false, - }, - sec_key: None, - requestor_pub_key: None, - service_id: None, - report_url: None, - }; + let config = exe_unit_config(temp_dir, &resource!("agreement.json"), "ya-mock-runtime"); let exe = exe_unit(config).await.unwrap(); let mut finish = exe.send(FinishNotifier {}).await??; @@ -65,7 +33,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow log::info!("Sending [deploy, start] batch for execution."); - send_script( + let batch_id = send_script( exe.clone(), None, vec![ @@ -80,7 +48,10 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow ExeScriptCommand::Start { args: vec![] }, ], ) - .await; + .await + .unwrap(); + + exe.wait_for_batch(&batch_id).await.unwrap(); log::info!("Sending shutdown request."); diff --git a/test-utils/test-framework/framework-basic/src/file.rs b/test-utils/test-framework/framework-basic/src/file.rs index e18df9e037..89a83ecc32 100644 --- a/test-utils/test-framework/framework-basic/src/file.rs +++ b/test-utils/test-framework/framework-basic/src/file.rs @@ -1,17 +1,66 @@ +use rand::rngs::ThreadRng; use rand::Rng; use sha2::Digest; +use std::fs; use std::fs::OpenOptions; use std::io::Write; use std::path::{Path, PathBuf}; use crate::hash::HashOutput; -pub fn generate_file_with_hash( +trait ContentGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec; +} + +struct RandomGenerator(pub ThreadRng); +struct ZeroGenerator {} + +impl ContentGenerator for RandomGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec { + (0..chunk_size) + .map(|_| self.0.gen_range(0..256) as u8) + .collect() + } +} + +impl ContentGenerator for ZeroGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec { + vec![0; chunk_size] + } +} + +pub fn generate_random_file_with_hash( path: &Path, name: &str, chunk_size: usize, chunk_count: usize, ) -> HashOutput { + generate_file_with_hash_( + path, + name, + chunk_size, + chunk_count, + RandomGenerator(rand::thread_rng()), + ) +} + +pub fn generate_image( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, +) -> HashOutput { + generate_file_with_hash_(path, name, chunk_size, chunk_count, ZeroGenerator {}) +} + +fn generate_file_with_hash_( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, + mut gen: impl ContentGenerator, +) -> HashOutput { + fs::create_dir_all(&path).ok(); let path = path.join(name); log::debug!( @@ -25,8 +74,6 @@ pub fn generate_file_with_hash( .open(path) .expect("rnd file"); - let mut rng = rand::thread_rng(); - for i in 0..chunk_count { log::trace!( "Generating chunk {i}/{chunk_count}. File size: {}/{}", @@ -34,9 +81,7 @@ pub fn generate_file_with_hash( chunk_count * chunk_size ); - let input: Vec = (0..chunk_size) - .map(|_| rng.gen_range(0..256) as u8) - .collect(); + let input: Vec = gen.generate(chunk_size); hasher.input(&input); let _ = file_src.write(&input).unwrap(); From 4a1b17bf70a0a6112105b431af43e788d1380a21 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 19 Jan 2024 17:58:22 +0100 Subject: [PATCH 22/52] Send terminate after test --- Cargo.lock | 5 +++ exe-unit/components/mock-runtime/Cargo.toml | 5 +++ .../mock-runtime/src/testing/exe_unit_ext.rs | 31 ++++++++++++++++++- exe-unit/tests/test_exe_unit_basic.rs | 7 +++-- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 516887adfd..cb5ff44d24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8743,16 +8743,21 @@ dependencies = [ "bytes 1.5.0", "env_logger 0.10.1", "futures 0.3.30", + "hex", "log", + "rand 0.8.5", "serde", "serde_json", "thiserror", "tokio", "tokio-util", "url", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.0", + "ya-service-bus", ] [[package]] diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index c73f95478f..6f36159b6e 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -32,6 +32,11 @@ ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils ya-exe-unit = "0.4" ya-framework-basic = { version = "0.1" } +ya-client-model = "0.6" +ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } +ya-service-bus = "0.6" async-trait = "0.1.77" actix = { version = "0.13", default-features = false } +hex = "0.4.3" +rand = "0.8.5" diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index 6dee0f450c..cad5eb993b 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -1,16 +1,24 @@ use actix::Addr; -use anyhow::bail; +use anyhow::{anyhow, bail}; use std::path::Path; use std::time::Duration; +use ya_client_model::activity::ExeScriptCommand; +use ya_core_model::activity; use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; use ya_framework_basic::async_drop::AsyncDroppable; use ya_framework_basic::test_dirs::cargo_binary; +use ya_service_bus::RpcEnvelope; #[async_trait::async_trait] pub trait ExeUnitExt { + async fn exec( + &self, + activity_id: Option, + exe_script: Vec, + ) -> anyhow::Result; async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; } @@ -50,6 +58,27 @@ pub fn exe_unit_config(temp_dir: &Path, agreement_path: &Path, binary: &str) -> #[async_trait::async_trait] impl ExeUnitExt for Addr> { + async fn exec( + &self, + activity_id: Option, + exe_script: Vec, + ) -> anyhow::Result { + log::debug!("Executing commands: {:?}", exe_script); + + let batch_id = hex::encode(rand::random::<[u8; 16]>()); + let msg = activity::Exec { + activity_id: activity_id.unwrap_or_default(), + batch_id: batch_id.clone(), + exe_script, + timeout: None, + }; + self.send(RpcEnvelope::with_caller(String::new(), msg)) + .await + .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))? + .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))?; + Ok(batch_id) + } + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { let delay = Duration::from_secs_f32(0.5); loop { diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 4fc29b002c..ee35beca46 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -55,12 +55,13 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow log::info!("Sending shutdown request."); - exe.send(Shutdown(ShutdownReason::Finished)) + exe.exec(None, vec![ExeScriptCommand::Terminate {}]) .await - .unwrap() .unwrap(); + exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); - finish.recv().await.unwrap(); + log::info!("Waiting for shutdown.."); + finish.recv().await.unwrap(); Ok(()) } From e955e5126d05c190faf5a405439c6d212f755c89 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 22 Jan 2024 17:11:16 +0100 Subject: [PATCH 23/52] Fix goth test due to chenged DebitNote ordering --- goth_tests/domain/exe_units/test_runtime_counters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goth_tests/domain/exe_units/test_runtime_counters.py b/goth_tests/domain/exe_units/test_runtime_counters.py index 6215b2e1c1..287912419f 100644 --- a/goth_tests/domain/exe_units/test_runtime_counters.py +++ b/goth_tests/domain/exe_units/test_runtime_counters.py @@ -113,7 +113,7 @@ async def test_custom_runtime_counter( ) debit_notes = await requestor.api.payment.get_debit_notes() - last_debit_note = debit_notes[0] + last_debit_note = debit_notes[len(debit_notes) - 1] logger.info("last debit note: %r", last_debit_note) assert len(last_debit_note.usage_counter_vector) == len(usage_vector) From 04b6dbf34c829bbe176b720b821c7c0f2a57c0a7 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 24 Jan 2024 16:24:26 +0100 Subject: [PATCH 24/52] Fix: Build test binary in test if it wasn't built --- Cargo.lock | 61 ++++++++++++++++ exe-unit/Cargo.toml | 1 + .../mock-runtime/src/testing/exe_unit_ext.rs | 9 ++- exe-unit/tests/test_exe_unit_basic.rs | 7 +- exe-unit/tests/test_progress.rs | 72 +++++++++++++++++++ .../test-framework/framework-basic/Cargo.toml | 2 + .../framework-basic/src/test_dirs.rs | 23 ++++-- 7 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 exe-unit/tests/test_progress.rs diff --git a/Cargo.lock b/Cargo.lock index cb5ff44d24..4c8211108c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,6 +1085,52 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.21", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.21", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.0.83" @@ -6898,6 +6944,18 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "test-binary" +version = "3.0.2" +source = "git+https://github.com/golemfactory/test-binary.git#c9ebfa3e257455f8365e042b8838a518f2106169" +dependencies = [ + "camino", + "cargo_metadata 0.15.4", + "once_cell", + "paste", + "thiserror", +] + [[package]] name = "test-case" version = "2.2.2" @@ -8386,6 +8444,7 @@ dependencies = [ "derivative", "derive_more", "dotenv", + "duration-string", "env_logger 0.7.1", "flexi_logger 0.22.6", "futures 0.3.30", @@ -8471,6 +8530,7 @@ dependencies = [ "async-trait", "awc", "bytes 1.5.0", + "cargo_metadata 0.18.1", "crossterm 0.26.1", "env_logger 0.7.1", "futures 0.3.30", @@ -8488,6 +8548,7 @@ dependencies = [ "sha2 0.8.2", "sha3 0.8.2", "tempdir", + "test-binary", "test-context", "thiserror", "tokio", diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 27ceca7cf7..5dbed9afc2 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -107,6 +107,7 @@ ya-sb-router = "0.6.1" actix-files = "0.6" actix-web = "4" +duration-string = "0.3" env_logger = "0.7" rustyline = "7.0.0" serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index cad5eb993b..6fd31ac1ea 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -9,7 +9,6 @@ use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; use ya_framework_basic::async_drop::AsyncDroppable; -use ya_framework_basic::test_dirs::cargo_binary; use ya_service_bus::RpcEnvelope; #[async_trait::async_trait] @@ -36,14 +35,18 @@ impl AsyncDroppable for ExeUnitHandle { } } -pub fn exe_unit_config(temp_dir: &Path, agreement_path: &Path, binary: &str) -> ExeUnitConfig { +pub fn exe_unit_config( + temp_dir: &Path, + agreement_path: &Path, + binary: impl AsRef, +) -> ExeUnitConfig { ExeUnitConfig { args: RunArgs { agreement: agreement_path.to_path_buf(), cache_dir: temp_dir.join("cache"), work_dir: temp_dir.join("work"), }, - binary: cargo_binary(binary).unwrap(), + binary: binary.as_ref().to_path_buf(), runtime_args: vec![], supervise: SuperviseCli { hardware: false, diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index ee35beca46..7ca451ba9f 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -7,6 +7,7 @@ use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; @@ -25,7 +26,11 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow .await .expect("unable to start http servers"); - let config = exe_unit_config(temp_dir, &resource!("agreement.json"), "ya-mock-runtime"); + let config = exe_unit_config( + temp_dir, + &resource!("agreement.json"), + cargo_binary("ya-mock-runtime")?, + ); let exe = exe_unit(config).await.unwrap(); let mut finish = exe.send(FinishNotifier {}).await??; diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs new file mode 100644 index 0000000000..4b1b0492bb --- /dev/null +++ b/exe-unit/tests/test_progress.rs @@ -0,0 +1,72 @@ +use duration_string::DurationString; +use std::str::FromStr; +use test_context::test_context; +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::ExeScriptCommand; +use ya_exe_unit::message::{Shutdown, ShutdownReason}; +use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::file::generate_image; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::{resource, temp_dir}; +use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("exe-unit-start-terminate")?; + let temp_dir = dir.path(); + let image_repo = temp_dir.join("images"); + + let hash = generate_image(&image_repo, "image-big", 4096_usize, 10 * 1024); + log::info!("{}", hex::encode(&hash)); + start_http(ctx, image_repo) + .await + .expect("unable to start http servers"); + + let config = exe_unit_config( + temp_dir, + &resource!("agreement.json"), + cargo_binary("ya-mock-runtime")?, + ); + + let exe = exe_unit(config).await.unwrap(); + let mut finish = exe.send(FinishNotifier {}).await??; + ctx.register(ExeUnitHandle(exe.clone())); + + log::info!("Sending [deploy, start] batch for execution."); + + let batch_id = send_script( + exe.clone(), + None, + vec![ + ExeScriptCommand::Deploy { + net: vec![], + progress: Some(ProgressArgs { + update_interval: Some(DurationString::from_str("300ms").unwrap()), + update_step: None, + }), + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }, + ExeScriptCommand::Start { args: vec![] }, + ], + ) + .await + .unwrap(); + + exe.wait_for_batch(&batch_id).await.unwrap(); + exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); + + log::info!("Waiting for shutdown.."); + + finish.recv().await.unwrap(); + Ok(()) +} diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 53474a3257..b21a87124f 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -29,6 +29,7 @@ async-compression = { version = "=0.3.7", features = [ "xz", ] } bytes = "1.0" +cargo_metadata = "0.18" crossterm = "0.26.1" env_logger = "0.7" futures = "0.3.4" @@ -45,6 +46,7 @@ serde = "1.0.104" sha2 = "0.8.1" sha3 = "0.8.2" tempdir = "0.3.7" +test-binary = { version = "3.0", git = "https://github.com/golemfactory/test-binary.git" } test-context = "0.1.4" thiserror = "1.0.11" tokio = { version = "1", features = ["fs", "io-util"] } diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index bf0e1c892b..0be3a63cd2 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -1,7 +1,9 @@ use anyhow::{anyhow, bail}; use std::fs; use std::path::PathBuf; +use std::str::FromStr; use tempdir::TempDir; +use test_binary::TestBinary; pub mod macros { /// Creates temporary directory in cargo target directory. @@ -19,9 +21,6 @@ pub mod macros { #[macro_export] macro_rules! resource { ($name:literal) => { - // CARGO_MANIFEST_DIR is available in compile time only in binary modules, so we can't - // use it in this library. Thanks to using macro it will be resolved in final code not here - // and it will work. ya_framework_basic::test_dirs::resource_(env!("CARGO_MANIFEST_DIR"), $name) }; } @@ -56,8 +55,7 @@ pub fn extension() -> String { "".to_string() } -/// Returns absolute path to cargo project binary. -pub fn cargo_binary(bin_name: &str) -> anyhow::Result { +fn find_binary(bin_name: &str) -> anyhow::Result { let current = std::env::current_exe() .map_err(|e| anyhow!("Failed to get path to current binary. {e}"))? .parent() @@ -81,6 +79,21 @@ pub fn cargo_binary(bin_name: &str) -> anyhow::Result { Ok(bin_path) } +/// Returns path to test binary from workspace. +pub fn cargo_binary(bin_name: &str) -> anyhow::Result { + // Check if binary is already compiled. + if let Err(_) = find_binary(bin_name) { + TestBinary::from_workspace(&bin_name)? + .build() + .map_err(|e| anyhow!("Failed to compile binary: {e}"))? + .to_str() + .map(PathBuf::from_str) + .ok_or(anyhow!("Failed to convert path from OsString"))??; + }; + + find_binary(bin_name) +} + /// Returns resource from `resources` directory in tests. pub fn resource_(base_dir: &str, name: &str) -> PathBuf { PathBuf::from(base_dir) From 1a0458b7ae78ef81b9efa59ae42c02deb558da1a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 26 Jan 2024 17:05:00 +0100 Subject: [PATCH 25/52] Test progress report from streaming batch results --- Cargo.lock | 1 + exe-unit/components/mock-runtime/Cargo.toml | 1 + .../components/mock-runtime/src/testing.rs | 2 +- .../mock-runtime/src/testing/exe_unit_ext.rs | 123 +++++++++++++++--- exe-unit/src/handlers/rpc.rs | 1 + exe-unit/src/lib.rs | 8 +- .../tests/resources/agreement.template.json | 80 ++++++++++++ exe-unit/tests/test_exe_unit_basic.rs | 15 +-- exe-unit/tests/test_progress.rs | 114 +++++++++++----- .../framework-basic/src/test_dirs.rs | 23 +++- 10 files changed, 303 insertions(+), 65 deletions(-) create mode 100644 exe-unit/tests/resources/agreement.template.json diff --git a/Cargo.lock b/Cargo.lock index 4c8211108c..22148afcb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8813,6 +8813,7 @@ dependencies = [ "tokio", "tokio-util", "url", + "uuid 0.8.2", "ya-client-model", "ya-core-model", "ya-exe-unit", diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 6f36159b6e..426b232110 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -40,3 +40,4 @@ async-trait = "0.1.77" actix = { version = "0.13", default-features = false } hex = "0.4.3" rand = "0.8.5" +uuid = { version = "0.8.2", features = ["v4"] } diff --git a/exe-unit/components/mock-runtime/src/testing.rs b/exe-unit/components/mock-runtime/src/testing.rs index cb349f200a..7b2c833428 100644 --- a/exe-unit/components/mock-runtime/src/testing.rs +++ b/exe-unit/components/mock-runtime/src/testing.rs @@ -1,3 +1,3 @@ mod exe_unit_ext; -pub use exe_unit_ext::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; +pub use exe_unit_ext::{create_exe_unit, exe_unit_config, ExeUnitExt, ExeUnitHandle}; diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index 6fd31ac1ea..ad1e26e666 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -1,35 +1,68 @@ use actix::Addr; use anyhow::{anyhow, bail}; use std::path::Path; +use std::sync::Arc; use std::time::Duration; +use tokio::sync::broadcast; +use uuid::Uuid; -use ya_client_model::activity::ExeScriptCommand; +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::{ExeScriptCommand, State, StatePair}; use ya_core_model::activity; -use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; +use ya_exe_unit::message::{GetBatchResults, GetState, GetStateResponse, Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; -use ya_framework_basic::async_drop::AsyncDroppable; +use ya_exe_unit::{exe_unit, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; +use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; use ya_service_bus::RpcEnvelope; #[async_trait::async_trait] pub trait ExeUnitExt { async fn exec( &self, - activity_id: Option, + batch_id: Option, exe_script: Vec, ) -> anyhow::Result; + + async fn deploy(&self, progress: Option) -> anyhow::Result; + async fn start(&self, args: Vec) -> anyhow::Result; + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; + + /// Waits until ExeUnit will be ready to receive commands. + async fn await_init(&self) -> anyhow::Result<()>; } #[derive(Debug, Clone)] -pub struct ExeUnitHandle(pub Addr>); +pub struct ExeUnitHandle { + pub addr: Addr>, + pub config: Arc, +} + +impl ExeUnitHandle { + pub fn new( + addr: Addr>, + config: ExeUnitConfig, + ) -> anyhow::Result { + Ok(ExeUnitHandle { + addr, + config: Arc::new(config), + }) + } + + pub async fn finish_notifier(&self) -> anyhow::Result> { + Ok(self.addr.send(FinishNotifier {}).await??) + } +} #[async_trait::async_trait] impl AsyncDroppable for ExeUnitHandle { async fn async_drop(&self) { - let finish = self.0.send(FinishNotifier {}).await; - self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); - if let Ok(Ok(mut finish)) = finish { + let finish = self.finish_notifier().await; + self.addr + .send(Shutdown(ShutdownReason::Finished)) + .await + .ok(); + if let Ok(mut finish) = finish { finish.recv().await.ok(); } } @@ -54,38 +87,81 @@ pub fn exe_unit_config( }, sec_key: None, requestor_pub_key: None, - service_id: None, + //service_id: None, + service_id: Some(Uuid::new_v4().to_simple().to_string()), + //report_url: Some("/local/activity".to_string()), report_url: None, } } +pub async fn create_exe_unit( + config: ExeUnitConfig, + ctx: &mut DroppableTestContext, +) -> anyhow::Result { + let exe = exe_unit(config.clone()).await.unwrap(); + let handle = ExeUnitHandle::new(exe, config)?; + ctx.register(handle.clone()); + Ok(handle) +} + #[async_trait::async_trait] -impl ExeUnitExt for Addr> { +impl ExeUnitExt for ExeUnitHandle { async fn exec( &self, - activity_id: Option, + batch_id: Option, exe_script: Vec, ) -> anyhow::Result { log::debug!("Executing commands: {:?}", exe_script); - let batch_id = hex::encode(rand::random::<[u8; 16]>()); + let batch_id = if let Some(batch_id) = batch_id { + batch_id + } else { + hex::encode(rand::random::<[u8; 16]>()) + }; + let msg = activity::Exec { - activity_id: activity_id.unwrap_or_default(), + activity_id: self.config.service_id.clone().unwrap_or_default(), batch_id: batch_id.clone(), exe_script, timeout: None, }; - self.send(RpcEnvelope::with_caller(String::new(), msg)) + self.addr + .send(RpcEnvelope::with_caller(String::new(), msg)) .await .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))? .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))?; Ok(batch_id) } + async fn deploy(&self, progress: Option) -> anyhow::Result { + Ok(self + .exec( + None, + vec![ExeScriptCommand::Deploy { + net: vec![], + progress, + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }], + ) + .await + .unwrap()) + } + + async fn start(&self, args: Vec) -> anyhow::Result { + Ok(self + .exec(None, vec![ExeScriptCommand::Start { args }]) + .await + .unwrap()) + } + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { let delay = Duration::from_secs_f32(0.5); loop { match self + .addr .send(GetBatchResults { batch_id: batch_id.to_string(), idx: None, @@ -104,4 +180,21 @@ impl ExeUnitExt for Addr> { tokio::time::sleep(delay).await; } } + + async fn await_init(&self) -> anyhow::Result<()> { + let delay = Duration::from_secs_f32(0.3); + loop { + match self.addr.send(GetState).await { + Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, + Ok(GetStateResponse(StatePair(State::Terminated, _))) + | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) + | Err(_) => { + log::error!("ExeUnit has terminated"); + bail!("ExeUnit has terminated"); + } + _ => tokio::time::sleep(delay).await, + } + } + Ok(()) + } } diff --git a/exe-unit/src/handlers/rpc.rs b/exe-unit/src/handlers/rpc.rs index 44ccd344ca..9788175683 100644 --- a/exe-unit/src/handlers/rpc.rs +++ b/exe-unit/src/handlers/rpc.rs @@ -22,6 +22,7 @@ impl Handler> for ExeUnit { type Result = as Message>::Result; fn handle(&mut self, msg: RpcEnvelope, ctx: &mut Self::Context) -> Self::Result { + log::debug!("Received Exec message: {:?}", msg.as_ref()); self.ctx.verify_activity_id(&msg.activity_id)?; let batch_id = msg.batch_id.clone(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 38608f5561..5e259fc843 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -205,12 +205,10 @@ pub async fn send_script( exe_script, timeout: None, }; - if let Err(e) = exe_unit + + exe_unit .send(RpcEnvelope::with_caller(String::new(), msg)) - .await - { - log::error!("Unable to execute exe script: {:?}", e); - } + .await??; Ok(batch_id) } diff --git a/exe-unit/tests/resources/agreement.template.json b/exe-unit/tests/resources/agreement.template.json new file mode 100644 index 0000000000..a3488f8354 --- /dev/null +++ b/exe-unit/tests/resources/agreement.template.json @@ -0,0 +1,80 @@ +{ + "agreementId": "0ec929f5acc8f98a47ab72d61a2c2f343d45d8438d3aa4ccdc84e717c219e185", + "proposedSignature": "NoSignature", + "state": "Pending", + "timestamp": "2022-05-22T10:41:42.564784259Z", + "validTo": "2022-05-22T11:41:42.562457Z", + + "offer": { + "properties": { + "golem.activity.caps.transfer.protocol": [ + "gftp", + "https", + "http" + ], + "golem.activity.caps.transfer.report-progress": true, + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.payment.platform.erc20-goerli-tglm.address": "0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a", + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [ + 0.001388888888888889, + 0.0002777777777777778, + 0.0 + ], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": [ + "golem.usage.cpu_sec", + "golem.usage.duration_sec" + ], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", + "golem.inf.cpu.capabilities": [], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 11, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 28.0, + "golem.inf.storage.gib": 10.188121795654297, + "golem.node.id.name": "mock-provider", + "golem.node.net.is-public": true, + "golem.runtime.capabilities": [], + "golem.runtime.name": "ya-mock-runtime", + "golem.runtime.version": "0.1.0", + "golem.srv.caps.multi-activity": true, + "golem.srv.caps.payload-manifest": true + }, + "constraints": "(&\n (golem.srv.comp.expiration>1705586871777)\n)", + "offerId": "afce49b1ea5b45db91bdd6e5481479f9-9095fca9dea0a91ce95cf994125b33cdd838fcc963a1106f2be9e4b5b65a52f0", + "providerId": "0x86a269498fb5270f20bdc6fdcf6039122b0d3b23", + "timestamp": "2022-05-22T10:41:42.564784259Z" + }, + + "demand": { + "constraints": "(&(golem.com.payment.platform.erc20-goerli-tglm.address=*)\n\t(golem.com.pricing.model=linear)\n\t(&(golem.inf.mem.gib>=0.5)\n\t(golem.inf.storage.gib>=2.0)\n\t(golem.inf.cpu.threads>=1)\n\t(golem.runtime.name=ya-mock-runtime)))", + "demandId": "773035fc685c46da8e61473ac2a2568e-3f3eb86d6ef9a01708d0f57d0b19cc69fd74422150c120e33cc1b5f4a1a12b96", + "properties": { + "golem": { + "com.payment": { + "chosen-platform": "erc20-goerli-tglm", + "debit-notes.accept-timeout?": 240, + "platform.erc20-goerli-tglm.address": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7" + }, + "node": {}, + "srv": { + "caps.multi-activity": true, + "comp": { + "expiration": 1653216996555, + "task_package": "${task-package}", + "vm": { + "package_format": "gvmkit-squash" + } + } + } + } + }, + "requestorId": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7", + "timestamp": "2022-05-22T10:41:42.564784259Z" + } +} diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 7ca451ba9f..a0ae09d0b5 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -2,14 +2,14 @@ use test_context::test_context; use ya_client_model::activity::ExeScriptCommand; use ya_exe_unit::message::{Shutdown, ShutdownReason}; -use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; +use ya_exe_unit::send_script; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; -use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; +use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] @@ -32,15 +32,14 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow cargo_binary("ya-mock-runtime")?, ); - let exe = exe_unit(config).await.unwrap(); - let mut finish = exe.send(FinishNotifier {}).await??; - ctx.register(ExeUnitHandle(exe.clone())); + let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + let mut finish = exe.finish_notifier().await?; log::info!("Sending [deploy, start] batch for execution."); let batch_id = send_script( - exe.clone(), - None, + exe.addr.clone(), + config.service_id.clone(), vec![ ExeScriptCommand::Deploy { net: vec![], @@ -63,7 +62,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow exe.exec(None, vec![ExeScriptCommand::Terminate {}]) .await .unwrap(); - exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); + exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); log::info!("Waiting for shutdown.."); diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 4b1b0492bb..e72d3b4966 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -1,72 +1,116 @@ +use anyhow::Context; use duration_string::DurationString; +use futures::StreamExt; use std::str::FromStr; use test_context::test_context; +use url::Url; + use ya_client_model::activity::exe_script_command::ProgressArgs; -use ya_client_model::activity::ExeScriptCommand; +use ya_client_model::activity::RuntimeEventKind; +use ya_core_model::activity; use ya_exe_unit::message::{Shutdown, ShutdownReason}; -use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; -use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::test_dirs::{cargo_binary, template}; use ya_framework_basic::{resource, temp_dir}; -use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; +use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; + +use ya_service_bus::typed as bus; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] -async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); +async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(false); - let dir = temp_dir!("exe-unit-start-terminate")?; + let dir = temp_dir!("progress_reporting")?; let temp_dir = dir.path(); let image_repo = temp_dir.join("images"); - let hash = generate_image(&image_repo, "image-big", 4096_usize, 10 * 1024); - log::info!("{}", hex::encode(&hash)); - start_http(ctx, image_repo) + let chunk_size = 4096_usize; + let chunk_count = 1024 * 1; + let file_size = (chunk_size * chunk_count) as u64; + + let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); + let package = format!( + "hash://sha3:{}:http://127.0.0.1:8001/image-big", + hex::encode(&hash) + ); + start_http(ctx, image_repo.clone()) .await .expect("unable to start http servers"); + let gsb_url = Url::from_str(&format!("unix://{}/mock-yagna.sock", temp_dir.display())).unwrap(); + std::env::set_var("GSB_URL", gsb_url.to_string()); + ya_sb_router::bind_gsb_router(Some(gsb_url)) + .await + .context("binding service bus router")?; + let config = exe_unit_config( temp_dir, - &resource!("agreement.json"), + &template( + &resource!("agreement.template.json"), + temp_dir.join("agreement.json"), + &[("task-package", package)], + )?, cargo_binary("ya-mock-runtime")?, ); - let exe = exe_unit(config).await.unwrap(); - let mut finish = exe.send(FinishNotifier {}).await??; - ctx.register(ExeUnitHandle(exe.clone())); + let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + let mut finish = exe.finish_notifier().await?; + exe.await_init().await.unwrap(); log::info!("Sending [deploy, start] batch for execution."); - let batch_id = send_script( - exe.clone(), - None, - vec![ - ExeScriptCommand::Deploy { - net: vec![], - progress: Some(ProgressArgs { - update_interval: Some(DurationString::from_str("300ms").unwrap()), - update_step: None, - }), - env: Default::default(), - hosts: Default::default(), - hostname: None, - volumes: vec![], - }, - ExeScriptCommand::Start { args: vec![] }, - ], - ) - .await - .unwrap(); + let batch_id = exe + .deploy(Some(ProgressArgs { + update_interval: Some(DurationString::from_str("300ms").unwrap()), + update_step: None, + })) + .await + .unwrap(); + + let msg = activity::StreamExecBatchResults { + activity_id: config.service_id.unwrap(), + batch_id: batch_id.clone(), + }; + + // Note: Since we have already sent commands, we may loose a few events on the beginning. + // Our API has a problem here. We can't call `StreamExecBatchResults` before Exeunit knows + // `batch_id`. Even if we would generate id ourselves (possible in test, but not possible for Requestor), + // we still can't call this function too early. + let mut stream = bus::service(activity::exeunit::bus_id(&msg.activity_id)).call_streaming(msg); + + let mut last_progress = 0u64; + while let Some(Ok(Ok(item))) = stream.next().await { + if item.index == 0 { + match item.kind { + RuntimeEventKind::Finished { return_code, .. } => { + assert_eq!(return_code, 0); + break; + } + RuntimeEventKind::Progress(progress) => { + log::info!("Progress report: {:?}", progress); + + assert_eq!(progress.step, (0, 1)); + assert_eq!(progress.unit, Some("Bytes".to_string())); + assert_eq!(progress.progress.1.unwrap(), file_size); + assert!(progress.progress.0 >= last_progress); + + last_progress = progress.progress.0; + } + _ => (), + } + } + } exe.wait_for_batch(&batch_id).await.unwrap(); - exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); log::info!("Waiting for shutdown.."); + exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); finish.recv().await.unwrap(); Ok(()) } diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index 0be3a63cd2..c0039e6577 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, bail}; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use tempdir::TempDir; use test_binary::TestBinary; @@ -101,3 +101,24 @@ pub fn resource_(base_dir: &str, name: &str) -> PathBuf { .join("resources") .join(name) } + +/// Generates resource from template by replacing occurrences of `${name}` pattern +/// using variables from `vars` dictionary. +/// Returns path to generated file, which is the same as `target` param, but makes it easier +/// to use this function in code. +pub fn template( + template: &Path, + target: impl AsRef, + vars: &[(&str, String)], +) -> anyhow::Result { + let mut template = fs::read_to_string(&template) + .map_err(|e| anyhow!("Loading template {} failed: {e}", template.display()))?; + for var in vars { + template = template.replace(&format!("${{{}}}", var.0), &var.1); + } + + let target = target.as_ref(); + fs::write(target, template) + .map_err(|e| anyhow!("Saving template {} failed: {e}", target.display()))?; + Ok(target.to_path_buf()) +} From 9a8042ef39b95dfbd871f80a294af6bf7b556cdc Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 26 Jan 2024 17:46:54 +0100 Subject: [PATCH 26/52] Handle GSB router setup in test utils --- Cargo.lock | 11 ++++++ exe-unit/components/mock-runtime/Cargo.toml | 2 ++ .../mock-runtime/src/testing/exe_unit_ext.rs | 34 +++++++++++++++++-- exe-unit/tests/test_progress.rs | 8 ----- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22148afcb1..ef4055fc60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4662,6 +4662,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "portpicker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -8806,6 +8815,7 @@ dependencies = [ "futures 0.3.30", "hex", "log", + "portpicker", "rand 0.8.5", "serde", "serde_json", @@ -8819,6 +8829,7 @@ dependencies = [ "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.0", + "ya-sb-router", "ya-service-bus", ] diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 426b232110..dc65e9dee9 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -35,9 +35,11 @@ ya-framework-basic = { version = "0.1" } ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } ya-service-bus = "0.6" +ya-sb-router = "0.6" async-trait = "0.1.77" actix = { version = "0.13", default-features = false } hex = "0.4.3" +portpicker = "0.1.1" rand = "0.8.5" uuid = { version = "0.8.2", features = ["v4"] } diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index ad1e26e666..d8b26d0666 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -1,9 +1,12 @@ use actix::Addr; use anyhow::{anyhow, bail}; -use std::path::Path; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use tokio::sync::broadcast; +use url::Url; use uuid::Uuid; use ya_client_model::activity::exe_script_command::ProgressArgs; @@ -87,9 +90,7 @@ pub fn exe_unit_config( }, sec_key: None, requestor_pub_key: None, - //service_id: None, service_id: Some(Uuid::new_v4().to_simple().to_string()), - //report_url: Some("/local/activity".to_string()), report_url: None, } } @@ -98,6 +99,33 @@ pub async fn create_exe_unit( config: ExeUnitConfig, ctx: &mut DroppableTestContext, ) -> anyhow::Result { + if config.service_id.is_some() { + let gsb_url = match std::env::consts::FAMILY { + "unix" => Url::from_str(&format!( + "unix://{}/gsb.sock", + config.args.work_dir.display() + ))?, + _ => Url::from_str(&format!( + "tcp://127.0.0.1:{}", + portpicker::pick_unused_port().ok_or(anyhow!("No ports free"))? + ))?, + }; + + if gsb_url.scheme() == "unix" { + let dir = PathBuf::from_str(gsb_url.path())? + .parent() + .map(|path| path.to_path_buf()) + .ok_or(anyhow!("`gsb_url` unix socket has no parent directory."))?; + fs::create_dir_all(dir)?; + } + + // GSB takes url from this variable and we can't set it directly. + std::env::set_var("GSB_URL", gsb_url.to_string()); + ya_sb_router::bind_gsb_router(Some(gsb_url.clone())) + .await + .map_err(|e| anyhow!("Error binding service bus router to '{}': {e}", &gsb_url))?; + } + let exe = exe_unit(config.clone()).await.unwrap(); let handle = ExeUnitHandle::new(exe, config)?; ctx.register(handle.clone()); diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index e72d3b4966..5fd33edc73 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -1,9 +1,7 @@ -use anyhow::Context; use duration_string::DurationString; use futures::StreamExt; use std::str::FromStr; use test_context::test_context; -use url::Url; use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::RuntimeEventKind; @@ -42,12 +40,6 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu .await .expect("unable to start http servers"); - let gsb_url = Url::from_str(&format!("unix://{}/mock-yagna.sock", temp_dir.display())).unwrap(); - std::env::set_var("GSB_URL", gsb_url.to_string()); - ya_sb_router::bind_gsb_router(Some(gsb_url)) - .await - .context("binding service bus router")?; - let config = exe_unit_config( temp_dir, &template( From 83f62863dd585ebe59b5ead26512c906edf17813 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 26 Jan 2024 17:55:18 +0100 Subject: [PATCH 27/52] Hide shutdown function in tests --- .../mock-runtime/src/testing/exe_unit_ext.rs | 18 +++++++++++++----- exe-unit/tests/test_exe_unit_basic.rs | 7 +------ exe-unit/tests/test_progress.rs | 10 +++------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index d8b26d0666..bbded7e242 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -55,19 +55,27 @@ impl ExeUnitHandle { pub async fn finish_notifier(&self) -> anyhow::Result> { Ok(self.addr.send(FinishNotifier {}).await??) } -} -#[async_trait::async_trait] -impl AsyncDroppable for ExeUnitHandle { - async fn async_drop(&self) { + pub async fn shutdown(&self) -> anyhow::Result<()> { let finish = self.finish_notifier().await; + + log::info!("Waiting for shutdown.."); + self.addr .send(Shutdown(ShutdownReason::Finished)) .await .ok(); if let Ok(mut finish) = finish { - finish.recv().await.ok(); + finish.recv().await?; } + Ok(()) + } +} + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + self.shutdown().await.ok(); } } diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index a0ae09d0b5..618fa20595 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -1,7 +1,6 @@ use test_context::test_context; use ya_client_model::activity::ExeScriptCommand; -use ya_exe_unit::message::{Shutdown, ShutdownReason}; use ya_exe_unit::send_script; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; @@ -33,7 +32,6 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow ); let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); - let mut finish = exe.finish_notifier().await?; log::info!("Sending [deploy, start] batch for execution."); @@ -62,10 +60,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow exe.exec(None, vec![ExeScriptCommand::Terminate {}]) .await .unwrap(); - exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); - log::info!("Waiting for shutdown.."); - - finish.recv().await.unwrap(); + exe.shutdown().await.unwrap(); Ok(()) } diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 5fd33edc73..4befc7c2c1 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -6,7 +6,6 @@ use test_context::test_context; use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::RuntimeEventKind; use ya_core_model::activity; -use ya_exe_unit::message::{Shutdown, ShutdownReason}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; @@ -51,7 +50,6 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu ); let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); - let mut finish = exe.finish_notifier().await?; exe.await_init().await.unwrap(); log::info!("Sending [deploy, start] batch for execution."); @@ -76,6 +74,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let mut stream = bus::service(activity::exeunit::bus_id(&msg.activity_id)).call_streaming(msg); let mut last_progress = 0u64; + let mut num_progresses = 0u64; while let Some(Ok(Ok(item))) = stream.next().await { if item.index == 0 { match item.kind { @@ -92,17 +91,14 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu assert!(progress.progress.0 >= last_progress); last_progress = progress.progress.0; + num_progresses += 1; } _ => (), } } } + assert!(num_progresses > 1); exe.wait_for_batch(&batch_id).await.unwrap(); - - log::info!("Waiting for shutdown.."); - - exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); - finish.recv().await.unwrap(); Ok(()) } From 39a790f3a58222669fe5bf696cfa404121f9ac3c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 15:02:34 +0100 Subject: [PATCH 28/52] Deploy - adding additional volumes (not included in vm image); Test transfer progress --- .../mock-runtime/src/testing/exe_unit_ext.rs | 2 + exe-unit/src/exe_unit.rs | 12 ++++ exe-unit/tests/test_progress.rs | 70 ++++++++++++++++--- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index bbded7e242..8559a20c27 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -107,6 +107,8 @@ pub async fn create_exe_unit( config: ExeUnitConfig, ctx: &mut DroppableTestContext, ) -> anyhow::Result { + // If activity id was provided, ExeUnit will bind endpoints on remote GSB. + // For this to work we need to setup gsb router. if config.service_id.is_some() { let gsb_url = match std::env::consts::FAMILY { "unix" => Url::from_str(&format!( diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index aa4a426038..10f97c967d 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -15,6 +15,7 @@ use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, use ya_core_model::activity; use ya_core_model::activity::local::Credentials; use ya_runtime_api::deploy; +use ya_runtime_api::deploy::ContainerVolume; use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; use ya_transfer::transfer::{ AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, @@ -313,8 +314,19 @@ impl RuntimeRef { net, hosts, progress, + volumes, .. } => { + let volumes = volumes + .iter() + .enumerate() + .map(|(idx, vol)| ContainerVolume { + name: format!("vol-custom-{idx}"), + path: vol.to_string(), + }) + .collect::>(); + transfer_service.send(AddVolumes::new(volumes)).await??; + // TODO: We should pass `task_package` here not in `TransferService` initialization. let mut msg = DeployImage::default(); if let Some(args) = progress { diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 4befc7c2c1..2c23548083 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use test_context::test_context; use ya_client_model::activity::exe_script_command::ProgressArgs; -use ya_client_model::activity::RuntimeEventKind; +use ya_client_model::activity::{ExeScriptCommand, RuntimeEventKind, TransferArgs}; use ya_core_model::activity; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; @@ -16,13 +16,15 @@ use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; use ya_service_bus::typed as bus; +/// Test if progress reporting mechanisms work on gsb level +/// with full ExeUnit setup. #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { enable_logs(false); - let dir = temp_dir!("progress_reporting")?; + let dir = temp_dir!("progress-reporting")?; let temp_dir = dir.path(); let image_repo = temp_dir.join("images"); @@ -52,18 +54,67 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); exe.await_init().await.unwrap(); - log::info!("Sending [deploy, start] batch for execution."); + log::info!("Sending [deploy] batch for execution."); let batch_id = exe - .deploy(Some(ProgressArgs { - update_interval: Some(DurationString::from_str("300ms").unwrap()), - update_step: None, - })) + .exec( + None, + vec![ExeScriptCommand::Deploy { + net: vec![], + progress: Some(ProgressArgs { + update_interval: Some(DurationString::from_str("300ms").unwrap()), + update_step: None, + }), + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec!["/input".to_owned()], + }], + ) .await .unwrap(); + validate_progress( + config.service_id.clone().unwrap(), + batch_id.clone(), + file_size, + ) + .await; + + exe.wait_for_batch(&batch_id).await.unwrap(); + exe.wait_for_batch(&exe.start(vec![]).await.unwrap()) + .await + .unwrap(); + + let batch_id = exe + .exec( + None, + vec![ExeScriptCommand::Transfer { + args: TransferArgs::default(), + progress: Some(ProgressArgs { + update_interval: Some(DurationString::from_str("100ms").unwrap()), + update_step: None, + }), + // Important: Use hashed transfer, because it is significantly slower in debug mode. + // Otherwise we won't get any progress message, because it is too fast. + from: format!( + "hash://sha3:{}:http://127.0.0.1:8001/image-big", + hex::encode(&hash) + ), + to: "container:/input/image-copy".to_string(), + }], + ) + .await + .unwrap(); + + validate_progress(config.service_id.unwrap(), batch_id.clone(), file_size).await; + exe.wait_for_batch(&batch_id).await.unwrap(); + Ok(()) +} + +async fn validate_progress(activity_id: String, batch_id: String, file_size: u64) { let msg = activity::StreamExecBatchResults { - activity_id: config.service_id.unwrap(), + activity_id: activity_id.clone(), batch_id: batch_id.clone(), }; @@ -98,7 +149,4 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu } } assert!(num_progresses > 1); - - exe.wait_for_batch(&batch_id).await.unwrap(); - Ok(()) } From 22c04f6db0451bbf1ade1a0274f04f181bc4a93c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 16:21:55 +0100 Subject: [PATCH 29/52] Rename framework-test to system-test in exeunits --- Cargo.toml | 2 +- exe-unit/Cargo.toml | 2 +- exe-unit/components/transfer/Cargo.toml | 2 +- exe-unit/components/transfer/tests/test_deploy.rs | 4 ++-- exe-unit/components/transfer/tests/test_transfer_abort.rs | 2 +- exe-unit/components/transfer/tests/test_transfer_resume.rs | 2 +- exe-unit/components/transfer/tests/test_transfer_service.rs | 4 ++-- exe-unit/tests/test_exe_unit_basic.rs | 2 +- exe-unit/tests/test_progress.rs | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 99643039c9..ca1444b979 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] erc20next-driver = ['ya-erc20next-driver'] tos = [] -framework-test = ['ya-exe-unit/framework-test'] +framework-test = ['ya-exe-unit/system-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 5dbed9afc2..55a66d937c 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -24,7 +24,7 @@ sgx = [ 'ya-transfer/sgx', ] packet-trace-enable = ["ya-packet-trace/enable"] -framework-test = ["ya-transfer/framework-test"] +system-test = ["ya-transfer/system-test"] [target.'cfg(target_family = "unix")'.dependencies] nix = "0.22.0" diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index e9e67ddae0..468b6dd483 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -59,7 +59,7 @@ sgx = [ 'ya-core-model/sgx', 'reqwest/trust-dns', ] -framework-test = [] +system-test = [] [dependencies.zip] version = "0.5.6" diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 2f84fb4d24..19e15078d2 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -18,7 +18,7 @@ use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, Transf /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -78,7 +78,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 3b8e7e7ec6..12d2dc8f92 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -50,7 +50,7 @@ async fn interrupted_transfer( Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_abort(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 03920246b6..bf62391943 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -128,7 +128,7 @@ async fn transfer( transfer_with_args(addr, from, to, TransferArgs::default()).await } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index 500b21ecf8..4c4217bfec 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -38,7 +38,7 @@ async fn transfer_with_args( Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -151,7 +151,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu } #[ignore] -//#[cfg_attr(not(feature = "framework-test"), ignore)] +//#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 618fa20595..5d97f6bfe9 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -10,7 +10,7 @@ use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 2c23548083..d6c2146988 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -18,11 +18,11 @@ use ya_service_bus::typed as bus; /// Test if progress reporting mechanisms work on gsb level /// with full ExeUnit setup. -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(false); + enable_logs(true); let dir = temp_dir!("progress-reporting")?; let temp_dir = dir.path(); From 0fbd17126d8ad1e12f44c3ec867e0187415c94cf Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 17:59:19 +0100 Subject: [PATCH 30/52] Replace basic exeunit test with helper deploy, start commands --- exe-unit/tests/test_exe_unit_basic.rs | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 5d97f6bfe9..5c354e4b6c 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -1,7 +1,6 @@ use test_context::test_context; use ya_client_model::activity::ExeScriptCommand; -use ya_exe_unit::send_script; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; @@ -32,28 +31,16 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow ); let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + exe.await_init().await.unwrap(); log::info!("Sending [deploy, start] batch for execution."); - let batch_id = send_script( - exe.addr.clone(), - config.service_id.clone(), - vec![ - ExeScriptCommand::Deploy { - net: vec![], - progress: None, - env: Default::default(), - hosts: Default::default(), - hostname: None, - volumes: vec![], - }, - ExeScriptCommand::Start { args: vec![] }, - ], - ) - .await - .unwrap(); - - exe.wait_for_batch(&batch_id).await.unwrap(); + exe.wait_for_batch(&exe.deploy(None).await.unwrap()) + .await + .unwrap(); + exe.wait_for_batch(&exe.start(vec![]).await.unwrap()) + .await + .unwrap(); log::info!("Sending shutdown request."); From 148362f44acdc79b45c78472ff7afa18e92d9f3d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 18:09:24 +0100 Subject: [PATCH 31/52] Fix clippy --- exe-unit/components/transfer/src/progress.rs | 4 ++-- test-utils/test-framework/framework-basic/src/file.rs | 2 +- test-utils/test-framework/framework-basic/src/test_dirs.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 289ed049b2..b8df121ef6 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -57,7 +57,7 @@ impl ProgressReporter { .unwrap_or(Duration::from_secs(1)); let _update_step = self.config.update_step; - self.inner.lock().unwrap().as_mut().map(|inner| { + if let Some(inner) = self.inner.lock().unwrap().as_mut() { inner.last.progress = (progress, size); if inner.last_send + update_interval <= Instant::now() { inner.last_send = Instant::now(); @@ -69,7 +69,7 @@ impl ProgressReporter { }) .ok(); } - }); + } } pub fn report_message(&self, message: String) { diff --git a/test-utils/test-framework/framework-basic/src/file.rs b/test-utils/test-framework/framework-basic/src/file.rs index 89a83ecc32..b2e1c0b9bf 100644 --- a/test-utils/test-framework/framework-basic/src/file.rs +++ b/test-utils/test-framework/framework-basic/src/file.rs @@ -60,7 +60,7 @@ fn generate_file_with_hash_( chunk_count: usize, mut gen: impl ContentGenerator, ) -> HashOutput { - fs::create_dir_all(&path).ok(); + fs::create_dir_all(path).ok(); let path = path.join(name); log::debug!( diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index c0039e6577..374c697b2a 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -82,8 +82,8 @@ fn find_binary(bin_name: &str) -> anyhow::Result { /// Returns path to test binary from workspace. pub fn cargo_binary(bin_name: &str) -> anyhow::Result { // Check if binary is already compiled. - if let Err(_) = find_binary(bin_name) { - TestBinary::from_workspace(&bin_name)? + if find_binary(bin_name).is_err() { + TestBinary::from_workspace(bin_name)? .build() .map_err(|e| anyhow!("Failed to compile binary: {e}"))? .to_str() @@ -111,7 +111,7 @@ pub fn template( target: impl AsRef, vars: &[(&str, String)], ) -> anyhow::Result { - let mut template = fs::read_to_string(&template) + let mut template = fs::read_to_string(template) .map_err(|e| anyhow!("Loading template {} failed: {e}", template.display()))?; for var in vars { template = template.replace(&format!("${{{}}}", var.0), &var.1); From 15eeb5ae49eb77910225fdf434b34342f77e7be3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 18:20:16 +0100 Subject: [PATCH 32/52] Fix clipy mut sgx --- exe-unit/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 5e259fc843..7d9ac9e7db 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -308,7 +308,9 @@ pub struct ExeUnitConfig { pub requestor_pub_key: Option, } -pub async fn exe_unit(config: ExeUnitConfig) -> anyhow::Result>> { +// Mut is necessary in case of sgx compilation :((((( +#[allow(unused_mut)] +pub async fn exe_unit(mut config: ExeUnitConfig) -> anyhow::Result>> { let args = config.args; if !args.agreement.exists() { bail!( From bc8c02be3c46ecefb4e9e2cf3c76e1ce30a9dfe4 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 18:34:34 +0100 Subject: [PATCH 33/52] Allow large enum variant --- exe-unit/src/message.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/exe-unit/src/message.rs b/exe-unit/src/message.rs index 8c19e9c3ee..d105cc392d 100644 --- a/exe-unit/src/message.rs +++ b/exe-unit/src/message.rs @@ -125,6 +125,7 @@ impl ExecuteCommand { } } +#[allow(clippy::large_enum_variant)] #[derive(Clone, Debug)] pub enum RuntimeEvent { Process(activity::RuntimeEvent), From 7a6613dab6c00fce4d651fe5674ede70daf91723 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 19:33:00 +0100 Subject: [PATCH 34/52] Run system tests for ya-transfer and ya-exe-unit --- .github/workflows/system-test.yml | 2 +- exe-unit/components/mock-runtime/src/lib.rs | 4 ++-- exe-unit/components/transfer/src/progress.rs | 2 +- tests/readme.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/system-test.yml b/.github/workflows/system-test.yml index bf10fc47a9..50898e7003 100644 --- a/.github/workflows/system-test.yml +++ b/.github/workflows/system-test.yml @@ -73,4 +73,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --test '*' --features framework-test + args: --test '*' -p yagna -p ya-exe-unit -p ya-transfer --features framework-test diff --git a/exe-unit/components/mock-runtime/src/lib.rs b/exe-unit/components/mock-runtime/src/lib.rs index c818b73da4..ee9ed1bab4 100644 --- a/exe-unit/components/mock-runtime/src/lib.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -60,12 +60,12 @@ impl RuntimeService for RuntimeMock { // client // holds last received status -#[derive(Clone)] +#[derive(Clone, Default)] pub struct EventMock(Arc>); impl EventMock { pub fn new() -> Self { - Self(Arc::new(Mutex::new(Default::default()))) + Self::default() } pub fn get_last_status(&self) -> ProcessStatus { diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index b8df121ef6..332c5d6d3b 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -255,7 +255,7 @@ mod tests { Duration::from_millis(525) - update }; - assert!(diff <= Duration::from_millis(20)); + assert!(diff <= Duration::from_millis(40)); // `ProgressReporter` should ignore 10 messages in each loop. assert_eq!(event.progress.0, counter * 10); diff --git a/tests/readme.md b/tests/readme.md index 84919fcc7d..02f4742e73 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -11,7 +11,7 @@ To run all tests including framework tests and unit tests (but without market te `cargo test --workspace --features framework-test` To run only framework tests use command: -`cargo test --test '*' --features framework-test` +`cargo test --test '*' -p yagna -p ya-exe-unit -p ya-transfer --features framework-test` ## Creating tests From 50ca348a6a9f08e0c3526e1e5167c17d732ebf8e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 19:46:14 +0100 Subject: [PATCH 35/52] Fix ya-transfer features of dependencies --- exe-unit/components/transfer/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 468b6dd483..97ecf9a6f9 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] ya-client-model = "0.6" -ya-core-model = { version = "^0.9" } +ya-core-model = { version = "0.9", features = ["activity"] } ya-service-bus = "0.6.1" ya-utils-path = { version ="0.1", path = "../../../utils/path" } ya-utils-futures = { version = "0.3", path = "../../../utils/futures" } @@ -42,7 +42,7 @@ tokio-util = { version = "0.7", features = ["io"] } url = "2.1.1" walkdir = "2.3.1" async-trait = "0.1.74" -tokio-stream = "0.1.14" +tokio-stream = { version = "0.1.14", features = ["sync"] } [target.'cfg(target_family = "unix")'.dependencies] awc = { version = "3", features = ["openssl"] } From d7f3df0732c1a2ed020cc8632f17c379eb05b13a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 19:52:42 +0100 Subject: [PATCH 36/52] New clippy fixes --- exe-unit/tests/test_progress.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index f27a950669..dd9f039128 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -29,13 +29,13 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let image_repo = temp_dir.join("images"); let chunk_size = 4096_usize; - let chunk_count = 1024 * 1; + let chunk_count = 1024 * 2; let file_size = (chunk_size * chunk_count) as u64; let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); let package = format!( "hash://sha3:{}:http://127.0.0.1:8001/image-big", - hex::encode(&hash) + hex::encode(hash) ); start_http(ctx, image_repo.clone()) .await @@ -99,7 +99,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu // Otherwise we won't get any progress message, because it is too fast. from: format!( "hash://sha3:{}:http://127.0.0.1:8001/image-big", - hex::encode(&hash) + hex::encode(hash) ), to: "container:/input/image-copy".to_string(), }], From 63eb34bf23f61d65363e2ced89b194be85d6b964 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jan 2024 15:33:31 +0100 Subject: [PATCH 37/52] Extend time periods in progress tests --- exe-unit/components/transfer/src/progress.rs | 14 +++++++------- exe-unit/tests/test_progress.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 332c5d6d3b..7c9d1ba2b9 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -216,7 +216,7 @@ mod tests { Some(ProgressConfig { progress: tx, progress_args: ProgressArgs { - update_interval: Some("500ms".parse::().unwrap()), + update_interval: Some("1s".parse::().unwrap()), update_step: None, }, }), @@ -228,11 +228,11 @@ mod tests { let mut before = Instant::now(); tokio::task::spawn_local(async move { for step in 0..2 { - tokio::time::sleep(Duration::from_millis(25)).await; + tokio::time::sleep(Duration::from_millis(50)).await; for i in 0..=size { report.report_progress(i, Some(size)); - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(Duration::from_millis(100)).await; } if step == 0 { report.next_step(); @@ -249,13 +249,13 @@ mod tests { counter += 1; let update = Instant::now().duration_since(before); before = Instant::now(); - let diff = if update > Duration::from_millis(525) { - update - Duration::from_millis(525) + let diff = if update > Duration::from_millis(1050) { + update - Duration::from_millis(1050) } else { - Duration::from_millis(525) - update + Duration::from_millis(1050) - update }; - assert!(diff <= Duration::from_millis(40)); + assert!(diff <= Duration::from_millis(60)); // `ProgressReporter` should ignore 10 messages in each loop. assert_eq!(event.progress.0, counter * 10); diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index dd9f039128..7dc6b1dc39 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -22,7 +22,7 @@ use ya_service_bus::typed as bus; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(false); + enable_logs(true); let dir = temp_dir!("progress-reporting")?; let temp_dir = dir.path(); From 80277a41ff6957f8fb53301828e041b4fd4e352e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jan 2024 15:42:40 +0100 Subject: [PATCH 38/52] Fix ExeUnit binary compilation with packet-trace flag --- exe-unit/components/transfer/tests/test_deploy.rs | 2 +- exe-unit/src/bin.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 19e15078d2..81345ab00a 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -97,7 +97,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< } let chunk_size = 4096_usize; - let chunk_count = 1024 * 1; + let chunk_count = 1024; let file_size = (chunk_size * chunk_count) as u64; let hash = generate_random_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); diff --git a/exe-unit/src/bin.rs b/exe-unit/src/bin.rs index 4dc9e0b7bc..95fb3e5822 100644 --- a/exe-unit/src/bin.rs +++ b/exe-unit/src/bin.rs @@ -27,8 +27,11 @@ async fn main() { }; dotenv::dotenv().ok(); + #[cfg(feature = "packet-trace-enable")] - init_packet_trace()?; + if let Err(error) = init_packet_trace() { + log::warn!("Initializing packet tracing failed: {error:?}"); + } let cli: Cli = Cli::from_args(); From ab7b65f6f4c3dbdcb2d9dd2e0437b0fb91210445 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 1 Feb 2024 17:38:29 +0100 Subject: [PATCH 39/52] Better error message from gftp --- core/gftp/src/rpc.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/gftp/src/rpc.rs b/core/gftp/src/rpc.rs index afbccf377f..948816d45c 100644 --- a/core/gftp/src/rpc.rs +++ b/core/gftp/src/rpc.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::fmt::Display; use std::io::Write; use std::path::PathBuf; use structopt::StructOpt; @@ -24,9 +25,9 @@ pub enum JsonRpcError { ServerError = -32000, } -impl ToString for JsonRpcError { - fn to_string(&self) -> String { - format!("{:?}", self) +impl Display for JsonRpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", format!("{:?}", self)) } } @@ -89,8 +90,8 @@ impl RpcMessage { Self::response(id, RpcResult::Files(items)) } - pub fn error + ToString>(id: Option<&RpcId>, err: E) -> Self { - let message = err.to_string(); + pub fn error + std::fmt::Display>(id: Option<&RpcId>, err: E) -> Self { + let message = format!("{err:#}"); RpcMessage { jsonrpc: JSON_RPC_VERSION.to_string(), id: id.cloned(), From 26d7d53b7d6bba696aeb7bb84595d46959c5d43b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 1 Feb 2024 17:48:41 +0100 Subject: [PATCH 40/52] Fix clippy --- core/gftp/src/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/gftp/src/rpc.rs b/core/gftp/src/rpc.rs index 948816d45c..2c1b4f9f12 100644 --- a/core/gftp/src/rpc.rs +++ b/core/gftp/src/rpc.rs @@ -27,7 +27,7 @@ pub enum JsonRpcError { impl Display for JsonRpcError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", format!("{:?}", self)) + write!(f, "{self:?}") } } From 26c0fca201de8cb18020706829052a3439a4c22e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 5 Feb 2024 18:05:40 +0100 Subject: [PATCH 41/52] Increase progress test duration to fix windows tests --- exe-unit/components/transfer/src/progress.rs | 26 ++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 7a9ffb4b4e..d7e6cf155c 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -210,13 +210,25 @@ mod tests { #[actix_rt::test] async fn test_progress_reporter_interval() { + // Note: This test is time dependent and you can expect it to fail on very slow machines. + // If this happens, you could scale intervals by increasing `loop_interval`. Rather you + // shouldn't touch relations between these variables, if you don't know what are you doing. + let loop_interval = 200u64; + let update_interval = 10 * loop_interval; + let offset = loop_interval / 2; + let margin = loop_interval * 8 / 10; + let mut report = ProgressReporter::default(); let (tx, mut rx) = tokio::sync::broadcast::channel(10); report.register_reporter( Some(ProgressConfig { progress: tx, progress_args: ProgressArgs { - update_interval: Some("1s".parse::().unwrap()), + update_interval: Some( + format!("{update_interval}ms") + .parse::() + .unwrap(), + ), update_step: None, }, }), @@ -228,11 +240,11 @@ mod tests { let mut before = Instant::now(); tokio::task::spawn_local(async move { for step in 0..2 { - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(Duration::from_millis(offset)).await; for i in 0..=size { report.report_progress(i, Some(size)); - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(loop_interval)).await; } if step == 0 { report.next_step(); @@ -249,13 +261,13 @@ mod tests { counter += 1; let update = Instant::now().duration_since(before); before = Instant::now(); - let diff = if update > Duration::from_millis(1050) { - update - Duration::from_millis(1050) + let diff = if update > Duration::from_millis(update_interval + offset) { + update - Duration::from_millis(update_interval + offset) } else { - Duration::from_millis(1050) - update + Duration::from_millis(update_interval + offset) - update }; - assert!(diff <= Duration::from_millis(80)); + assert!(diff <= Duration::from_millis(margin)); // `ProgressReporter` should ignore 10 messages in each loop. assert_eq!(event.progress.0, counter * 10); From 36f659a6ce30d67e95acb366cb0846b38362bd85 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Feb 2024 16:23:31 +0100 Subject: [PATCH 42/52] Deploy larger file in test so we will get at least 1 progress event --- exe-unit/tests/test_progress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 7dc6b1dc39..a815134b39 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -29,7 +29,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let image_repo = temp_dir.join("images"); let chunk_size = 4096_usize; - let chunk_count = 1024 * 2; + let chunk_count = 1024 * 20; let file_size = (chunk_size * chunk_count) as u64; let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); From fac0b8eb3683f21865bbaef09d0839f3dd32936d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Feb 2024 17:18:14 +0100 Subject: [PATCH 43/52] Add Command progress report capability to docs --- docs/provider/capabilities.md | 21 ++++++++++++++------- docs/provider/exe-unit/command-progress.md | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 docs/provider/exe-unit/command-progress.md diff --git a/docs/provider/capabilities.md b/docs/provider/capabilities.md index e5ddaf928e..7e618ab5ee 100644 --- a/docs/provider/capabilities.md +++ b/docs/provider/capabilities.md @@ -2,10 +2,17 @@ ## Protocol -| Capability | Yagna package version | Backwards-compatible? | Description | -|-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use (which?) property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | -| Restart Proposal Negotiations | 0.7.0 | Yes | Agent is allowed to restart negotiations, by sending `Counter Proposal`, after he rejected Proposal at some point. Counter-party will receive regular `ProposalEvent` in this case. Only Agent rejecting Proposal has initiative in restarting negotiations, rejected Agent can only wait for this to happen. To indicate, that Proposal rejection isn't final and negotiations can be restarted later, Agent can set `golem.proposal.rejection.is-final` (bool) field in `Reason` structure. If this value is set to false, Agent can free any state related to this negotiation. The same field can be set in `Reason` sent in `Reject Agreement` operation. Requestor can send new counter Proposal after some period of time or propose the same Agreement for the second time. (No change to specification) | -| manifest-support | | | TODO | -| inet | | | TODO | -| start-entrypoint | | | TODO | +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| +| Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use `golem.srv.caps.multi-activity` property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | golem.srv.caps.multi-activity | +| Restart Proposal Negotiations | 0.7.0 | Yes | Agent is allowed to restart negotiations, by sending `Counter Proposal`, after he rejected Proposal at some point. Counter-party will receive regular `ProposalEvent` in this case. Only Agent rejecting Proposal has initiative in restarting negotiations, rejected Agent can only wait for this to happen. To indicate, that Proposal rejection isn't final and negotiations can be restarted later, Agent can set `golem.proposal.rejection.is-final` (bool) field in `Reason` structure. If this value is set to false, Agent can free any state related to this negotiation. The same field can be set in `Reason` sent in `Reject Agreement` operation. Requestor can send new counter Proposal after some period of time or propose the same Agreement for the second time. (No change to specification) | | + + +## ExeUnit + +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|----------------------------------------------| +| manifest-support | | | TODO | | +| inet | | | TODO | | +| start-entrypoint | | | TODO | | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | golem.activity.caps.transfer.report-progress | diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md new file mode 100644 index 0000000000..6850256211 --- /dev/null +++ b/docs/provider/exe-unit/command-progress.md @@ -0,0 +1,2 @@ +# Command progress reporting + From 9a11470ee6db42636c952000f88469590519abb6 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 14:55:18 +0100 Subject: [PATCH 44/52] Update capabilities for deploy --- docs/provider/capabilities.md | 14 ++++++++------ docs/provider/exe-unit/command-progress.md | 2 -- exe-unit/src/exe_unit.rs | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 docs/provider/exe-unit/command-progress.md diff --git a/docs/provider/capabilities.md b/docs/provider/capabilities.md index 7e618ab5ee..e60d8f00d1 100644 --- a/docs/provider/capabilities.md +++ b/docs/provider/capabilities.md @@ -2,6 +2,7 @@ ## Protocol + | Capability | Yagna package version | Backwards-compatible? | Description | Property | |-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| | Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use `golem.srv.caps.multi-activity` property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | golem.srv.caps.multi-activity | @@ -10,9 +11,10 @@ ## ExeUnit -| Capability | Yagna package version | Backwards-compatible? | Description | Property | -|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|----------------------------------------------| -| manifest-support | | | TODO | | -| inet | | | TODO | | -| start-entrypoint | | | TODO | | -| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | golem.activity.caps.transfer.report-progress | + +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| +| manifest-support | | | TODO | | +| inet | | | TODO | | +| start-entrypoint | | | TODO | | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | `golem.activity.caps.transfer.report-progress` `golem.activity.caps.deploy.report-progress` | diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md deleted file mode 100644 index 6850256211..0000000000 --- a/docs/provider/exe-unit/command-progress.md +++ /dev/null @@ -1,2 +0,0 @@ -# Command progress reporting - diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index 10f97c967d..086d33fc43 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -86,6 +86,7 @@ impl ExeUnit { "golem.com.usage.vector": MetricsService::usage_vector(), "golem.activity.caps.transfer.protocol": TransferService::schemes(), "golem.activity.caps.transfer.report-progress": true, + "golem.activity.caps.deploy.report-progress": true, })); Ok(supervisor_template.patch(runtime_template)) From 41b5317a00d1375c18b2a67d489688fe943316b3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 17:04:15 +0100 Subject: [PATCH 45/52] Document Exeunit progress reporting behavior --- docs/provider/exe-unit/command-progress.md | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/provider/exe-unit/command-progress.md diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md new file mode 100644 index 0000000000..99a77a147d --- /dev/null +++ b/docs/provider/exe-unit/command-progress.md @@ -0,0 +1,42 @@ +# Command progress reporting + +ExeUnit behaves according to specification defined [here](https://golemfactory.github.io/golem-architecture/specs/command-progress.html) +and support progress reporting for commands: `deploy` and `transfer`. + +This document aims to describe implementation details not covered by specification. + +## Specification + + +| Name | Description | +|-----------------------------|----------------------| +| Minimum ExeUnit version | {TODO} | +| Minimum Runtime API version | Always compatible | +| Minimum yagna version | {TODO} | +| Minimum provider version | Always compatible | +| Supported commands | `deploy`, `transfer` | + +## Runtime event + +### Steps + +`Deploy` and `transfer` command consist of only single step. + +### Progress + +- Progress is reported as `Bytes`. Fields is never a `None`. +- Size of file is always checked and put as second element of tuple. +- Initially `Size` element of tuple is set to `None` and if progress with `message` field is sent + than it can be received by Requestor agent + +### Message + +Two messages are currently possible: +- `Deployed image from cache` +- `Retry in {}s because of error: {err}` - indicates error during transfer, which will result in retry. + +When sending message, the rest of `CommandProgress` structure fields will be set to latest values. + +## Requestor Example + +PoC implementation using yapapi: https://github.com/golemfactory/yapapi/pull/1153 From edc2b042efd9ebd96c8b232d0def05c01d1c9a2d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 17:40:29 +0100 Subject: [PATCH 46/52] Describe ProgressArgs behavior --- docs/provider/exe-unit/command-progress.md | 9 ++++++++- docs/yagna/capabilities.md | 7 ++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md index 99a77a147d..2a14634f0c 100644 --- a/docs/provider/exe-unit/command-progress.md +++ b/docs/provider/exe-unit/command-progress.md @@ -16,7 +16,14 @@ This document aims to describe implementation details not covered by specificati | Minimum provider version | Always compatible | | Supported commands | `deploy`, `transfer` | -## Runtime event + +## [ProgressArgs](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#/model-ProgressArgs) + +ExeUnit supports only `update-interval`. If value is set to `1s` default will be used. + +`update-step` is not implemented. + +## [Runtime event](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#model-RuntimeEventKindProgress) ### Steps diff --git a/docs/yagna/capabilities.md b/docs/yagna/capabilities.md index 2844de7103..fd867bb1ec 100644 --- a/docs/yagna/capabilities.md +++ b/docs/yagna/capabilities.md @@ -6,6 +6,7 @@ Capabilities requiring Provider agent support are listed [here](../provider/capa ## Yagna API -| Capability | Yagna package version | Backwards-compatible? | Description | -|-------------|-----------------------|-----------------------|----------------------------------------------------------------------------| -| Cors Policy | 0.12.0 | Yes | Yagna is able to respond with Cors headers. [Spec](./capabilities/cors.md) | +| Capability | Yagna package version | Backwards-compatible? | Description | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------| +| Cors Policy | 0.12.0 | Yes | Yagna is able to respond with Cors headers. [Spec](./capabilities/cors.md) | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | \ No newline at end of file From a0aac51a004e8c310645501d1cd5db0bad75be1a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 17:42:49 +0100 Subject: [PATCH 47/52] Fix documentation --- docs/provider/exe-unit/command-progress.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md index 2a14634f0c..6fe412c985 100644 --- a/docs/provider/exe-unit/command-progress.md +++ b/docs/provider/exe-unit/command-progress.md @@ -19,7 +19,7 @@ This document aims to describe implementation details not covered by specificati ## [ProgressArgs](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#/model-ProgressArgs) -ExeUnit supports only `update-interval`. If value is set to `1s` default will be used. +ExeUnit supports only `update-interval`. If value is not set, `1s` default will be used. `update-step` is not implemented. From 9bfdd8984ffa02abf1e2fe34e1fa8a6f3dc97331 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 14 Feb 2024 13:56:27 +0100 Subject: [PATCH 48/52] Fix deb-build action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0199295814..f75c268bab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -269,7 +269,7 @@ jobs: VERSION=${TAGV_NAME#v} echo "::set-output name=tagv::${TAG_NAME}" echo "::set-output name=version::${VERSION}" - - uses: golemfactory/build-deb-action@main + - uses: golemfactory/build-deb-action@latest id: deb with: debVersion: ${{ steps.version.outputs.version }} From e39661c1db14dcb5cba20f2d09e8b8187af8d75e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 14 Feb 2024 14:39:48 +0100 Subject: [PATCH 49/52] Reference build-deb-action@v5 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f75c268bab..07508b6e79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -269,7 +269,7 @@ jobs: VERSION=${TAGV_NAME#v} echo "::set-output name=tagv::${TAG_NAME}" echo "::set-output name=version::${VERSION}" - - uses: golemfactory/build-deb-action@latest + - uses: golemfactory/build-deb-action@v5 id: deb with: debVersion: ${{ steps.version.outputs.version }} From 7096e036b9614f4e5124d75d7ca2f35a234355b8 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 23 Feb 2024 14:17:39 +0100 Subject: [PATCH 50/52] Try to build documents structure --- docs/readme.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/readme.md b/docs/readme.md index 3de0b8ae24..7407a538c0 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,4 +1,23 @@ # Developer documentation -[Provider](./provider/architecture.md) -[Payment Driver](./../core/payment-driver/erc20/Readme.md) \ No newline at end of file +- [Architecture](https://golemfactory.github.io/golem-architecture/) +- [REST API specification](https://golemfactory.github.io/ya-client/) +- Developer guides + - [Installation](./provider/overview.md#installation) + - [Logging guidelines](./logging-guidelines.md) +- Implementation documentation + - Overview + - [Provider](./provider/architecture.md) + - [ExeUnit](./provider/exe-unit/exe-units.md) + - Yagna + - Identity + - Market + - Activity + - Payments + - [Payment Driver](./../core/payment-driver/erc20/Readme.md) + - Net + - GSB + - VPN + - Runtimes + - ya-runtime-vm + - ya-runtime-wasm \ No newline at end of file From e8c7acc400376d6e55054d498bddb23f752a8179 Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Thu, 12 Sep 2024 10:50:51 +0200 Subject: [PATCH 51/52] exe-unit: progress merge fixups --- Cargo.lock | 268 +++++++++--- Cargo.toml | 433 ++++++++++---------- exe-unit/Cargo.toml | 106 ++--- exe-unit/components/mock-runtime/Cargo.toml | 30 +- exe-unit/src/exe_unit.rs | 37 +- exe-unit/src/lib.rs | 8 +- 6 files changed, 525 insertions(+), 357 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcd9a373c6..cc02d13360 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1163,6 +1163,52 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.1.18" @@ -2957,7 +3003,7 @@ dependencies = [ "tokio", "url", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-service-bus", ] @@ -3024,7 +3070,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] @@ -3061,7 +3107,7 @@ dependencies = [ "url", "ya-client", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-provider", "ya-utils-path", "ya-utils-process 0.3.0", @@ -3452,6 +3498,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime 2.1.0", + "serde", +] + [[package]] name = "hyper" version = "0.14.30" @@ -7476,6 +7532,18 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "test-binary" +version = "3.0.2" +source = "git+https://github.com/golemfactory/test-binary.git#c9ebfa3e257455f8365e042b8838a518f2106169" +dependencies = [ + "camino", + "cargo_metadata 0.15.4", + "once_cell", + "paste", + "thiserror", +] + [[package]] name = "test-case" version = "2.2.2" @@ -8765,8 +8833,8 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-gsb-http-proxy", "ya-net", "ya-persistence", @@ -8790,13 +8858,13 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] name = "ya-client" version = "0.9.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=b5369b76044f0a1584532f603c542587e15be852#b5369b76044f0a1584532f603c542587e15be852" +source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" dependencies = [ "actix-codec", "awc", @@ -8814,19 +8882,37 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.7.0", +] + +[[package]] +name = "ya-client-model" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" +dependencies = [ + "bigdecimal 0.2.2", + "chrono", + "derive_more", + "rand 0.8.5", + "serde", + "serde_json", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", ] [[package]] name = "ya-client-model" version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=b5369b76044f0a1584532f603c542587e15be852#b5369b76044f0a1584532f603c542587e15be852" +source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", "hex", + "humantime-serde", "openssl", "rand 0.8.5", "secp256k1 0.27.0", @@ -8849,6 +8935,26 @@ dependencies = [ "vergen", ] +[[package]] +name = "ya-core-model" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4e52e7d5cf8bafaf04daf6abf6bf5b199d291d65d014b7b8ca409ad7b0c599" +dependencies = [ + "chrono", + "derive_more", + "log", + "serde", + "serde_bytes", + "serde_json", + "structopt", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", + "ya-client-model 0.6.0", + "ya-service-bus", +] + [[package]] name = "ya-core-model" version = "0.10.0" @@ -8869,7 +8975,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", "ya-service-bus", ] @@ -8921,8 +9027,8 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-payment-driver", "ya-persistence", "ya-service-api-interfaces", @@ -8966,8 +9072,8 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -9007,12 +9113,14 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "sha3 0.8.2", "shell-words", "signal-hook", "socket2 0.4.10", "structopt", "tempdir", + "test-context", "thiserror", "tokio", "tokio-stream", @@ -9020,14 +9128,16 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-counters", + "ya-framework-basic", "ya-gsb-http-proxy", "ya-manifest-utils", + "ya-mock-runtime", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-sb-router", "ya-service-bus", "ya-std-utils", @@ -9072,6 +9182,7 @@ dependencies = [ "async-trait", "awc", "bytes 1.7.1", + "cargo_metadata 0.18.1", "crossterm 0.26.1", "env_logger 0.7.1", "futures 0.3.30", @@ -9089,6 +9200,7 @@ dependencies = [ "sha2 0.8.2", "sha3 0.8.2", "tempdir", + "test-binary", "test-context", "thiserror", "tokio", @@ -9097,8 +9209,8 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-net", "ya-service-bus", "ya-utils-futures", @@ -9143,8 +9255,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9187,8 +9299,8 @@ dependencies = [ "thiserror", "tokio", "uuid 1.10.0", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -9225,8 +9337,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9267,8 +9379,8 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-framework-basic", "ya-framework-mocks", "ya-persistence", @@ -9322,7 +9434,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9374,7 +9486,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-core-model", + "ya-core-model 0.10.0", "ya-diesel-utils", "ya-framework-basic", "ya-framework-mocks", @@ -9424,12 +9536,42 @@ dependencies = [ "structopt", "tokio", "url", - "ya-core-model", + "ya-core-model 0.10.0", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", ] +[[package]] +name = "ya-mock-runtime" +version = "0.1.0" +dependencies = [ + "actix", + "anyhow", + "async-trait", + "bytes 1.7.1", + "env_logger 0.10.2", + "futures 0.3.30", + "hex", + "log", + "portpicker", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "url", + "uuid 0.8.2", + "ya-client-model 0.7.0", + "ya-core-model 0.9.1", + "ya-exe-unit", + "ya-framework-basic", + "ya-runtime-api 0.7.0", + "ya-sb-router", + "ya-service-bus", +] + [[package]] name = "ya-net" version = "0.3.0" @@ -9459,8 +9601,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", @@ -9537,8 +9679,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9583,8 +9725,8 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-bus", ] @@ -9607,8 +9749,8 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-service-api", "ya-service-api-interfaces", "ya-utils-process 0.3.0", @@ -9675,9 +9817,9 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-file-logging", "ya-framework-basic", "ya-manifest-test-utils", @@ -9746,7 +9888,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.7.0", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -9830,6 +9972,25 @@ dependencies = [ "derive_more", ] +[[package]] +name = "ya-runtime-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0db25811f107d62be6c6ac7444d9c6c3e39714b6f76d72798b66ecce47506f" +dependencies = [ + "anyhow", + "bytes 1.7.1", + "futures 0.3.30", + "log", + "prost 0.10.4", + "prost-build 0.10.4", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "ya-runtime-api" version = "0.7.1" @@ -9917,7 +10078,7 @@ dependencies = [ "lazy_static", "serde", "url", - "ya-core-model", + "ya-core-model 0.10.0", "ya-utils-cli", ] @@ -9968,7 +10129,7 @@ dependencies = [ "structopt", "url", "ya-client", - "ya-core-model", + "ya-core-model 0.10.0", "ya-identity", "ya-persistence", "ya-sb-router", @@ -10011,8 +10172,8 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-service-bus", ] @@ -10096,16 +10257,17 @@ dependencies = [ "test-context", "thiserror", "tokio", + "tokio-stream", "tokio-tar", "tokio-util", "url", "walkdir", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-exe-unit", "ya-framework-basic", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-service-bus", "ya-utils-futures", "ya-utils-path", @@ -10229,7 +10391,7 @@ dependencies = [ "tokio", "ya-client", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -10264,8 +10426,8 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-persistence", @@ -10307,9 +10469,9 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-exe-unit", diff --git a/Cargo.toml b/Cargo.toml index 76fe508d37..62c16458f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,28 +1,28 @@ [package] -name = "yagna" -version = "0.16.0" -description = "Open platform and marketplace for distributed computations" -readme = "README.md" authors = ["Golem Factory "] +description = "Open platform and marketplace for distributed computations" +edition = "2018" homepage = "https://github.com/golemfactory/yagna/core/serv" -repository = "https://github.com/golemfactory/yagna" license = "GPL-3.0" -edition = "2018" +name = "yagna" +readme = "README.md" +repository = "https://github.com/golemfactory/yagna" +version = "0.16.0" [features] default = ['erc20-driver', 'gftp/bin'] -static-openssl = ["openssl/vendored", "openssl-probe"] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] +framework-test = ['ya-exe-unit/system-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] +static-openssl = ["openssl/vendored", "openssl-probe"] tos = [] -framework-test = ['ya-exe-unit/framework-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ - "ya-vpn/packet-trace-enable", - "ya-file-logging/packet-trace-enable", - "ya-net/packet-trace-enable", - "ya-service-bus/packet-trace-enable", + "ya-vpn/packet-trace-enable", + "ya-file-logging/packet-trace-enable", + "ya-net/packet-trace-enable", + "ya-service-bus/packet-trace-enable", ] [[bin]] @@ -30,40 +30,40 @@ name = "yagna" path = "core/serv/src/main.rs" [dependencies] +gftp = {workspace = true, optional = true} ya-activity = "0.4" +ya-client-model.workspace = true +ya-client.workspace = true ya-compile-time-utils = "0.2" ya-core-model.workspace = true -ya-dummy-driver = { version = "0.3", optional = true } +ya-dummy-driver = {version = "0.3", optional = true} +ya-erc20-driver = {version = "0.4", optional = true} +ya-fd-metrics = {path = "utils/fd-metrics"} ya-file-logging = "0.1" ya-gsb-api = "0.1" -ya-erc20-driver = { version = "0.4", optional = true } ya-identity = "0.3" ya-market = "0.4" ya-metrics = "0.2" -ya-net = { version = "0.3", features = ["service"] } +ya-net = {version = "0.3", features = ["service"]} ya-payment = "0.3" -ya-persistence = { version = "0.3", features = ["service"] } -ya-sb-proto = { workspace = true } -ya-sb-router = { workspace = true } +ya-persistence = {version = "0.3", features = ["service"]} +ya-sb-proto = {workspace = true} +ya-sb-router = {workspace = true} ya-service-api = "0.1" ya-service-api-derive = "0.2" ya-service-api-interfaces = "0.2" ya-service-api-web = "0.2" -ya-service-bus = { workspace = true } +ya-service-bus = {workspace = true} ya-sgx = "0.2" -ya-utils-path = "0.1" ya-utils-futures.workspace = true -ya-utils-process = { version = "0.2", features = ["lock"] } ya-utils-networking = "0.2" -ya-fd-metrics = { path = "utils/fd-metrics" } +ya-utils-path = "0.1" +ya-utils-process = {version = "0.2", features = ["lock"]} ya-version = "0.2" ya-vpn = "0.2" -ya-client.workspace = true -ya-client-model.workspace = true -gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb -ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests -ya-exe-unit = { version = "0.4", optional = true, path = "exe-unit" } # just to enable conditionally running some tests +ya-exe-unit = {version = "0.4", optional = true, path = "exe-unit"}# just to enable conditionally running some tests +ya-provider = {version = "0.3", optional = true}# just to enable conditionally running some tests actix-rt = "2.7" actix-service = "2" @@ -74,163 +74,161 @@ directories = "2.0.2" dotenv = "0.15.0" futures = "0.3" lazy_static = "1.4" +libsqlite3-sys = {workspace = true} log = "0.4" metrics = "0.12" num_cpus = "1" +openssl-probe = {version = "0.1", optional = true} openssl.workspace = true -openssl-probe = { version = "0.1", optional = true } serde = "1.0" serde_json = "1.0" structopt = "0.3" -tokio = { version = "1", features = ["net"] } -tokio-util = { version = "0.7", features = ["codec"] } -tokio-stream = { version = "0.1.8", features = ["io-util"] } +tokio = {version = "1", features = ["net"]} +tokio-stream = {version = "0.1.8", features = ["io-util"]} +tokio-util = {version = "0.7", features = ["codec"]} url = "2.1.1" -libsqlite3-sys = { workspace = true } - [dev-dependencies] -erc20_processor = { workspace = true } +erc20_processor = {workspace = true} +ya-exe-unit = {version = "0.4", path = "exe-unit"} ya-test-framework = "0.1" -ya-exe-unit = { version = "0.4", path = "exe-unit" } [package.metadata.deb] -name = "golem-requestor" assets = [ - [ - "target/release/yagna", - "usr/bin/", - "755", - ], - [ - "target/release/gftp", - "usr/bin/", - "755", - ], - [ - "README.md", - "usr/share/doc/yagna/", - "644", - ], - [ - "core/serv/README.md", - "usr/share/doc/yagna/service.md", - "644", - ], + [ + "target/release/yagna", + "usr/bin/", + "755", + ], + [ + "target/release/gftp", + "usr/bin/", + "755", + ], + [ + "README.md", + "usr/share/doc/yagna/", + "644", + ], + [ + "core/serv/README.md", + "usr/share/doc/yagna/service.md", + "644", + ], ] conflicts = "ya-provider" -features = ["static-openssl"] -maintainer-scripts = "debian/core" depends = "libgcc1, libc6 (>= 2.23)" extended-description = """The Next Golem Milestone. An open platform and marketplace for distributed computations. """ - +features = ["static-openssl"] +maintainer-scripts = "debian/core" +name = "golem-requestor" [package.metadata.deb.variants.provider] -name = "golem-provider" -replaces = "golem-requestor" -maintainer-scripts = "debian/provider" -features = ["static-openssl"] -depends = "libgcc1, libc6 (>= 2.23)" assets = [ - [ - "target/release/yagna", - "usr/bin/", - "755", - ], - [ - "target/release/ya-provider", - "usr/bin/", - "755", - ], - [ - "target/release/gftp", - "usr/bin/", - "755", - ], - [ - "target/release/exe-unit", - "usr/lib/yagna/plugins/", - "755", - ], - [ - "target/release/golemsp", - "usr/bin/", - "755", - ], - [ - "README.md", - "usr/share/doc/yagna/", - "644", - ], - [ - "core/serv/README.md", - "usr/share/doc/yagna/service.md", - "644", - ], - [ - "agent/provider/readme.md", - "usr/share/doc/yagna/run-provider.md", - "644", - ], + [ + "target/release/yagna", + "usr/bin/", + "755", + ], + [ + "target/release/ya-provider", + "usr/bin/", + "755", + ], + [ + "target/release/gftp", + "usr/bin/", + "755", + ], + [ + "target/release/exe-unit", + "usr/lib/yagna/plugins/", + "755", + ], + [ + "target/release/golemsp", + "usr/bin/", + "755", + ], + [ + "README.md", + "usr/share/doc/yagna/", + "644", + ], + [ + "core/serv/README.md", + "usr/share/doc/yagna/service.md", + "644", + ], + [ + "agent/provider/readme.md", + "usr/share/doc/yagna/run-provider.md", + "644", + ], ] +depends = "libgcc1, libc6 (>= 2.23)" +features = ["static-openssl"] +maintainer-scripts = "debian/provider" +name = "golem-provider" +replaces = "golem-requestor" [workspace.lints.clippy] arc_with_non_send_sync = "allow" -get_first = "allow" blocks_in_conditions = "allow" +get_first = "allow" [workspace] members = [ - "agent/provider", - "core/activity", - "core/gftp", - "core/gsb-api", - "core/identity", - "core/market", - "core/market/resolver", - "core/model", - "core/net", - "core/payment", - "core/payment-driver/base", - "core/payment-driver/dummy", - "core/payment-driver/erc20", - "core/persistence", - "core/serv-api", - "core/serv-api/derive", - "core/serv-api/interfaces", - "core/serv-api/web", - "core/sgx", - "core/version", - "core/vpn", - "exe-unit/components/counters", - "exe-unit/components/gsb-http-proxy", - "exe-unit", - "exe-unit/runtime-api", - "exe-unit/tokio-process-ns", - "exe-unit/components/transfer", - "golem_cli", - "utils/actix_utils", - "utils/agreement-utils", - "utils/cli", - "utils/compile-time-utils", - "utils/file-logging", - "utils/futures", - "utils/manifest-utils", - "utils/manifest-utils/test-utils", - "utils/networking", - "utils/path", - "utils/process", - "utils/std-utils", - "utils/diesel-utils", - "utils/fd-metrics", - "core/metrics", - "test-utils/test-framework", - "test-utils/test-framework/framework-macro", - "test-utils/test-framework/framework-basic", - "test-utils/test-framework/framework-mocks", + "agent/provider", + "core/activity", + "core/gftp", + "core/gsb-api", + "core/identity", + "core/market", + "core/market/resolver", + "core/model", + "core/net", + "core/payment", + "core/payment-driver/base", + "core/payment-driver/dummy", + "core/payment-driver/erc20", + "core/persistence", + "core/serv-api", + "core/serv-api/derive", + "core/serv-api/interfaces", + "core/serv-api/web", + "core/sgx", + "core/version", + "core/vpn", + "exe-unit/components/counters", + "exe-unit/components/gsb-http-proxy", + "exe-unit", + "exe-unit/runtime-api", + "exe-unit/tokio-process-ns", + "exe-unit/components/transfer", + "golem_cli", + "utils/actix_utils", + "utils/agreement-utils", + "utils/cli", + "utils/compile-time-utils", + "utils/file-logging", + "utils/futures", + "utils/manifest-utils", + "utils/manifest-utils/test-utils", + "utils/networking", + "utils/path", + "utils/process", + "utils/std-utils", + "utils/diesel-utils", + "utils/fd-metrics", + "core/metrics", + "test-utils/test-framework", + "test-utils/test-framework/framework-macro", + "test-utils/test-framework/framework-basic", + "test-utils/test-framework/framework-mocks", ] [workspace.dependencies] @@ -238,34 +236,34 @@ members = [ # diesel 1.4.* supports up to 0.23.0, but sqlx 0.5.9 requires 0.22.0 # sqlx 0.5.10 need 0.23.2, so 0.5.9 is last version possible derive_more = "0.99.11" -erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } -erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } +erc20_payment_lib = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} +erc20_processor = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} #erc20_payment_lib = { path = "../../payments/erc20_payment_lib/crates/erc20_payment_lib" } #erc20_processor = { path = "../../payments/erc20_payment_lib" } #erc20_payment_lib = { version = "0.4.7" } #erc20_processor = { version = "0.4.7" } -gftp = { version = "0.4.1", path = "core/gftp" } +gftp = {version = "0.4.1", path = "core/gftp"} hex = "0.4.3" -libsqlite3-sys = { version = "0.26.0", features = ["bundled"] } +libsqlite3-sys = {version = "0.26.0", features = ["bundled"]} openssl = "0.10" rand = "0.8.5" -strum = { version = "0.24", features = ["derive"] } +regex = "1.10.4" +strum = {version = "0.24", features = ["derive"]} trust-dns-resolver = "0.22" url = "2.3.1" -regex = "1.10.4" -ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } -ya-core-model = { version = "0.10", path = "core/model" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489" } -ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } -ya-utils-futures = { path = "utils/futures" } +ya-agreement-utils = {version = "0.6", path = "utils/agreement-utils"} +ya-core-model = {version = "0.10", path = "core/model"} +ya-relay-client = {git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489"} +ya-relay-stack = {git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5"} +ya-utils-futures = {path = "utils/futures"} -ya-service-bus = { version = "0.7.3", features = ['tls'] } -ya-sb-router = { version = "0.6.4" } -ya-sb-proto = { version = "0.6.2" } -ya-sb-util = { version = "0.5.1" } -parking_lot = "0.12.3" mime = "0.3.17" +parking_lot = "0.12.3" +ya-sb-proto = {version = "0.6.2"} +ya-sb-router = {version = "0.6.4"} +ya-sb-util = {version = "0.5.1"} +ya-service-bus = {version = "0.7.3", features = ['tls']} # true version is given in patches section ya-client = "0.9" # true version is given in patches section @@ -273,34 +271,34 @@ ya-client-model = "0.7" [patch.crates-io] ## SERVICES -ya-identity = { path = "core/identity" } -ya-net = { path = "core/net" } -ya-market = { path = "core/market" } -ya-market-resolver = { path = "core/market/resolver" } -ya-activity = { path = "core/activity" } -ya-sgx = { path = "core/sgx" } -ya-payment = { path = "core/payment" } -ya-payment-driver = { path = "core/payment-driver/base" } -ya-dummy-driver = { path = "core/payment-driver/dummy" } -ya-erc20-driver = { path = "core/payment-driver/erc20" } -ya-version = { path = "core/version" } -ya-vpn = { path = "core/vpn" } -ya-gsb-api = { path = "core/gsb-api" } +ya-activity = {path = "core/activity"} +ya-dummy-driver = {path = "core/payment-driver/dummy"} +ya-erc20-driver = {path = "core/payment-driver/erc20"} +ya-gsb-api = {path = "core/gsb-api"} +ya-identity = {path = "core/identity"} +ya-market = {path = "core/market"} +ya-market-resolver = {path = "core/market/resolver"} +ya-net = {path = "core/net"} +ya-payment = {path = "core/payment"} +ya-payment-driver = {path = "core/payment-driver/base"} +ya-sgx = {path = "core/sgx"} +ya-version = {path = "core/version"} +ya-vpn = {path = "core/vpn"} ## CORE UTILS -ya-core-model = { path = "core/model" } -ya-persistence = { path = "core/persistence" } -ya-service-api = { path = "core/serv-api" } -ya-service-api-derive = { path = "core/serv-api/derive" } -ya-service-api-interfaces = { path = "core/serv-api/interfaces" } -ya-service-api-web = { path = "core/serv-api/web" } +ya-core-model = {path = "core/model"} +ya-persistence = {path = "core/persistence"} +ya-service-api = {path = "core/serv-api"} +ya-service-api-derive = {path = "core/serv-api/derive"} +ya-service-api-interfaces = {path = "core/serv-api/interfaces"} +ya-service-api-web = {path = "core/serv-api/web"} ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad" } +ya-client = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad" } +ya-client-model = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} #ya-client-model = { path = "../ya-client/model" } -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415" } +golem-certificate = {git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415"} ## RELAY and networking stack @@ -309,39 +307,38 @@ golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.g #ya-relay-core = { path = "../ya-relay/crates/core" } #ya-relay-proto = { path = "../ya-relay/crates/proto" } - ## OTHERS -gftp = { path = "core/gftp" } -tokio-process-ns = { path = "exe-unit/tokio-process-ns" } -ya-agreement-utils = { path = "utils/agreement-utils" } -ya-std-utils = { path = "utils/std-utils" } -ya-compile-time-utils = { path = "utils/compile-time-utils" } -ya-exe-unit = { path = "exe-unit" } -ya-file-logging = { path = "utils/file-logging" } -ya-manifest-utils = { path = "utils/manifest-utils" } -ya-transfer = { path = "exe-unit/components/transfer" } -ya-utils-actix = { path = "utils/actix_utils" } -ya-utils-cli = { path = "utils/cli" } -ya-utils-networking = { path = "utils/networking" } -ya-utils-path = { path = "utils/path" } -ya-utils-process = { path = "utils/process" } -ya-diesel-utils = { path = "utils/diesel-utils" } -ya-metrics = { path = "core/metrics" } -ya-provider = { path = "agent/provider" } -ya-counters = { path = "exe-unit/components/counters" } -ya-gsb-http-proxy = { path = "exe-unit/components/gsb-http-proxy" } +gftp = {path = "core/gftp"} +tokio-process-ns = {path = "exe-unit/tokio-process-ns"} +ya-agreement-utils = {path = "utils/agreement-utils"} +ya-compile-time-utils = {path = "utils/compile-time-utils"} +ya-counters = {path = "exe-unit/components/counters"} +ya-diesel-utils = {path = "utils/diesel-utils"} +ya-exe-unit = {path = "exe-unit"} +ya-file-logging = {path = "utils/file-logging"} +ya-gsb-http-proxy = {path = "exe-unit/components/gsb-http-proxy"} +ya-manifest-utils = {path = "utils/manifest-utils"} +ya-metrics = {path = "core/metrics"} +ya-provider = {path = "agent/provider"} +ya-std-utils = {path = "utils/std-utils"} +ya-transfer = {path = "exe-unit/components/transfer"} +ya-utils-actix = {path = "utils/actix_utils"} +ya-utils-cli = {path = "utils/cli"} +ya-utils-networking = {path = "utils/networking"} +ya-utils-path = {path = "utils/path"} +ya-utils-process = {path = "utils/process"} ## TEST UTILS -ya-manifest-test-utils = { path = "utils/manifest-utils/test-utils" } -ya-test-framework = { path = "test-utils/test-framework" } -ya-framework-macro = { path = "test-utils/test-framework/framework-macro" } -ya-framework-basic = { path = "test-utils/test-framework/framework-basic" } -ya-framework-mocks = { path = "test-utils/test-framework/framework-mocks" } +ya-framework-basic = {path = "test-utils/test-framework/framework-basic"} +ya-framework-macro = {path = "test-utils/test-framework/framework-macro"} +ya-framework-mocks = {path = "test-utils/test-framework/framework-mocks"} +ya-manifest-test-utils = {path = "utils/manifest-utils/test-utils"} +ya-test-framework = {path = "test-utils/test-framework"} -ethereum-tx-sign = { git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } -graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } +ethereum-tx-sign = {git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195"} +graphene-sgx = {git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407"} -diesel = { git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563" } +diesel = {git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563"} # Speed up builds on macOS (will be default in next rust version probably) # https://jakedeichert.com/blog/reducing-rust-incremental-compilation-times-on-macos-by-70-percent/ diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 53d30476d2..ff1eb42248 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "ya-exe-unit" -version = "0.4.0" authors = ["Golem Factory "] edition = "2018" +name = "ya-exe-unit" +version = "0.4.0" [lib] name = "ya_exe_unit" @@ -13,64 +13,64 @@ name = "exe-unit" path = "src/bin.rs" [features] -default = ['compat-deployment'] compat-deployment = [] +default = ['compat-deployment'] +packet-trace-enable = ["ya-packet-trace/enable"] sgx = [ - 'graphene-sgx', - 'openssl/vendored', - 'secp256k1/rand', - 'ya-client-model/sgx', - 'ya-core-model/sgx', - 'ya-transfer/sgx', + 'graphene-sgx', + 'openssl/vendored', + 'secp256k1/rand', + 'ya-client-model/sgx', + 'ya-core-model/sgx', + 'ya-transfer/sgx', ] -packet-trace-enable = ["ya-packet-trace/enable"] system-test = ["ya-transfer/system-test"] [dependencies] -ya-agreement-utils = { workspace = true } -ya-manifest-utils = { version = "0.2" } +ya-agreement-utils = {workspace = true} ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { workspace = true, features = ["activity", "appkey"] } -ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ - "server", -] } -ya-service-bus = { workspace = true } +ya-core-model = {workspace = true, features = ["activity", "appkey"]} +ya-counters = {path = "../exe-unit/components/counters", features = ["os"]} +ya-gsb-http-proxy = {path = "../exe-unit/components/gsb-http-proxy"} +ya-manifest-utils = {version = "0.2"} +ya-packet-trace = {git = "https://github.com/golemfactory/ya-packet-trace"} +ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ + "server", +]} +ya-service-bus = {workspace = true} +ya-std-utils = "0.1" ya-transfer = "0.3" -ya-utils-path = "0.1" ya-utils-futures.workspace = true -ya-std-utils = "0.1" -ya-utils-networking = { version = "0.2", default-features = false, features = [ - "dns", - "vpn", -] } +ya-utils-networking = {version = "0.2", default-features = false, features = [ + "dns", + "vpn", +]} +ya-utils-path = "0.1" ya-utils-process = "0.3" -ya-packet-trace = { git = "https://github.com/golemfactory/ya-packet-trace" } -ya-gsb-http-proxy = { path = "../exe-unit/components/gsb-http-proxy" } -ya-counters = { path = "../exe-unit/components/counters", features = ["os"] } -actix = { version = "0.13", default-features = false } +actix = {version = "0.13", default-features = false} actix-rt = "2.7" anyhow = "1.0" async-trait = "0.1.24" bytes = "1" chrono = "0.4" derivative = "2.1" -derive_more = { workspace = true } +derive_more = {workspace = true} dotenv = "0.15.0" -flexi_logger = { version = "0.22", features = ["colors"] } +flexi_logger = {version = "0.22", features = ["colors"]} futures = "0.3" -graphene-sgx = { version = "0.3.3", optional = true } +graphene-sgx = {version = "0.3.3", optional = true} hex = "0.4.2" ipnet = "2.3" lazy_static = "1.4.0" log = "0.4" -openssl = { workspace = true, optional = true } +openssl = {workspace = true, optional = true} rand = "0.8.5" regex = "1.5" -reqwest = { version = "0.11", optional = false, features = ["stream"] } -secp256k1 = { version = "0.27.0", optional = true } -serde = { version = "^1.0", features = ["derive"] } +reqwest = {version = "0.11", optional = false, features = ["stream"]} +secp256k1 = {version = "0.27.0", optional = true} +serde = {version = "^1.0", features = ["derive"]} serde_json = "1.0" serde_yaml = "0.8" sha3 = "0.8.2" @@ -79,39 +79,39 @@ socket2 = "0.4" structopt = "0.3" thiserror = "1.0" # keep the "rt-multi-thread" feature -tokio = { version = "1", features = [ - "process", - "signal", - "time", - "net", - "rt-multi-thread", -] } -tokio-util = { version = "0.7.2", features = ["codec", "net"] } -tokio-stream = { version = "0.1.8", features = ["io-util", "sync"] } +async-stream = "0.3.5" +tokio = {version = "1", features = [ + "process", + "signal", + "time", + "net", + "rt-multi-thread", +]} +tokio-stream = {version = "0.1.8", features = ["io-util", "sync"]} +tokio-util = {version = "0.7.2", features = ["codec", "net"]} +trust-dns-resolver = {workspace = true} url = "2.1" yansi = "0.5.0" -trust-dns-resolver = { workspace = true } -async-stream = "0.3.5" [dev-dependencies] -ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ - "codec", - "server", -] } -ya-sb-router = { workspace = true } +ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ + "codec", + "server", +]} +ya-sb-router = {workspace = true} actix-files = "0.6" actix-web = "4" env_logger = "0.7" rustyline = "7.0.0" -serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} +serial_test = {git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} sha3 = "0.8.2" shell-words = "1.0.0" tempdir = "0.3.7" test-context = "0.1.4" -ya-framework-basic = { version = "0.1" } -ya-mock-runtime = { path = "components/mock-runtime"} +ya-framework-basic = {version = "0.1"} +ya-mock-runtime = {path = "components/mock-runtime"} [lints] workspace = true diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 5055bb4a03..13921a1367 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -1,10 +1,9 @@ [package] -name = "ya-mock-runtime" -version = "0.1.0" authors = ["Golem Factory "] -edition = "2021" description = "Mock runtime for testing purposes and set of libraries for testing ExeUnits in tests." - +edition = "2021" +name = "ya-mock-runtime" +version = "0.1.0" [lib] path = "src/lib.rs" @@ -13,33 +12,32 @@ path = "src/lib.rs" name = "ya-mock-runtime" path = "src/main.rs" - [dependencies] anyhow = "1.0.31" bytes = "1.0" -futures = { version = "0.3" } -log = "0.4" env_logger = "0.10" -serde = { version = "1.0", features = ["derive"] } +futures = {version = "0.3"} +log = "0.4" +serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"] } -tokio-util = { version = "0.7", features = ["codec"] } +tokio = {version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"]} +tokio-util = {version = "0.7", features = ["codec"]} url = "2.3" ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils +ya-client-model.workspace = true +ya-core-model = {version = "^0.9", features = ["activity", "appkey"]} ya-exe-unit = "0.4" -ya-framework-basic = { version = "0.1" } -ya-client-model = "0.6" -ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } -ya-service-bus = { workspace = true } +ya-framework-basic = {version = "0.1"} ya-sb-router = "0.6" +ya-service-bus = {workspace = true} +actix = {version = "0.13", default-features = false} async-trait = "0.1.77" -actix = { version = "0.13", default-features = false } hex = "0.4.3" portpicker = "0.1.1" rand = "0.8.5" -uuid = { version = "0.8.2", features = ["v4"] } +uuid = {version = "0.8.2", features = ["v4"]} diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index 086d33fc43..5b8936ace3 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -9,6 +9,9 @@ use futures::{FutureExt, SinkExt}; use std::path::PathBuf; use std::time::Duration; use tokio::sync::broadcast; +use ya_counters::error::CounterError; +use ya_counters::message::GetCounters; +use ya_counters::service::CountersService; use ya_agreement_utils::OfferTemplate; use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; @@ -26,12 +29,11 @@ use crate::acl::Acl; use crate::agreement::Agreement; use crate::error::Error; use crate::message::{ - ExecuteCommand, GetMetrics, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, - ShutdownReason, SignExeScript, Stop, UpdateDeployment, + ExecuteCommand, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, ShutdownReason, + SignExeScript, Stop, UpdateDeployment, }; use crate::runtime::{Runtime, RuntimeMode}; -use crate::service::metrics::MetricsService; -use crate::service::{ServiceAddr, ServiceControl}; +use crate::service::{self, ServiceAddr, ServiceControl}; use crate::state::{ExeUnitState, StateError, Supervision}; use crate::Result; @@ -48,7 +50,7 @@ pub struct ExeUnit { pub(crate) state: ExeUnitState, pub(crate) events: Channel, pub(crate) runtime: Addr, - pub(crate) metrics: Addr, + pub(crate) counters: Addr, pub(crate) transfers: Addr, pub(crate) services: Vec>, pub(crate) shutdown_tx: broadcast::Sender<()>, @@ -57,7 +59,7 @@ pub struct ExeUnit { impl ExeUnit { pub fn new( ctx: ExeUnitContext, - metrics: Addr, + counters: Addr, transfers: Addr, runtime: Addr, ) -> Self { @@ -67,10 +69,10 @@ impl ExeUnit { state: ExeUnitState::default(), events: Channel::default(), runtime: runtime.clone(), - metrics: metrics.clone(), + counters: counters.clone(), transfers: transfers.clone(), services: vec![ - Box::new(ServiceAddr::new(metrics)), + Box::new(ServiceAddr::new(counters)), Box::new(ServiceAddr::new(transfers)), Box::new(ServiceAddr::new(runtime)), ], @@ -83,7 +85,7 @@ impl ExeUnit { let runtime_template = RuntimeProcess::offer_template(binary, args)?; let supervisor_template = OfferTemplate::new(serde_json::json!({ - "golem.com.usage.vector": MetricsService::usage_vector(), + "golem.com.usage.vector": service::counters::usage_vector(), "golem.activity.caps.transfer.protocol": TransferService::schemes(), "golem.activity.caps.transfer.report-progress": true, "golem.activity.caps.deploy.report-progress": true, @@ -105,7 +107,7 @@ impl ExeUnit { self.ctx.report_url.clone().unwrap(), self.ctx.activity_id.clone().unwrap(), context.address(), - self.metrics.clone(), + self.counters.clone(), ); context.spawn(fut.into_actor(self)); } @@ -555,9 +557,9 @@ async fn report_usage( report_url: String, activity_id: String, exe_unit: Addr>, - metrics: Addr, + metrics: Addr, ) { - match metrics.send(GetMetrics).await { + match metrics.send(GetCounters).await { Ok(resp) => match resp { Ok(data) => { let msg = activity::local::SetUsage { @@ -575,7 +577,7 @@ async fn report_usage( } } Err(err) => match err { - Error::UsageLimitExceeded(info) => { + CounterError::UsageLimitExceeded(info) => { log::warn!("Usage limit exceeded: {}", info); exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); } @@ -602,3 +604,12 @@ impl Handler for TransferService { async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() } } + +impl Handler for CountersService { + type Result = ResponseFuture>; + + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + let addr = ctx.address(); + async move { Ok(addr.send(ya_counters::message::Shutdown {}).await??) }.boxed_local() + } +} diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 7d9ac9e7db..d76bac574a 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -6,6 +6,7 @@ use anyhow::{bail, Context}; use std::convert::TryFrom; use std::path::PathBuf; use structopt::clap; +use ya_counters::service::CountersServiceBuilder; use ya_client_model::activity::ExeScriptCommand; use ya_core_model::activity; @@ -18,7 +19,6 @@ use crate::error::Error; use crate::manifest::ManifestContext; use crate::message::{GetState, GetStateResponse, Register}; use crate::runtime::process::RuntimeProcess; -use crate::service::metrics::MetricsService; use crate::service::signal::SignalMonitor; use crate::state::Supervision; @@ -31,11 +31,9 @@ mod handlers; pub mod logger; pub mod manifest; pub mod message; -pub mod metrics; mod network; mod notify; mod output; -pub mod process; pub mod runtime; pub mod service; pub mod state; @@ -371,7 +369,9 @@ pub async fn exe_unit(mut config: ExeUnitConfig) -> anyhow::Result Date: Thu, 12 Sep 2024 11:40:10 +0200 Subject: [PATCH 52/52] exe-unit: fix mock-runtime dependencies --- Cargo.lock | 139 +++++++------------- exe-unit/components/mock-runtime/Cargo.toml | 2 +- 2 files changed, 52 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc02d13360..599776fd2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3003,7 +3003,7 @@ dependencies = [ "tokio", "url", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-service-bus", ] @@ -3070,7 +3070,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model 0.7.0", + "ya-client-model", ] [[package]] @@ -3107,7 +3107,7 @@ dependencies = [ "url", "ya-client", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-provider", "ya-utils-path", "ya-utils-process 0.3.0", @@ -8833,8 +8833,8 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-gsb-http-proxy", "ya-net", "ya-persistence", @@ -8858,7 +8858,7 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", ] [[package]] @@ -8882,24 +8882,7 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model 0.7.0", -] - -[[package]] -name = "ya-client-model" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" -dependencies = [ - "bigdecimal 0.2.2", - "chrono", - "derive_more", - "rand 0.8.5", - "serde", - "serde_json", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", + "ya-client-model", ] [[package]] @@ -8935,26 +8918,6 @@ dependencies = [ "vergen", ] -[[package]] -name = "ya-core-model" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed4e52e7d5cf8bafaf04daf6abf6bf5b199d291d65d014b7b8ca409ad7b0c599" -dependencies = [ - "chrono", - "derive_more", - "log", - "serde", - "serde_bytes", - "serde_json", - "structopt", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", - "ya-client-model 0.6.0", - "ya-service-bus", -] - [[package]] name = "ya-core-model" version = "0.10.0" @@ -8975,7 +8938,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", "ya-service-bus", ] @@ -9027,8 +8990,8 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-payment-driver", "ya-persistence", "ya-service-api-interfaces", @@ -9072,8 +9035,8 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -9128,9 +9091,9 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-counters", "ya-framework-basic", "ya-gsb-http-proxy", @@ -9209,8 +9172,8 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-net", "ya-service-bus", "ya-utils-futures", @@ -9255,8 +9218,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9299,8 +9262,8 @@ dependencies = [ "thiserror", "tokio", "uuid 1.10.0", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -9337,8 +9300,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9379,8 +9342,8 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-framework-basic", "ya-framework-mocks", "ya-persistence", @@ -9434,7 +9397,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9486,7 +9449,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-core-model 0.10.0", + "ya-core-model", "ya-diesel-utils", "ya-framework-basic", "ya-framework-mocks", @@ -9536,7 +9499,7 @@ dependencies = [ "structopt", "tokio", "url", - "ya-core-model 0.10.0", + "ya-core-model", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", @@ -9563,8 +9526,8 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.9.1", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.0", @@ -9601,8 +9564,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", @@ -9679,8 +9642,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9725,8 +9688,8 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-persistence", "ya-service-bus", ] @@ -9749,8 +9712,8 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-service-api", "ya-service-api-interfaces", "ya-utils-process 0.3.0", @@ -9817,9 +9780,9 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-file-logging", "ya-framework-basic", "ya-manifest-test-utils", @@ -9888,7 +9851,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.7.0", + "ya-client-model", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -10078,7 +10041,7 @@ dependencies = [ "lazy_static", "serde", "url", - "ya-core-model 0.10.0", + "ya-core-model", "ya-utils-cli", ] @@ -10129,7 +10092,7 @@ dependencies = [ "structopt", "url", "ya-client", - "ya-core-model 0.10.0", + "ya-core-model", "ya-identity", "ya-persistence", "ya-sb-router", @@ -10172,8 +10135,8 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-service-bus", ] @@ -10263,8 +10226,8 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.1", @@ -10391,7 +10354,7 @@ dependencies = [ "tokio", "ya-client", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -10426,8 +10389,8 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-persistence", @@ -10469,9 +10432,9 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-exe-unit", diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 13921a1367..564a6e5e67 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -29,7 +29,7 @@ ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils ya-client-model.workspace = true -ya-core-model = {version = "^0.9", features = ["activity", "appkey"]} +ya-core-model = {workspace = true, features = ["activity", "appkey"]} ya-exe-unit = "0.4" ya-framework-basic = {version = "0.1"} ya-sb-router = "0.6"