diff --git a/CHANGELOG.md b/CHANGELOG.md index f04e92590c..b2d0e4fcdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### fix: `dfx canister install` and `dfx deploy` with `--no-asset-upgrade` no longer hang indefinitely when wasm is not up to date +### feat: streamlined output during asset synchronization + # 0.25.0 ### fix: correctly detects hyphenated Rust bin crates diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index 787c0024a8..fdb1294869 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -905,14 +905,14 @@ check_permission_failure() { dd if=/dev/urandom of=src/e2e_project_frontend/assets/asset2.bin bs=400000 count=1 dfx_start - assert_command dfx deploy + assert_command dfx deploy -v assert_match '/asset1.bin 1/1' assert_match '/asset2.bin 1/1' dd if=/dev/urandom of=src/e2e_project_frontend/assets/asset2.bin bs=400000 count=1 - assert_command dfx deploy + assert_command dfx deploy -v assert_match '/asset1.bin.*is already installed' assert_match '/asset2.bin 1/1' } @@ -1649,7 +1649,7 @@ EOF assert_match "200 OK" "$stderr" # redirect survives upgrade - assert_command dfx deploy --upgrade-unchanged + assert_command dfx deploy --upgrade-unchanged -v assert_match "is already installed" assert_command curl --fail -vv http://localhost:"$PORT"/test_alias_file.html?canisterId="$ID" @@ -1922,7 +1922,7 @@ WARN: { }, ]' > src/e2e_project_frontend/assets/somedir/.ic-assets.json5 - assert_command dfx deploy + assert_command dfx deploy -v assert_match '/somedir/upload-me.txt 1/1 \(8 bytes\) sha [0-9a-z]* \(with cache and 1 header\)' } diff --git a/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs b/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs index bd8d367d37..417496200f 100644 --- a/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs +++ b/src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs @@ -10,12 +10,13 @@ use crate::error::CreateEncodingError; use crate::error::CreateEncodingError::EncodeContentFailed; use crate::error::CreateProjectAssetError; use crate::error::SetEncodingError; +use crate::AssetSyncProgressRenderer; use candid::Nat; use futures::future::try_join_all; use futures::TryFutureExt; use ic_utils::Canister; use mime::Mime; -use slog::{debug, info, Logger}; +use slog::{debug, Logger}; use std::collections::BTreeMap; use std::collections::HashMap; use std::path::PathBuf; @@ -89,14 +90,22 @@ impl<'agent> ChunkUploader<'agent> { &self, contents: &[u8], semaphores: &Semaphores, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { let uploader_chunk_id = self.chunks.fetch_add(1, Ordering::SeqCst); self.bytes.fetch_add(contents.len(), Ordering::SeqCst); if contents.len() == MAX_CHUNK_SIZE || self.api_version < 2 { - let canister_chunk_id = - create_chunk(&self.canister, &self.batch_id, contents, semaphores).await?; + let canister_chunk_id = create_chunk( + &self.canister, + &self.batch_id, + contents, + semaphores, + progress, + ) + .await?; let mut map = self.id_mapping.lock().await; map.insert(uploader_chunk_id, canister_chunk_id); + Ok(uploader_chunk_id) } else { self.add_to_upload_queue(uploader_chunk_id, contents).await; @@ -106,7 +115,8 @@ impl<'agent> ChunkUploader<'agent> { // - Tested with: `for i in $(seq 1 50); do dd if=/dev/urandom of="src/hello_frontend/assets/file_$i.bin" bs=$(shuf -i 1-2000000 -n 1) count=1; done && dfx deploy hello_frontend` // - Result: Roughly 15% of batches under 90% full. // With other byte ranges (e.g. `shuf -i 1-3000000 -n 1`) stats improve significantly - self.upload_chunks(4 * MAX_CHUNK_SIZE, semaphores).await?; + self.upload_chunks(4 * MAX_CHUNK_SIZE, semaphores, progress) + .await?; Ok(uploader_chunk_id) } } @@ -115,6 +125,7 @@ impl<'agent> ChunkUploader<'agent> { &self, semaphores: &Semaphores, mode: Mode, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result<(), CreateChunkError> { let max_retained_bytes = if mode == Mode::ByProposal { // Never add data to the commit_batch args, because they have to fit in a single call. @@ -124,7 +135,8 @@ impl<'agent> ChunkUploader<'agent> { MAX_CHUNK_SIZE / 2 }; - self.upload_chunks(max_retained_bytes, semaphores).await + self.upload_chunks(max_retained_bytes, semaphores, progress) + .await } pub(crate) fn bytes(&self) -> usize { @@ -174,6 +186,7 @@ impl<'agent> ChunkUploader<'agent> { &self, max_retained_bytes: usize, semaphores: &Semaphores, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result<(), CreateChunkError> { let mut queue = self.upload_queue.lock().await; @@ -202,7 +215,7 @@ impl<'agent> ChunkUploader<'agent> { try_join_all(batches.into_iter().map(|chunks| async move { let (uploader_chunk_ids, chunks): (Vec<_>, Vec<_>) = chunks.into_iter().unzip(); let canister_chunk_ids = - create_chunks(&self.canister, &self.batch_id, chunks, semaphores).await?; + create_chunks(&self.canister, &self.batch_id, chunks, semaphores, progress).await?; let mut map = self.id_mapping.lock().await; for (uploader_id, canister_id) in uploader_chunk_ids .into_iter() @@ -227,6 +240,7 @@ async fn make_project_asset_encoding( content_encoding: &str, semaphores: &Semaphores, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { let sha256 = content.sha256(); @@ -249,7 +263,7 @@ async fn make_project_asset_encoding( }; let uploader_chunk_ids = if already_in_place { - info!( + debug!( logger, " {}{} ({} bytes) sha {} is already installed", &asset_descriptor.key, @@ -267,10 +281,11 @@ async fn make_project_asset_encoding( content_encoding, semaphores, logger, + progress, ) .await? } else { - info!( + debug!( logger, " {}{} ({} bytes) sha {} will be uploaded", &asset_descriptor.key, @@ -298,6 +313,7 @@ async fn make_encoding( force_encoding: bool, semaphores: &Semaphores, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result, CreateEncodingError> { match encoder { ContentEncoder::Identity => { @@ -309,6 +325,7 @@ async fn make_encoding( CONTENT_ENCODING_IDENTITY, semaphores, logger, + progress, ) .await .map_err(CreateEncodingError::CreateChunkFailed)?; @@ -331,6 +348,7 @@ async fn make_encoding( &content_encoding, semaphores, logger, + progress, ) .await .map_err(CreateEncodingError::CreateChunkFailed)?; @@ -349,12 +367,14 @@ async fn make_encodings( content: &Content, semaphores: &Semaphores, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result, CreateEncodingError> { let encoders = asset_descriptor .config .encodings .clone() .unwrap_or_else(|| default_encoders(&content.media_type)); + // The identity encoding is always uploaded if it's in the list of chosen encodings. // Other encoding are only uploaded if they save bytes compared to identity. // The encoding is forced through the filter if there is no identity encoding to compare against. @@ -372,6 +392,7 @@ async fn make_encodings( force_encoding, semaphores, logger, + progress, ) }) .collect(); @@ -392,6 +413,7 @@ async fn make_project_asset( canister_assets: &HashMap, semaphores: &Semaphores, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { let file_size = dfx_core::fs::metadata(&asset_descriptor.source)?.len(); let permits = std::cmp::max( @@ -412,9 +434,13 @@ async fn make_project_asset( &content, semaphores, logger, + progress, ) .await?; + if let Some(progress) = progress { + progress.increment_complete_assets(); + } Ok(ProjectAsset { asset_descriptor, media_type: content.media_type, @@ -428,9 +454,13 @@ pub(crate) async fn make_project_assets( canister_assets: &HashMap, mode: Mode, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result, CreateProjectAssetError> { let semaphores = Semaphores::new(); + if let Some(progress) = progress { + progress.set_total_assets(asset_descriptors.len()); + } let project_asset_futures: Vec<_> = asset_descriptors .iter() .map(|loc| { @@ -440,13 +470,14 @@ pub(crate) async fn make_project_assets( canister_assets, &semaphores, logger, + progress, ) }) .collect(); let project_assets = try_join_all(project_asset_futures).await?; if let Some(uploader) = chunk_upload_target { uploader - .finalize_upload(&semaphores, mode) + .finalize_upload(&semaphores, mode, progress) .await .map_err(|err| { CreateProjectAssetError::CreateEncodingError( @@ -470,11 +501,14 @@ async fn upload_content_chunks( content_encoding: &str, semaphores: &Semaphores, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result, CreateChunkError> { if content.data.is_empty() { let empty = vec![]; - let chunk_id = chunk_uploader.create_chunk(&empty, semaphores).await?; - info!( + let chunk_id = chunk_uploader + .create_chunk(&empty, semaphores, progress) + .await?; + debug!( logger, " {}{} 1/1 (0 bytes) sha {}", &asset_descriptor.key, @@ -484,6 +518,9 @@ async fn upload_content_chunks( return Ok(vec![chunk_id]); } + if let Some(progress) = progress { + progress.add_total_bytes(content.data.len()); + } let count = (content.data.len() + MAX_CHUNK_SIZE - 1) / MAX_CHUNK_SIZE; let chunks_futures: Vec<_> = content .data @@ -491,9 +528,9 @@ async fn upload_content_chunks( .enumerate() .map(|(i, data_chunk)| { chunk_uploader - .create_chunk(data_chunk, semaphores) + .create_chunk(data_chunk, semaphores, progress) .map_ok(move |chunk_id| { - info!( + debug!( logger, " {}{} {}/{} ({} bytes) sha {} {}", &asset_descriptor.key, diff --git a/src/canisters/frontend/ic-asset/src/canister_api/methods/asset_properties.rs b/src/canisters/frontend/ic-asset/src/canister_api/methods/asset_properties.rs index 39d37a5610..1c6169a2e3 100644 --- a/src/canisters/frontend/ic-asset/src/canister_api/methods/asset_properties.rs +++ b/src/canisters/frontend/ic-asset/src/canister_api/methods/asset_properties.rs @@ -6,6 +6,7 @@ use crate::{ methods::method_names::GET_ASSET_PROPERTIES, types::asset::{AssetDetails, AssetProperties, GetAssetPropertiesArgument}, }, + AssetSyncProgressRenderer, }; use backoff::backoff::Backoff; use backoff::ExponentialBackoffBuilder; @@ -20,9 +21,13 @@ const MAX_CONCURRENT_REQUESTS: usize = 50; pub(crate) async fn get_assets_properties( canister: &Canister<'_>, canister_assets: &HashMap, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result, GetAssetPropertiesError> { let semaphore = SharedSemaphore::new(true, MAX_CONCURRENT_REQUESTS); + if let Some(progress) = progress { + progress.set_asset_properties_to_retrieve(canister_assets.len()); + } let asset_ids = canister_assets.keys().cloned().collect::>(); let futs = asset_ids .iter() @@ -40,7 +45,12 @@ pub(crate) async fn get_assets_properties( let response = get_asset_properties(canister, asset_id).await; match response { - Ok(asset_properties) => break Ok(asset_properties), + Ok(asset_properties) => { + if let Some(progress) = progress { + progress.inc_asset_properties_retrieved(); + } + break Ok(asset_properties); + } Err(agent_err) if !retryable(&agent_err) => { break Err(agent_err); } diff --git a/src/canisters/frontend/ic-asset/src/canister_api/methods/chunk.rs b/src/canisters/frontend/ic-asset/src/canister_api/methods/chunk.rs index 79c6ea1096..190f799f15 100644 --- a/src/canisters/frontend/ic-asset/src/canister_api/methods/chunk.rs +++ b/src/canisters/frontend/ic-asset/src/canister_api/methods/chunk.rs @@ -5,6 +5,7 @@ use crate::canister_api::types::batch_upload::common::{ CreateChunkRequest, CreateChunkResponse, CreateChunksRequest, CreateChunksResponse, }; use crate::error::CreateChunkError; +use crate::AssetSyncProgressRenderer; use backoff::backoff::Backoff; use backoff::ExponentialBackoffBuilder; use candid::{Decode, Nat}; @@ -19,6 +20,7 @@ pub(crate) async fn create_chunk( batch_id: &Nat, content: &[u8], semaphores: &Semaphores, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { let _chunk_releaser = semaphores.create_chunk.acquire(1).await; let batch_id = batch_id.clone(); @@ -58,6 +60,9 @@ pub(crate) async fn create_chunk( match wait_result { Ok((chunk_id,)) => { + if let Some(progress) = progress { + progress.add_uploaded_bytes(content.len()); + } return Ok(chunk_id); } Err(agent_err) if !retryable(&agent_err) => { @@ -76,7 +81,9 @@ pub(crate) async fn create_chunks( batch_id: &Nat, content: Vec>, semaphores: &Semaphores, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result, CreateChunkError> { + let content_byte_len = content.iter().fold(0, |acc, x| acc + x.len()); let _chunk_releaser = semaphores.create_chunk.acquire(1).await; let batch_id = batch_id.clone(); let args = CreateChunksRequest { batch_id, content }; @@ -115,6 +122,9 @@ pub(crate) async fn create_chunks( match wait_result { Ok((chunk_ids,)) => { + if let Some(progress) = progress { + progress.add_uploaded_bytes(content_byte_len); + } return Ok(chunk_ids); } Err(agent_err) if !retryable(&agent_err) => { diff --git a/src/canisters/frontend/ic-asset/src/evidence/mod.rs b/src/canisters/frontend/ic-asset/src/evidence/mod.rs index 47ce78291a..8f840098d1 100644 --- a/src/canisters/frontend/ic-asset/src/evidence/mod.rs +++ b/src/canisters/frontend/ic-asset/src/evidence/mod.rs @@ -15,6 +15,7 @@ use crate::error::ComputeEvidenceError; use crate::error::HashContentError; use crate::error::HashContentError::EncodeContentFailed; use crate::sync::gather_asset_descriptors; +use crate::AssetSyncProgressRenderer; use ic_utils::Canister; use sha2::{Digest, Sha256}; use slog::{info, trace, Logger}; @@ -39,6 +40,7 @@ pub async fn compute_evidence( canister: &Canister<'_>, dirs: &[&Path], logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { let asset_descriptors = gather_asset_descriptors(dirs, logger)?; @@ -49,18 +51,21 @@ pub async fn compute_evidence( logger, "Fetching properties for all assets in the canister." ); - let canister_asset_properties = get_assets_properties(canister, &canister_assets).await?; + let canister_asset_properties = + get_assets_properties(canister, &canister_assets, progress).await?; info!( logger, "Computing evidence for batch operations for assets in the project.", ); + let project_assets = make_project_assets( None, asset_descriptors, &canister_assets, crate::batch_upload::plumbing::Mode::ByProposal, logger, + progress, ) .await?; diff --git a/src/canisters/frontend/ic-asset/src/lib.rs b/src/canisters/frontend/ic-asset/src/lib.rs index 87aa17a3cb..fa92caddc7 100644 --- a/src/canisters/frontend/ic-asset/src/lib.rs +++ b/src/canisters/frontend/ic-asset/src/lib.rs @@ -20,7 +20,7 @@ //! .with_agent(&agent) //! .build()?; //! let logger = slog::Logger::root(slog::Discard, slog::o!()); -//! ic_asset::sync(&canister, &[concat!(env!("CARGO_MANIFEST_DIR"), "assets/").as_ref()], false, &logger).await?; +//! ic_asset::sync(&canister, &[concat!(env!("CARGO_MANIFEST_DIR"), "assets/").as_ref()], false, &logger, None).await?; //! # Ok(()) //! # } @@ -36,11 +36,13 @@ mod batch_upload; mod canister_api; pub mod error; mod evidence; +mod progress; pub mod security_policy; mod sync; mod upload; pub use evidence::compute_evidence; +pub use progress::{AssetSyncProgressRenderer, AssetSyncState}; pub use sync::prepare_sync_for_proposal; pub use sync::sync; pub use upload::upload; diff --git a/src/canisters/frontend/ic-asset/src/progress.rs b/src/canisters/frontend/ic-asset/src/progress.rs new file mode 100644 index 0000000000..9872cb087b --- /dev/null +++ b/src/canisters/frontend/ic-asset/src/progress.rs @@ -0,0 +1,57 @@ +/// . +#[derive(Debug)] +pub enum AssetSyncState { + /// Walk the source directories and build a list of assets to sync + GatherAssetDescriptors, + + /// List all assets already in the canister + ListAssets, + + /// Get properties of all assets already in the canister + GetAssetProperties, + + /// Create the batch + CreateBatch, + + /// Upload content encodings (chunks) + StageContents, + + /// Build the list of operations to apply all changes + AssembleBatch, + + /// Commit (execute) batch operations + CommitBatch, + + /// All done + Done, +} + +/// Display progress of the synchronization process +pub trait AssetSyncProgressRenderer { + /// Set the current state of the synchronization process + fn set_state(&self, state: AssetSyncState); + + /// Set the total number of assets to get properties for + fn set_asset_properties_to_retrieve(&self, total: usize); + + /// Increment the number of assets for which properties have been retrieved + fn inc_asset_properties_retrieved(&self); + + /// Set the total number of assets to sync + fn set_total_assets(&self, total: usize); + + /// Increment the number of assets that have been synced + fn increment_complete_assets(&self); + + /// Set the total number of bytes to upload + fn add_total_bytes(&self, add: usize); + + /// Increment the number of bytes that have been uploaded + fn add_uploaded_bytes(&self, add: usize); + + /// Set the total number of batch operations + fn set_total_batch_operations(&self, total: usize); + + /// Increase the number of batch operations that have been committed + fn add_committed_batch_operations(&self, add: usize); +} diff --git a/src/canisters/frontend/ic-asset/src/sync.rs b/src/canisters/frontend/ic-asset/src/sync.rs index 12fc27f489..b17379e4ad 100644 --- a/src/canisters/frontend/ic-asset/src/sync.rs +++ b/src/canisters/frontend/ic-asset/src/sync.rs @@ -31,6 +31,7 @@ use crate::error::SyncError; use crate::error::SyncError::CommitBatchFailed; use crate::error::UploadContentError; use crate::error::UploadContentError::{CreateBatchFailed, ListAssetsFailed}; +use crate::progress::{AssetSyncProgressRenderer, AssetSyncState}; use candid::Nat; use ic_agent::AgentError; use ic_utils::Canister; @@ -51,28 +52,42 @@ pub async fn upload_content_and_assemble_sync_operations( no_delete: bool, mode: batch_upload::plumbing::Mode, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result { + if let Some(progress) = progress { + progress.set_state(AssetSyncState::GatherAssetDescriptors); + } + let asset_descriptors = gather_asset_descriptors(dirs, logger)?; + if let Some(progress) = progress { + progress.set_state(AssetSyncState::ListAssets); + } let canister_assets = list_assets(canister).await.map_err(ListAssetsFailed)?; - info!( + debug!( logger, "Fetching properties for all assets in the canister." ); let now = std::time::Instant::now(); - let canister_asset_properties = get_assets_properties(canister, &canister_assets).await?; + if let Some(progress) = progress { + progress.set_state(AssetSyncState::GetAssetProperties); + } + let canister_asset_properties = + get_assets_properties(canister, &canister_assets, progress).await?; - info!( + debug!( logger, - "Done fetching properties for all assets in the canister. Took {:?}", + "Fetched properties for all assets in the canister in {:?}", now.elapsed() ); - info!(logger, "Starting batch."); + if let Some(progress) = progress { + progress.set_state(AssetSyncState::CreateBatch); + } let batch_id = create_batch(canister).await.map_err(CreateBatchFailed)?; - info!( + debug!( logger, "Staging contents of new and changed assets in batch {}:", batch_id ); @@ -80,16 +95,25 @@ pub async fn upload_content_and_assemble_sync_operations( let chunk_uploader = ChunkUploader::new(canister.clone(), canister_api_version, batch_id.clone()); + if let Some(progress) = progress { + progress.set_state(AssetSyncState::StageContents); + } + let project_assets = make_project_assets( Some(&chunk_uploader), asset_descriptors, &canister_assets, mode, logger, + progress, ) .await .map_err(UploadContentError::CreateProjectAssetError)?; + if let Some(progress) = progress { + progress.set_state(AssetSyncState::AssembleBatch); + } + let commit_batch_args = batch_upload::operations::assemble_commit_batch_arguments( &chunk_uploader, project_assets, @@ -129,6 +153,7 @@ pub async fn sync( dirs: &[&Path], no_delete: bool, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result<(), SyncError> { let canister_api_version = api_version(canister).await; let commit_batch_args = upload_content_and_assemble_sync_operations( @@ -138,25 +163,37 @@ pub async fn sync( no_delete, NormalDeploy, logger, + progress, ) .await?; debug!(logger, "Canister API version: {canister_api_version}. ic-asset API version: {BATCH_UPLOAD_API_VERSION}"); - info!(logger, "Committing batch."); + debug!(logger, "Committing batch."); + if let Some(progress) = progress { + progress.set_state(AssetSyncState::CommitBatch); + } match canister_api_version { 0 => { let commit_batch_args_v0 = v0::CommitBatchArguments::try_from(commit_batch_args).map_err(DowngradeV1TOV0Failed)?; warn!(logger, "The asset canister is running an old version of the API. It will not be able to set assets properties."); commit_batch(canister, commit_batch_args_v0).await } - BATCH_UPLOAD_API_VERSION.. => commit_in_stages(canister, commit_batch_args, logger).await, - }.map_err(CommitBatchFailed) + BATCH_UPLOAD_API_VERSION.. => commit_in_stages(canister, commit_batch_args, logger, progress).await, + }.map_err(CommitBatchFailed)?; + if let Some(progress) = progress { + progress.set_state(AssetSyncState::Done); + } + Ok(()) } async fn commit_in_stages( canister: &Canister<'_>, commit_batch_args: CommitBatchArguments, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result<(), AgentError> { + if let Some(progress) = progress { + progress.set_total_batch_operations(commit_batch_args.operations.len()); + } // Note that SetAssetProperties operations are only generated for assets that // already exist, since CreateAsset operations set all properties. let (set_properties_operations, other_operations): (Vec<_>, Vec<_>) = commit_batch_args @@ -166,7 +203,7 @@ async fn commit_in_stages( // This part seems reasonable in general as a separate batch for operations in set_properties_operations.chunks(500) { - info!(logger, "Setting properties of {} assets.", operations.len()); + debug!(logger, "Setting properties of {} assets.", operations.len()); commit_batch( canister, CommitBatchArguments { @@ -174,13 +211,16 @@ async fn commit_in_stages( operations: operations.into(), }, ) - .await? + .await?; + if let Some(progress) = progress { + progress.add_committed_batch_operations(operations.len()); + } } // Seen to work at 800 ({"SetAssetContent": 932, "Delete": 47, "CreateAsset": 58}) // so 500 shouldn't exceed per-message instruction limit for operations in other_operations.chunks(500) { - info!( + debug!( logger, "Committing batch with {} operations.", operations.len() @@ -192,7 +232,10 @@ async fn commit_in_stages( operations: operations.into(), }, ) - .await? + .await?; + if let Some(progress) = progress { + progress.add_committed_batch_operations(operations.len()); + } } // this just deletes the batch @@ -211,6 +254,7 @@ pub async fn prepare_sync_for_proposal( canister: &Canister<'_>, dirs: &[&Path], logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result<(Nat, ByteBuf), PrepareSyncForProposalError> { let canister_api_version = api_version(canister).await; let arg = upload_content_and_assemble_sync_operations( @@ -220,6 +264,7 @@ pub async fn prepare_sync_for_proposal( false, ByProposal, logger, + progress, ) .await?; let arg = sort_batch_operations(arg); diff --git a/src/canisters/frontend/ic-asset/src/upload.rs b/src/canisters/frontend/ic-asset/src/upload.rs index 10ed9a514a..4172a06463 100644 --- a/src/canisters/frontend/ic-asset/src/upload.rs +++ b/src/canisters/frontend/ic-asset/src/upload.rs @@ -14,6 +14,7 @@ use crate::canister_api::methods::{ use crate::canister_api::types::batch_upload::v0; use crate::error::CompatibilityError::DowngradeV1TOV0Failed; use crate::error::UploadError::{self, CommitBatchFailed, CreateBatchFailed, ListAssetsFailed}; +use crate::AssetSyncProgressRenderer; use ic_utils::Canister; use slog::{info, Logger}; use std::collections::HashMap; @@ -24,6 +25,7 @@ pub async fn upload( canister: &Canister<'_>, files: HashMap, logger: &Logger, + progress: Option<&dyn AssetSyncProgressRenderer>, ) -> Result<(), UploadError> { let asset_descriptors: Vec = files .iter() @@ -52,6 +54,7 @@ pub async fn upload( &canister_assets, NormalDeploy, logger, + progress, ) .await?; diff --git a/src/canisters/frontend/icx-asset/src/commands/sync.rs b/src/canisters/frontend/icx-asset/src/commands/sync.rs index 544dc6c761..f239aa77e1 100644 --- a/src/canisters/frontend/icx-asset/src/commands/sync.rs +++ b/src/canisters/frontend/icx-asset/src/commands/sync.rs @@ -9,6 +9,6 @@ pub(crate) async fn sync( logger: &Logger, ) -> anyhow::Result<()> { let dirs: Vec<&Path> = o.directory.iter().map(|d| d.as_path()).collect(); - ic_asset::sync(canister, &dirs, o.no_delete, logger).await?; + ic_asset::sync(canister, &dirs, o.no_delete, logger, None).await?; Ok(()) } diff --git a/src/canisters/frontend/icx-asset/src/commands/upload.rs b/src/canisters/frontend/icx-asset/src/commands/upload.rs index e7f974299a..a1a36d3046 100644 --- a/src/canisters/frontend/icx-asset/src/commands/upload.rs +++ b/src/canisters/frontend/icx-asset/src/commands/upload.rs @@ -12,7 +12,7 @@ pub(crate) async fn upload( logger: &Logger, ) -> anyhow::Result<()> { let key_map = get_key_map(&opts.files)?; - ic_asset::upload(canister, key_map, logger).await?; + ic_asset::upload(canister, key_map, logger, None).await?; Ok(()) } diff --git a/src/dfx-core/src/progress/mod.rs b/src/dfx-core/src/progress/mod.rs new file mode 100644 index 0000000000..da75a77976 --- /dev/null +++ b/src/dfx-core/src/progress/mod.rs @@ -0,0 +1,3 @@ +mod progress_bar; + +pub use progress_bar::ProgressBar; diff --git a/src/dfx/src/actors/mod.rs b/src/dfx/src/actors/mod.rs index 599d74b7e0..e6b2c2650e 100644 --- a/src/dfx/src/actors/mod.rs +++ b/src/dfx/src/actors/mod.rs @@ -1,3 +1,4 @@ +use self::pocketic::PocketIc; use crate::actors::btc_adapter::signals::BtcAdapterReadySubscribe; use crate::actors::btc_adapter::BtcAdapter; use crate::actors::canister_http_adapter::signals::CanisterHttpAdapterReadySubscribe; @@ -18,8 +19,6 @@ use post_start::PostStart; use std::fs; use std::path::PathBuf; -use self::pocketic::PocketIc; - pub mod btc_adapter; pub mod canister_http_adapter; pub mod pocketic; diff --git a/src/dfx/src/lib/installers/assets/mod.rs b/src/dfx/src/lib/installers/assets/mod.rs index be5b7ad7b4..db76562b64 100644 --- a/src/dfx/src/lib/installers/assets/mod.rs +++ b/src/dfx/src/lib/installers/assets/mod.rs @@ -1,17 +1,18 @@ use crate::lib::canister_info::assets::AssetsCanisterInfo; use crate::lib::canister_info::CanisterInfo; +use crate::lib::environment::Environment; use crate::lib::error::DfxResult; +use crate::lib::progress::EnvAssetSyncProgressRenderer; use anyhow::Context; use fn_error_context::context; use ic_agent::Agent; -use slog::Logger; use std::path::Path; #[context("Failed to store assets in canister '{}'.", info.get_name())] pub async fn post_install_store_assets( + env: &dyn Environment, info: &CanisterInfo, agent: &Agent, - logger: &Logger, ) -> DfxResult { let assets_canister_info = info.as_info::()?; let source_paths = assets_canister_info.get_source_paths(); @@ -27,23 +28,29 @@ pub async fn post_install_store_assets( .build() .context("Failed to build asset canister caller.")?; - ic_asset::sync(&canister, &source_paths, false, logger) - .await - .with_context(|| { - format!( - "Failed asset sync with canister {}.", - canister.canister_id_() - ) - })?; + let progress = EnvAssetSyncProgressRenderer::new(env); - Ok(()) + ic_asset::sync( + &canister, + &source_paths, + false, + env.get_logger(), + Some(&progress), + ) + .await + .with_context(|| { + format!( + "Failed asset sync with canister {}.", + canister.canister_id_() + ) + }) } #[context("Failed to store assets in canister '{}'.", info.get_name())] pub async fn prepare_assets_for_proposal( info: &CanisterInfo, agent: &Agent, - logger: &Logger, + env: &dyn Environment, ) -> DfxResult { let assets_canister_info = info.as_info::()?; let source_paths = assets_canister_info.get_source_paths(); @@ -59,7 +66,9 @@ pub async fn prepare_assets_for_proposal( .build() .context("Failed to build asset canister caller.")?; - ic_asset::prepare_sync_for_proposal(&canister, &source_paths, logger) + let r = EnvAssetSyncProgressRenderer::new(env); + + ic_asset::prepare_sync_for_proposal(&canister, &source_paths, env.get_logger(), Some(&r)) .await .with_context(|| { format!( diff --git a/src/dfx/src/lib/mod.rs b/src/dfx/src/lib/mod.rs index 8a93c56690..a8d4f0926e 100644 --- a/src/dfx/src/lib/mod.rs +++ b/src/dfx/src/lib/mod.rs @@ -26,6 +26,7 @@ pub mod nns_types; pub mod operations; pub mod package_arguments; pub mod program; +pub mod progress; pub mod progress_bar; pub mod project; pub mod replica; diff --git a/src/dfx/src/lib/operations/canister/deploy_canisters.rs b/src/dfx/src/lib/operations/canister/deploy_canisters.rs index 4b40fba0af..bdc1c6ab3d 100644 --- a/src/dfx/src/lib/operations/canister/deploy_canisters.rs +++ b/src/dfx/src/lib/operations/canister/deploy_canisters.rs @@ -376,7 +376,7 @@ async fn prepare_assets_for_commit( let agent = env.get_agent(); - prepare_assets_for_proposal(&canister_info, agent, env.get_logger()).await?; + prepare_assets_for_proposal(&canister_info, agent, env).await?; Ok(()) } @@ -414,7 +414,8 @@ async fn compute_evidence( .build() .context("Failed to build asset canister caller.")?; - let evidence = ic_asset::compute_evidence(&canister, &source_paths, env.get_logger()).await?; + let evidence = + ic_asset::compute_evidence(&canister, &source_paths, env.get_logger(), None).await?; println!("{}", evidence); Ok(()) diff --git a/src/dfx/src/lib/operations/canister/install_canister.rs b/src/dfx/src/lib/operations/canister/install_canister.rs index b2679e80f1..ca8c0f6e54 100644 --- a/src/dfx/src/lib/operations/canister/install_canister.rs +++ b/src/dfx/src/lib/operations/canister/install_canister.rs @@ -265,8 +265,8 @@ The command line value will be used.", .context("Failed to authorize your principal with the canister. You can still control the canister by using your wallet with the --wallet flag.")?; }; - info!(log, "Uploading assets to asset canister..."); - post_install_store_assets(canister_info, agent, log).await?; + debug!(log, "Uploading assets to asset canister..."); + post_install_store_assets(env, canister_info, agent).await?; } if !canister_info.get_post_install().is_empty() { let config = env.get_config()?; diff --git a/src/dfx/src/lib/progress/asset_sync_progress_renderer.rs b/src/dfx/src/lib/progress/asset_sync_progress_renderer.rs new file mode 100644 index 0000000000..17f5b3e900 --- /dev/null +++ b/src/dfx/src/lib/progress/asset_sync_progress_renderer.rs @@ -0,0 +1,161 @@ +use crate::lib::environment::Environment; +use crate::lib::progress_bar::ProgressBar; +use ic_asset::{AssetSyncProgressRenderer, AssetSyncState}; +use std::cell::OnceCell; +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub struct EnvAssetSyncProgressRenderer<'a> { + pub env: &'a dyn Environment, + + topline: ProgressBar, + bytes: OnceCell, + + // getting properties of assets already in canister + total_assets_to_retrieve_properties: AtomicUsize, + assets_retrieved_properties: AtomicUsize, + + // staging assets + total_assets: AtomicUsize, + complete_assets: AtomicUsize, + + // uploading content + total_bytes: AtomicUsize, + uploaded_bytes: AtomicUsize, + + // committing batch + total_batch_operations: AtomicUsize, + committed_batch_operations: AtomicUsize, +} + +impl<'a> EnvAssetSyncProgressRenderer<'a> { + pub fn new(env: &'a dyn Environment) -> Self { + let topline = env.new_spinner("Synchronizing assets".into()); + + let total_assets_to_retrieve_properties = AtomicUsize::new(0); + let assets_retrieved_properties = AtomicUsize::new(0); + + let total_assets = AtomicUsize::new(0); + let complete_assets = AtomicUsize::new(0); + + let bytes = OnceCell::new(); // env.new_spinner("Uploading content...".into()); + let total_bytes = AtomicUsize::new(0); + let uploaded_bytes = AtomicUsize::new(0); + + let total_batch_operations = AtomicUsize::new(0); + let committed_batch_operations = AtomicUsize::new(0); + + Self { + env, + topline, + total_assets_to_retrieve_properties, + assets_retrieved_properties, + total_assets, + complete_assets, + bytes, + total_bytes, + uploaded_bytes, + total_batch_operations, + committed_batch_operations, + } + } + + fn get_bytes_progress_bar(&self) -> &ProgressBar { + self.bytes + .get_or_init(|| self.env.new_spinner("Uploading content...".into())) + } + + fn update_get_asset_properties(&self) { + let total = self + .total_assets_to_retrieve_properties + .load(Ordering::SeqCst); + let got = self.assets_retrieved_properties.load(Ordering::SeqCst); + self.topline + .set_message(format!("Read asset properties: {}/{}", got, total).into()); + } + + fn update_assets(&self) { + let total = self.total_assets.load(Ordering::SeqCst); + let complete = self.complete_assets.load(Ordering::SeqCst); + self.topline + .set_message(format!("Staged: {}/{} assets", complete, total).into()); + } + + fn update_bytes(&self) { + let uploaded = self.uploaded_bytes.load(Ordering::SeqCst); + self.get_bytes_progress_bar() + .set_message(format!("Uploaded content: {} bytes", uploaded).into()); + } + + fn update_commit_batch(&self) { + let total = self.total_batch_operations.load(Ordering::SeqCst); + let committed = self.committed_batch_operations.load(Ordering::SeqCst); + self.topline + .set_message(format!("Committed batch: {}/{} operations", committed, total).into()); + } +} + +impl<'a> AssetSyncProgressRenderer for EnvAssetSyncProgressRenderer<'a> { + fn set_state(&self, state: AssetSyncState) { + if matches!(state, AssetSyncState::CommitBatch) { + if let Some(bar) = self.bytes.get() { + bar.finish_and_clear(); + } + } + if matches!(state, AssetSyncState::Done) { + self.topline.finish_and_clear(); + return; + } + + let msg = match state { + AssetSyncState::GatherAssetDescriptors => "Gathering asset descriptors", + AssetSyncState::ListAssets => "Listing assets", + AssetSyncState::GetAssetProperties => "Getting asset properties", + AssetSyncState::CreateBatch => "Creating batch", + AssetSyncState::StageContents => "Staging contents", + AssetSyncState::AssembleBatch => "Assembling batch", + AssetSyncState::CommitBatch => "Committing batch", + AssetSyncState::Done => unreachable!(), + }; + self.topline.set_message(msg.into()); + } + + fn set_asset_properties_to_retrieve(&self, total: usize) { + self.total_assets_to_retrieve_properties + .store(total, Ordering::SeqCst); + } + fn inc_asset_properties_retrieved(&self) { + self.assets_retrieved_properties + .fetch_add(1, Ordering::SeqCst); + self.update_get_asset_properties(); + } + + fn set_total_assets(&self, total: usize) { + self.total_assets.store(total, Ordering::SeqCst); + self.update_assets(); + } + + fn increment_complete_assets(&self) { + self.complete_assets.fetch_add(1, Ordering::SeqCst); + self.update_assets(); + } + + fn add_total_bytes(&self, add: usize) { + self.total_bytes.fetch_add(add, Ordering::SeqCst); + self.update_bytes(); + } + fn add_uploaded_bytes(&self, add: usize) { + self.uploaded_bytes.fetch_add(add, Ordering::SeqCst); + self.update_bytes(); + } + + fn set_total_batch_operations(&self, total: usize) { + self.total_batch_operations + .fetch_add(total, Ordering::SeqCst); + } + + fn add_committed_batch_operations(&self, add: usize) { + self.committed_batch_operations + .fetch_add(add, Ordering::SeqCst); + self.update_commit_batch(); + } +} diff --git a/src/dfx/src/lib/progress/mod.rs b/src/dfx/src/lib/progress/mod.rs new file mode 100644 index 0000000000..e751009df1 --- /dev/null +++ b/src/dfx/src/lib/progress/mod.rs @@ -0,0 +1,3 @@ +mod asset_sync_progress_renderer; + +pub use asset_sync_progress_renderer::EnvAssetSyncProgressRenderer;