Merge pull request #3307 from golemfactory/transfers/add-deploy-sha2

Deploy image using sha2 checksum

core/gftp/Cargo.toml

@@ -6,16 +6,16 @@ edition = "2018"
 homepage = "https://github.com/golemfactory/yagna"
 repository = "https://github.com/golemfactory/yagna"
 license = "LGPL-3.0"
-description="Golem File Transfer Protocol"
-keywords=["golem", "yagna"]
+description = "Golem File Transfer Protocol"
+keywords = ["golem", "yagna"]
 
 [features]
-default=[]
-bin=['env_logger', 'dotenv']
+default = []
+bin = ['env_logger', 'dotenv']
 
 [[bin]]
-name="gftp"
-required-features=['bin']
+name = "gftp"
+required-features = ['bin']
 
 [dependencies]
 ya-compile-time-utils = "0.2"
@@ -24,15 +24,15 @@ ya-service-bus = { workspace = true }
 
 actix-rt = "2.7"
 anyhow = "1.0"
-digest = "0.8.1"
+digest = "0.10"
 dotenv = { version = "0.15.0", optional = true }
 env_logger = { version = "0.7.1", optional = true }
 futures = "0.3"
 log = "0.4"
 rand = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-sha3 = "0.8.2"
+sha3 = "0.10"
 structopt = "0.3.9"
 thiserror = "1.0.20"
 tokio = { version = "1", features = ["io-std", "time"] }
@@ -41,7 +41,6 @@ url = { version = "2.1.1", features = ["serde"] }
 [dev-dependencies]
 dotenv = { version = "0.15.0" }
 env_logger = { version = "0.7.1" }
-sha3 = "0.8.2"
 tempdir = "0.3.7"
 tokio = { version = "1", features = ["process"] }
 

core/gftp/examples/gftp-server.rs

@@ -1,8 +1,7 @@
 use anyhow::{anyhow, Result};
+use digest::{Digest, Output};
 use futures::future::{FutureExt, LocalBoxFuture};
 use gftp::rpc::*;
-use sha3::digest::generic_array::GenericArray;
-use sha3::Digest;
 use std::ffi::OsString;
 use std::fs::OpenOptions;
 use std::io::Read;
@@ -14,7 +13,7 @@ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
 use tokio::process::{ChildStdin, ChildStdout, Command};
 
 static SEQ: AtomicUsize = AtomicUsize::new(0);
-type HashOutput = GenericArray<u8, <sha3::Sha3_512 as Digest>::OutputSize>;
+type HashOutput = Output<sha3::Sha3_512>;
 
 /// Build the GFTP binary, start the daemon and run:
 ///
@@ -94,12 +93,12 @@ fn hash_file(path: &Path) -> Result<HashOutput> {
     let mut chunk = vec![0; 4096];
 
     while let Ok(count) = file_src.read(&mut chunk[..]) {
-        hasher.input(&chunk[..count]);
+        hasher.update(&chunk[..count]);
         if count != 4096 {
             break;
         }
     }
-    Ok(hasher.result())
+    Ok(hasher.finalize())
 }
 
 #[actix_rt::main]

core/gftp/src/gftp.rs

@@ -323,7 +323,7 @@ fn hash_file_sha256(mut file: &mut fs::File) -> Result<String> {
         .with_context(|| "Can't seek file at offset 0.".to_string())?;
     io::copy(&mut file, &mut hasher)?;
 
-    Ok(format!("{:x}", hasher.result()))
+    Ok(format!("{:x}", hasher.finalize()))
 }
 
 /// Returns NodeId and file hash from gftp url.

core/identity/Cargo.toml

@@ -28,7 +28,7 @@ r2d2 = "0.8.8"
 rand = "0.8"
 rpassword = "3.0.2"
 serde_json = "1.0"
-sha2 = "0.9.1"
+sha2 = "0.10"
 structopt = "0.3"
 thiserror = "1.0"
 tokio = { version = "1", features = ["fs", "io-std", "signal", "io-util"] }

core/market/Cargo.toml

@@ -37,7 +37,7 @@ chrono = { version = "0.4", features = ["serde"] }
 derive_more = { workspace = true }
 diesel = { version = "1.4", features = ["chrono", "sqlite", "r2d2"] }
 diesel_migrations = "1.4"
-digest = "0.8.1"
+digest = "0.10"
 env_logger = { version = "0.7" }
 futures = "0.3"
 humantime = "2"
@@ -55,7 +55,7 @@ regex = "1.4.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_bytes = "0.11.14"
 serde_json = "1.0"
-sha3 = "0.8.2"
+sha3 = "0.10"
 structopt = "0.3"
 strum = { workspace = true }
 strum_macros = "0.24"

core/market/src/db/model/proposal_id.rs

@@ -137,11 +137,11 @@ pub fn hash_proposal(
 ) -> String {
     let mut hasher = Sha3_256::new();
 
-    hasher.input(offer_id.to_string());
-    hasher.input(demand_id.to_string());
-    hasher.input(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string());
+    hasher.update(offer_id.to_string());
+    hasher.update(demand_id.to_string());
+    hasher.update(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string());
 
-    format!("{:x}", hasher.result())
+    format!("{:x}", hasher.finalize())
 }
 
 impl FromStr for ProposalId {

core/market/src/db/model/subscription_id.rs

@@ -84,15 +84,15 @@ pub fn hash(
 ) -> String {
     let mut hasher = Sha3_256::new();
 
-    hasher.input(properties);
-    hasher.input(constraints);
-    hasher.input(node_id);
+    hasher.update(properties);
+    hasher.update(constraints);
+    hasher.update(node_id);
     // We can't change format freely, because it is important to compute hash.
     // Is there any other solution, to compute hash, that is format independent?
-    hasher.input(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string());
-    hasher.input(expiration_ts.format("%Y-%m-%d %H:%M:%f").to_string());
+    hasher.update(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string());
+    hasher.update(expiration_ts.format("%Y-%m-%d %H:%M:%f").to_string());
 
-    format!("{:x}", hasher.result())
+    format!("{:x}", hasher.finalize())
 }
 
 impl FromStr for SubscriptionId {

core/payment-driver/base/Cargo.toml

@@ -24,7 +24,7 @@ num-bigint = { version = "0.3", features = ["serde"] }
 num-traits = "0.2"
 num-derive = "0.3"
 r2d2 = "0.8"
-sha3 = "0.9"
+sha3 = "0.10"
 thiserror = "1.0"
 tokio = { version = "1", features = ["macros"] }
 

core/payment-driver/erc20/Cargo.toml

@@ -28,7 +28,7 @@ num-traits = "0.2"
 rlp = "0.5"
 serde = "1.0"
 serde_json = "^1.0"
-sha3 = "0.8"
+sha3 = "0.10"
 thiserror = "1.0"
 tiny-keccak = { version = "2.0", features = ["keccak"] }
 tokio = { version = "1", features = ["full"] }

core/vpn/Cargo.toml

@@ -45,8 +45,10 @@ default = []
 
 [dev-dependencies]
 ya-client = "0.8"
+
 actix-rt = "2.7"
-sha3 = "0.8.2"
+digest = "0.10"
+sha3 = "0.10"
 structopt = "0.3"
 tokio = { version = "1", features = ["time", "fs"] }
 url = "2.1"

core/vpn/examples/ws.rs

@@ -2,19 +2,19 @@ use actix::prelude::*;
 use actix_web_actors::ws;
 use actix_web_actors::ws::Frame;
 use bytes::Bytes;
+use digest::{Digest, Output};
 use futures::{SinkExt, StreamExt};
-use sha3::digest::generic_array::GenericArray;
-use sha3::Digest;
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;
 use tokio::fs::OpenOptions;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use url::Url;
+
 use ya_client::net::NetVpnApi;
 use ya_client::web::WebClient;
 use ya_client_model::net::{Address, NewNetwork, Node};
 
-type HashOutput = GenericArray<u8, <sha3::Sha3_512 as Digest>::OutputSize>;
+type HashOutput = Output<sha3::Sha3_512>;
 
 #[derive(StructOpt, Clone, Debug)]
 struct Cli {
@@ -50,10 +50,10 @@ async fn file_checksum<P: AsRef<Path>>(path: P) -> anyhow::Result<HashOutput> {
         if count == 0 {
             break;
         }
-        hasher.input(&chunk[..count]);
+        hasher.update(&chunk[..count]);
     }
 
-    Ok(hasher.result())
+    Ok(hasher.finalize())
 }
 
 #[actix_rt::main]

exe-unit/Cargo.toml

@@ -73,7 +73,7 @@ secp256k1 = { version = "0.27.0", optional = true }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.8"
-sha3 = "0.8.2"
+sha3 = "0.10"
 signal-hook = "0.3"
 socket2 = "0.4"
 structopt = "0.3"
@@ -104,7 +104,7 @@ actix-files = "0.6"
 actix-web = "4"
 env_logger = "0.7"
 rustyline = "7.0.0"
-sha3 = "0.8.2"
+sha3 = "0.10"
 shell-words = "1.0.0"
 tempdir = "0.3.7"
 

exe-unit/components/transfer/Cargo.toml

@@ -21,6 +21,8 @@ actix-rt = "2.7"
 anyhow = "1.0"
 # async-compression 0.3.8+ deprecates the "stream" module
 async-compression = { version = "=0.3.7", features = ["tokio", "futures-io", "stream", "bzip2", "gzip", "xz"] }
+blake2 = "0.10"
+blake3 = { version = "1.5", features = ["traits-preview"] }
 bytes = "1.0"
 futures = "0.3.4"
 globset = "0.4.5"
@@ -33,7 +35,8 @@ rand = "0.8"
 regex = "1.3.4"
 reqwest = { version = "0.11", optional = true }
 serde = "1.0.104"
-sha3 = "0.8.2"
+sha2 = "0.10"
+sha3 = "0.10"
 tempdir = "0.3.7"
 thiserror = "1.0.11"
 tokio = { version = "1", features = ["fs", "io-util"] }
@@ -72,10 +75,10 @@ actix-web = "4"
 actix-rt = "2.9.0"
 anyhow = "1.0"
 crossterm = "0.26.1"
+digest = "0.10"
 env_logger = "0.7"
 secp256k1 = { version = "0.27.0" }
 serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] }
-sha2 = "0.8.1"
 structopt = "0.3.15"
 test-context = "0.1.4"
 test-case = "3"

exe-unit/components/transfer/examples/gftp.rs

@@ -1,7 +1,6 @@
 use crossterm::{cursor, terminal, ExecutableCommand, QueueableCommand};
+use digest::{Digest, Output};
 use rand::RngCore;
-use sha3::digest::generic_array::GenericArray;
-use sha3::Digest;
 use std::env;
 use std::fs::OpenOptions;
 use std::io::{Read, Write};
@@ -14,7 +13,7 @@ use ya_transfer::{
     transfer, FileTransferProvider, GftpTransferProvider, TransferContext, TransferProvider,
 };
 
-type HashOutput = GenericArray<u8, <sha3::Sha3_512 as Digest>::OutputSize>;
+type HashOutput = Output<sha3::Sha3_512>;
 
 fn create_file(path: &Path, name: &str, chunk_size: usize, chunk_count: usize) -> HashOutput {
     let path = path.join(name);
@@ -32,11 +31,11 @@ fn create_file(path: &Path, name: &str, chunk_size: usize, chunk_count: usize) -
     for _ in 0..chunk_count {
         rng.fill_bytes(&mut input);
 
-        hasher.input(&input);
+        hasher.update(&input);
         file_src.write_all(&input).unwrap();
     }
     file_src.flush().unwrap();
-    hasher.result()
+    hasher.finalize()
 }
 
 fn hash_file(path: &Path) -> HashOutput {
@@ -46,12 +45,12 @@ fn hash_file(path: &Path) -> HashOutput {
     let mut chunk = vec![0; 4096];
 
     while let Ok(count) = file_src.read(&mut chunk[..]) {
-        hasher.input(&chunk[..count]);
+        hasher.update(&chunk[..count]);
         if count != 4096 {
             break;
         }
     }
-    hasher.result()
+    hasher.finalize()
 }
 
 // processing progress updates must not panic or the transfer will be aborted

exe-unit/components/transfer/src/cache.rs

@@ -147,9 +147,9 @@ impl CachePath {
     /// Creates the long version of path, including hash and the "random" token.
     pub fn temp_path(&self) -> PathBuf {
         let mut digest = sha3::Sha3_224::default();
-        digest.input(&self.hash);
-        digest.input(&self.nonce);
-        let hash = digest.result();
+        digest.update(&self.hash);
+        digest.update(&self.nonce);
+        let hash = digest.finalize();
         PathBuf::from(hex::encode(hash))
     }
 

exe-unit/components/transfer/src/gftp.rs

@@ -1,6 +1,7 @@
 use crate::error::Error;
 use crate::{abortable_sink, abortable_stream};
 use crate::{TransferContext, TransferData, TransferProvider, TransferSink, TransferStream};
+
 use bytes::Bytes;
 use futures::channel::mpsc;
 use futures::future::{ready, try_select, Either};
@@ -9,6 +10,7 @@ use gftp::DEFAULT_CHUNK_SIZE;
 use sha3::{Digest, Sha3_256};
 use tokio::task::spawn_local;
 use url::Url;
+
 use ya_core_model::gftp as model;
 use ya_core_model::gftp::Error as GftpError;
 use ya_core_model::gftp::GftpChunk;
@@ -109,12 +111,12 @@ impl TransferProvider<TransferData, Error> for GftpTransferProvider {
                             };
 
                             offset += chunk.content.len();
-                            digest.input(&chunk.content);
+                            digest.update(&chunk.content);
                             chunk_tx.send(Ok::<_, Error>(chunk)).await?;
                         }
                     }
 
-                    Ok::<_, Error>(digest.result())
+                    Ok::<_, Error>(digest.finalize())
                 };
 
                 let send_fut = chunk_rx.try_for_each_concurrent(concurrency, |chunk| async {