Merge pull request #127 from subspace/update-subspace

Update Subspace
autonomys · Feb 19, 2024 · ce6a737 · ce6a737
2 parents 041e2b0 + 29ffe55
commit ce6a737
Show file tree

Hide file tree

Showing 9 changed files with 147 additions and 107 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -49,7 +49,7 @@ jobs:
 
       - name: Install GTK4 and libfuse2 (Linux)
         # libfuse2 is needed for AppImage to run
-        run: sudo apt-get install --no-install-recommends -y libgtk-4-dev libfuse2
+        run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libgtk-4-dev libfuse2
         if: runner.os == 'Linux'
 
       - name: Configure GTK4 cache (Windows)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -67,7 +67,7 @@ jobs:
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0
 
       - name: Install GTK4 (Ubuntu)
-        run: sudo apt-get install --no-install-recommends -y libgtk-4-dev
+        run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libgtk-4-dev
         if: runner.os == 'Linux'
 
       - name: Install GTK4 (macOS)
@@ -187,7 +187,7 @@ jobs:
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0
 
       - name: Install GTK4 (Ubuntu)
-        run: sudo apt-get install --no-install-recommends -y libgtk-4-dev
+        run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libgtk-4-dev
         if: runner.os == 'Linux'
 
       - name: Install GTK4 (macOS)

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 name = "space-acres"
 description = "Space Acres is an opinionated GUI application for farming on Subspace Network"
 license = "0BSD"
-version = "0.1.2"
+version = "0.1.3"
 authors = ["Nazar Mokrynskyi <nazar@mokrynskyi.com>"]
 repository = "https://github.com/subspace/space-acres"
 edition = "2021"
@@ -63,25 +63,25 @@ sc-informant = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500
 sc-network = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
 sc-service = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
 sc-storage-monitor = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
-sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
+sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
 semver = "1.0.21"
 serde = { version = "1.0.196", features = ["derive"]}
 serde_json = "1.0.113"
 simple_moving_average = "1.0.2"
 sp-core = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
-sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
+sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
 sp-runtime = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
-subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746", default-features = false }
-subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-networking = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
-subspace-service = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
+subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165", default-features = false }
+subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-networking = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
+subspace-service = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
 supports-color = "2.1.0"
 thiserror = "1.0.56"
 tokio = { version = "1.35.1", features = ["fs", "time"] }

diff --git a/src/backend.rs b/src/backend.rs
@@ -25,7 +25,7 @@ use std::path::{Path, PathBuf};
 use std::pin::pin;
 use std::sync::Arc;
 use subspace_core_primitives::{BlockNumber, PublicKey};
-use subspace_farmer::piece_cache::{CacheWorker, PieceCache};
+use subspace_farmer::farmer_cache::{FarmerCache, FarmerCacheWorker};
 use subspace_farmer::single_disk_farm::SingleDiskFarm;
 use subspace_farmer::utils::plotted_pieces::PlottedPieces;
 use subspace_farmer::utils::run_future_in_dedicated_thread;
@@ -144,7 +144,7 @@ struct LoadedBackend {
     config_file_path: PathBuf,
     consensus_node: ConsensusNode,
     farmer: Farmer,
-    node_runner: NodeRunner<PieceCache>,
+    node_runner: NodeRunner<FarmerCache>,
 }
 
 enum BackendLoadingResult {
@@ -279,8 +279,8 @@ async fn load(
         node_runner,
         network_keypair,
         plotted_pieces,
-        piece_cache,
-        piece_cache_worker,
+        farmer_cache,
+        farmer_cache_worker,
     ) = create_networking_stack(
         &config,
         GENESIS_HASH.to_string(),
@@ -312,8 +312,8 @@ async fn load(
         config.farms.clone(),
         node,
         plotted_pieces,
-        piece_cache,
-        piece_cache_worker,
+        farmer_cache,
+        farmer_cache_worker,
         &maybe_node_client,
         notifications_sender,
     )
@@ -615,11 +615,11 @@ async fn create_networking_stack(
 ) -> anyhow::Result<(
     MaybeNodeRpcClient,
     Node,
-    NodeRunner<PieceCache>,
+    NodeRunner<FarmerCache>,
     Keypair,
     Arc<Mutex<Option<PlottedPieces>>>,
-    PieceCache,
-    CacheWorker<MaybeNodeRpcClient>,
+    FarmerCache,
+    FarmerCacheWorker<MaybeNodeRpcClient>,
 )> {
     notifications_sender
         .send(BackendNotification::Loading {
@@ -741,7 +741,7 @@ async fn create_networking_stack(
     let maybe_node_client = MaybeNodeRpcClient::default();
 
     let weak_plotted_pieces = Arc::downgrade(&plotted_pieces);
-    let (piece_cache, piece_cache_worker) = PieceCache::new(
+    let (farmer_cache, farmer_cache_worker) = FarmerCache::new(
         maybe_node_client.clone(),
         subspace_networking::libp2p::identity::PublicKey::from(network_keypair.public())
             .to_peer_id(),
@@ -753,7 +753,7 @@ async fn create_networking_stack(
         network_options,
         weak_plotted_pieces,
         maybe_node_client.clone(),
-        piece_cache.clone(),
+        farmer_cache.clone(),
     )?;
 
     notifications_sender
@@ -769,8 +769,8 @@ async fn create_networking_stack(
         node_runner,
         network_keypair,
         plotted_pieces,
-        piece_cache,
-        piece_cache_worker,
+        farmer_cache,
+        farmer_cache_worker,
     ))
 }
 
@@ -817,8 +817,8 @@ async fn create_farmer(
     disk_farms: Vec<DiskFarm>,
     node: Node,
     plotted_pieces: Arc<Mutex<Option<PlottedPieces>>>,
-    piece_cache: PieceCache,
-    piece_cache_worker: CacheWorker<MaybeNodeRpcClient>,
+    farmer_cache: FarmerCache,
+    farmer_cache_worker: FarmerCacheWorker<MaybeNodeRpcClient>,
     maybe_node_client: &MaybeNodeRpcClient,
     notifications_sender: &mut mpsc::Sender<BackendNotification>,
 ) -> anyhow::Result<Farmer> {
@@ -841,8 +841,8 @@ async fn create_farmer(
         node_client,
         node,
         plotted_pieces,
-        piece_cache,
-        piece_cache_worker,
+        farmer_cache,
+        farmer_cache_worker,
     };
 
     let farmer = farmer::create_farmer(farmer_options).await?;

diff --git a/src/backend/farmer.rs b/src/backend/farmer.rs
@@ -17,7 +17,7 @@ use std::{fmt, fs};
 use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg};
 use subspace_core_primitives::{PublicKey, Record, SectorIndex};
 use subspace_erasure_coding::ErasureCoding;
-use subspace_farmer::piece_cache::{CacheWorker, PieceCache};
+use subspace_farmer::farmer_cache::{FarmerCache, FarmerCacheWorker};
 use subspace_farmer::single_disk_farm::farming::FarmingNotification;
 use subspace_farmer::single_disk_farm::{
     SectorPlottingDetails, SectorUpdate, SingleDiskFarm, SingleDiskFarmError, SingleDiskFarmOptions,
@@ -56,7 +56,7 @@ pub enum FarmerNotification {
         farm_index: u8,
         notification: FarmingNotification,
     },
-    PieceCacheSyncProgress {
+    FarmerCacheSyncProgress {
         /// Progress so far in %
         progress: f32,
     },
@@ -66,22 +66,22 @@ type Notifications = Handler<FarmerNotification>;
 
 pub(super) struct Farmer {
     farm_fut: BoxFuture<'static, anyhow::Result<()>>,
-    piece_cache_worker_fut: BoxFuture<'static, ()>,
+    farmer_cache_worker_fut: BoxFuture<'static, ()>,
     initial_farm_states: Vec<InitialFarmState>,
     farm_during_initial_plotting: bool,
     notifications: Arc<Notifications>,
 }
 
 impl Farmer {
     pub(super) async fn run(self) -> anyhow::Result<()> {
-        let piece_cache_worker_fut = match run_future_in_dedicated_thread(
-            move || self.piece_cache_worker_fut,
-            "piece-cache-worker".to_string(),
+        let farmer_cache_worker_fut = match run_future_in_dedicated_thread(
+            move || self.farmer_cache_worker_fut,
+            "farmer-cache-worker".to_string(),
         ) {
-            Ok(piece_cache_worker_fut) => piece_cache_worker_fut,
+            Ok(farmer_cache_worker_fut) => farmer_cache_worker_fut,
             Err(error) => {
                 return Err(anyhow::anyhow!(
-                    "Failed to spawn piece future in background thread: {error}"
+                    "Failed to spawn farmer cache future in background thread: {error}"
                 ));
             }
         };
@@ -90,16 +90,16 @@ impl Farmer {
             move || self.farm_fut,
             "farmer-farm".to_string(),
         ) {
-            Ok(piece_cache_worker_fut) => piece_cache_worker_fut,
+            Ok(farmer_cache_worker_fut) => farmer_cache_worker_fut,
             Err(error) => {
                 return Err(anyhow::anyhow!(
-                    "Failed to spawn piece future in background thread: {error}"
+                    "Failed to spawn farm future in background thread: {error}"
                 ));
             }
         };
 
         select! {
-            _ = piece_cache_worker_fut.fuse() => {
+            _ = farmer_cache_worker_fut.fuse() => {
                 // Nothing to do, just exit
             }
             result = farm_fut.fuse() => {
@@ -151,8 +151,8 @@ pub(super) struct FarmerOptions {
     pub(super) node_client: NodeRpcClient,
     pub(super) node: Node,
     pub(super) plotted_pieces: Arc<Mutex<Option<PlottedPieces>>>,
-    pub(super) piece_cache: PieceCache,
-    pub(super) piece_cache_worker: CacheWorker<MaybeNodeRpcClient>,
+    pub(super) farmer_cache: FarmerCache,
+    pub(super) farmer_cache_worker: FarmerCacheWorker<MaybeNodeRpcClient>,
 }
 
 pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Result<Farmer> {
@@ -165,8 +165,8 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
         node_client,
         node,
         plotted_pieces,
-        piece_cache,
-        piece_cache_worker,
+        farmer_cache,
+        farmer_cache_worker,
     } = farmer_options;
 
     if disk_farms.is_empty() {
@@ -207,14 +207,14 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
 
     let piece_getter = Arc::new(FarmerPieceGetter::new(
         piece_provider,
-        piece_cache.clone(),
+        farmer_cache.clone(),
         node_client.clone(),
         Arc::clone(&plotted_pieces),
     ));
 
-    let piece_cache_worker_fut = Box::pin(
-        piece_cache_worker
-            .run(piece_getter.clone())
+    let farmer_cache_worker_fut = Box::pin(
+        farmer_cache_worker
+            .run(piece_getter.downgrade())
             .in_current_span(),
     );
 
@@ -253,6 +253,15 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
     let downloading_semaphore =
         Arc::new(Semaphore::new(plotting_thread_pool_core_indices.len() + 1));
 
+    let record_encoding_concurrency = {
+        let cpu_cores = plotting_thread_pool_core_indices
+            .first()
+            .expect("Guaranteed to have some CPU cores; qed");
+
+        NonZeroUsize::new((cpu_cores.cpu_cores().len() / 2).min(8))
+            .expect("Guaranteed to have some CPU cores; qed")
+    };
+
     let plotting_thread_pool_manager = create_plotting_thread_pool_manager(
         plotting_thread_pool_core_indices
             .into_iter()
@@ -278,10 +287,12 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
                 piece_getter: piece_getter.clone(),
                 cache_percentage: CACHE_PERCENTAGE,
                 downloading_semaphore: Arc::clone(&downloading_semaphore),
+                record_encoding_concurrency,
                 farm_during_initial_plotting,
                 farming_thread_pool_size: recommended_number_of_farming_threads(),
                 plotting_thread_pool_manager: plotting_thread_pool_manager.clone(),
                 plotting_delay: Some(plotting_delay_receiver),
+                disable_farm_locking: false,
             },
             disk_farm_index,
         );
@@ -322,7 +333,7 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
         single_disk_farms.push(single_disk_farm);
     }
 
-    let cache_acknowledgement_receiver = piece_cache
+    let cache_acknowledgement_receiver = farmer_cache
         .replace_backing_caches(
             single_disk_farms
                 .iter()
@@ -387,12 +398,12 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
 
     let notifications = Arc::new(Notifications::default());
 
-    piece_cache
+    farmer_cache
         .on_sync_progress(Arc::new({
             let notifications = Arc::clone(&notifications);
 
             move |progress| {
-                notifications.call_simple(&FarmerNotification::PieceCacheSyncProgress {
+                notifications.call_simple(&FarmerNotification::FarmerCacheSyncProgress {
                     progress: *progress,
                 });
             }
@@ -508,7 +519,7 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
 
     anyhow::Ok(Farmer {
         farm_fut,
-        piece_cache_worker_fut,
+        farmer_cache_worker_fut,
         initial_farm_states,
         farm_during_initial_plotting,
         notifications,

diff --git a/src/backend/networking.rs b/src/backend/networking.rs
@@ -2,8 +2,8 @@ use parking_lot::Mutex;
 use std::collections::HashSet;
 use std::path::Path;
 use std::sync::{Arc, Weak};
+use subspace_farmer::farmer_cache::FarmerCache;
 use subspace_farmer::node_client::NodeClientExt;
-use subspace_farmer::piece_cache::PieceCache;
 use subspace_farmer::utils::plotted_pieces::PlottedPieces;
 use subspace_farmer::KNOWN_PEERS_CACHE_SIZE;
 use subspace_networking::libp2p::identity::ed25519::Keypair;
@@ -87,8 +87,8 @@ pub fn create_network<NC>(
     }: NetworkOptions,
     weak_plotted_pieces: Weak<Mutex<Option<PlottedPieces>>>,
     node_client: NC,
-    piece_cache: PieceCache,
-) -> Result<(Node, NodeRunner<PieceCache>), anyhow::Error>
+    farmer_cache: FarmerCache,
+) -> Result<(Node, NodeRunner<FarmerCache>), anyhow::Error>
 where
     NC: NodeClientExt,
 {
@@ -106,7 +106,7 @@ where
     })
     .map(Box::new)?;
 
-    let default_config = Config::new(protocol_prefix, keypair.into(), piece_cache.clone(), None);
+    let default_config = Config::new(protocol_prefix, keypair.into(), farmer_cache.clone(), None);
     let config = Config {
         reserved_peers,
         listen_on,
@@ -117,11 +117,11 @@ where
                 debug!(?piece_index, "Piece request received. Trying cache...");
 
                 let weak_plotted_pieces = weak_plotted_pieces.clone();
-                let piece_cache = piece_cache.clone();
+                let farmer_cache = farmer_cache.clone();
 
                 async move {
                     let key = RecordKey::from(piece_index.to_multihash());
-                    let piece_from_store = piece_cache.get_piece(key).await;
+                    let piece_from_store = farmer_cache.get_piece(key).await;
 
                     if let Some(piece) = piece_from_store {
                         Some(PieceByIndexResponse { piece: Some(piece) })