Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Subspace #127

Merged
merged 3 commits into from
Feb 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:

- name: Install GTK4 and libfuse2 (Linux)
# libfuse2 is needed for AppImage to run
run: sudo apt-get install --no-install-recommends -y libgtk-4-dev libfuse2
run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libgtk-4-dev libfuse2
if: runner.os == 'Linux'

- name: Configure GTK4 cache (Windows)
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0

- name: Install GTK4 (Ubuntu)
run: sudo apt-get install --no-install-recommends -y libgtk-4-dev
run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libgtk-4-dev
if: runner.os == 'Linux'

- name: Install GTK4 (macOS)
Expand Down Expand Up @@ -187,7 +187,7 @@ jobs:
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v3.1.0

- name: Install GTK4 (Ubuntu)
run: sudo apt-get install --no-install-recommends -y libgtk-4-dev
run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libgtk-4-dev
if: runner.os == 'Linux'

- name: Install GTK4 (macOS)
Expand Down
108 changes: 66 additions & 42 deletions Cargo.lock

Large diffs are not rendered by default.

28 changes: 14 additions & 14 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
name = "space-acres"
description = "Space Acres is an opinionated GUI application for farming on Subspace Network"
license = "0BSD"
version = "0.1.2"
version = "0.1.3"
authors = ["Nazar Mokrynskyi <nazar@mokrynskyi.com>"]
repository = "https://github.com/subspace/space-acres"
edition = "2021"
Expand Down Expand Up @@ -63,25 +63,25 @@ sc-informant = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500
sc-network = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
sc-service = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
sc-storage-monitor = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
semver = "1.0.21"
serde = { version = "1.0.196", features = ["derive"]}
serde_json = "1.0.113"
simple_moving_average = "1.0.2"
sp-core = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
sp-runtime = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746", default-features = false }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-service = { git = "https://github.com/subspace/subspace", rev = "ad2cc2c740a657b653d2e9800c314f2e6a64e746" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165", default-features = false }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
subspace-service = { git = "https://github.com/subspace/subspace", rev = "a41b193837d8a57e2749cae30f21bee729cfb165" }
supports-color = "2.1.0"
thiserror = "1.0.56"
tokio = { version = "1.35.1", features = ["fs", "time"] }
Expand Down
34 changes: 17 additions & 17 deletions src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use std::path::{Path, PathBuf};
use std::pin::pin;
use std::sync::Arc;
use subspace_core_primitives::{BlockNumber, PublicKey};
use subspace_farmer::piece_cache::{CacheWorker, PieceCache};
use subspace_farmer::farmer_cache::{FarmerCache, FarmerCacheWorker};
use subspace_farmer::single_disk_farm::SingleDiskFarm;
use subspace_farmer::utils::plotted_pieces::PlottedPieces;
use subspace_farmer::utils::run_future_in_dedicated_thread;
Expand Down Expand Up @@ -144,7 +144,7 @@ struct LoadedBackend {
config_file_path: PathBuf,
consensus_node: ConsensusNode,
farmer: Farmer,
node_runner: NodeRunner<PieceCache>,
node_runner: NodeRunner<FarmerCache>,
}

enum BackendLoadingResult {
Expand Down Expand Up @@ -279,8 +279,8 @@ async fn load(
node_runner,
network_keypair,
plotted_pieces,
piece_cache,
piece_cache_worker,
farmer_cache,
farmer_cache_worker,
) = create_networking_stack(
&config,
GENESIS_HASH.to_string(),
Expand Down Expand Up @@ -312,8 +312,8 @@ async fn load(
config.farms.clone(),
node,
plotted_pieces,
piece_cache,
piece_cache_worker,
farmer_cache,
farmer_cache_worker,
&maybe_node_client,
notifications_sender,
)
Expand Down Expand Up @@ -615,11 +615,11 @@ async fn create_networking_stack(
) -> anyhow::Result<(
MaybeNodeRpcClient,
Node,
NodeRunner<PieceCache>,
NodeRunner<FarmerCache>,
Keypair,
Arc<Mutex<Option<PlottedPieces>>>,
PieceCache,
CacheWorker<MaybeNodeRpcClient>,
FarmerCache,
FarmerCacheWorker<MaybeNodeRpcClient>,
)> {
notifications_sender
.send(BackendNotification::Loading {
Expand Down Expand Up @@ -741,7 +741,7 @@ async fn create_networking_stack(
let maybe_node_client = MaybeNodeRpcClient::default();

let weak_plotted_pieces = Arc::downgrade(&plotted_pieces);
let (piece_cache, piece_cache_worker) = PieceCache::new(
let (farmer_cache, farmer_cache_worker) = FarmerCache::new(
maybe_node_client.clone(),
subspace_networking::libp2p::identity::PublicKey::from(network_keypair.public())
.to_peer_id(),
Expand All @@ -753,7 +753,7 @@ async fn create_networking_stack(
network_options,
weak_plotted_pieces,
maybe_node_client.clone(),
piece_cache.clone(),
farmer_cache.clone(),
)?;

notifications_sender
Expand All @@ -769,8 +769,8 @@ async fn create_networking_stack(
node_runner,
network_keypair,
plotted_pieces,
piece_cache,
piece_cache_worker,
farmer_cache,
farmer_cache_worker,
))
}

Expand Down Expand Up @@ -817,8 +817,8 @@ async fn create_farmer(
disk_farms: Vec<DiskFarm>,
node: Node,
plotted_pieces: Arc<Mutex<Option<PlottedPieces>>>,
piece_cache: PieceCache,
piece_cache_worker: CacheWorker<MaybeNodeRpcClient>,
farmer_cache: FarmerCache,
farmer_cache_worker: FarmerCacheWorker<MaybeNodeRpcClient>,
maybe_node_client: &MaybeNodeRpcClient,
notifications_sender: &mut mpsc::Sender<BackendNotification>,
) -> anyhow::Result<Farmer> {
Expand All @@ -841,8 +841,8 @@ async fn create_farmer(
node_client,
node,
plotted_pieces,
piece_cache,
piece_cache_worker,
farmer_cache,
farmer_cache_worker,
};

let farmer = farmer::create_farmer(farmer_options).await?;
Expand Down
57 changes: 34 additions & 23 deletions src/backend/farmer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use std::{fmt, fs};
use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg};
use subspace_core_primitives::{PublicKey, Record, SectorIndex};
use subspace_erasure_coding::ErasureCoding;
use subspace_farmer::piece_cache::{CacheWorker, PieceCache};
use subspace_farmer::farmer_cache::{FarmerCache, FarmerCacheWorker};
use subspace_farmer::single_disk_farm::farming::FarmingNotification;
use subspace_farmer::single_disk_farm::{
SectorPlottingDetails, SectorUpdate, SingleDiskFarm, SingleDiskFarmError, SingleDiskFarmOptions,
Expand Down Expand Up @@ -56,7 +56,7 @@ pub enum FarmerNotification {
farm_index: u8,
notification: FarmingNotification,
},
PieceCacheSyncProgress {
FarmerCacheSyncProgress {
/// Progress so far in %
progress: f32,
},
Expand All @@ -66,22 +66,22 @@ type Notifications = Handler<FarmerNotification>;

pub(super) struct Farmer {
farm_fut: BoxFuture<'static, anyhow::Result<()>>,
piece_cache_worker_fut: BoxFuture<'static, ()>,
farmer_cache_worker_fut: BoxFuture<'static, ()>,
initial_farm_states: Vec<InitialFarmState>,
farm_during_initial_plotting: bool,
notifications: Arc<Notifications>,
}

impl Farmer {
pub(super) async fn run(self) -> anyhow::Result<()> {
let piece_cache_worker_fut = match run_future_in_dedicated_thread(
move || self.piece_cache_worker_fut,
"piece-cache-worker".to_string(),
let farmer_cache_worker_fut = match run_future_in_dedicated_thread(
move || self.farmer_cache_worker_fut,
"farmer-cache-worker".to_string(),
) {
Ok(piece_cache_worker_fut) => piece_cache_worker_fut,
Ok(farmer_cache_worker_fut) => farmer_cache_worker_fut,
Err(error) => {
return Err(anyhow::anyhow!(
"Failed to spawn piece future in background thread: {error}"
"Failed to spawn farmer cache future in background thread: {error}"
));
}
};
Expand All @@ -90,16 +90,16 @@ impl Farmer {
move || self.farm_fut,
"farmer-farm".to_string(),
) {
Ok(piece_cache_worker_fut) => piece_cache_worker_fut,
Ok(farmer_cache_worker_fut) => farmer_cache_worker_fut,
Err(error) => {
return Err(anyhow::anyhow!(
"Failed to spawn piece future in background thread: {error}"
"Failed to spawn farm future in background thread: {error}"
));
}
};

select! {
_ = piece_cache_worker_fut.fuse() => {
_ = farmer_cache_worker_fut.fuse() => {
// Nothing to do, just exit
}
result = farm_fut.fuse() => {
Expand Down Expand Up @@ -151,8 +151,8 @@ pub(super) struct FarmerOptions {
pub(super) node_client: NodeRpcClient,
pub(super) node: Node,
pub(super) plotted_pieces: Arc<Mutex<Option<PlottedPieces>>>,
pub(super) piece_cache: PieceCache,
pub(super) piece_cache_worker: CacheWorker<MaybeNodeRpcClient>,
pub(super) farmer_cache: FarmerCache,
pub(super) farmer_cache_worker: FarmerCacheWorker<MaybeNodeRpcClient>,
}

pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Result<Farmer> {
Expand All @@ -165,8 +165,8 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
node_client,
node,
plotted_pieces,
piece_cache,
piece_cache_worker,
farmer_cache,
farmer_cache_worker,
} = farmer_options;

if disk_farms.is_empty() {
Expand Down Expand Up @@ -207,14 +207,14 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu

let piece_getter = Arc::new(FarmerPieceGetter::new(
piece_provider,
piece_cache.clone(),
farmer_cache.clone(),
node_client.clone(),
Arc::clone(&plotted_pieces),
));

let piece_cache_worker_fut = Box::pin(
piece_cache_worker
.run(piece_getter.clone())
let farmer_cache_worker_fut = Box::pin(
farmer_cache_worker
.run(piece_getter.downgrade())
.in_current_span(),
);

Expand Down Expand Up @@ -253,6 +253,15 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
let downloading_semaphore =
Arc::new(Semaphore::new(plotting_thread_pool_core_indices.len() + 1));

let record_encoding_concurrency = {
let cpu_cores = plotting_thread_pool_core_indices
.first()
.expect("Guaranteed to have some CPU cores; qed");

NonZeroUsize::new((cpu_cores.cpu_cores().len() / 2).min(8))
.expect("Guaranteed to have some CPU cores; qed")
};

let plotting_thread_pool_manager = create_plotting_thread_pool_manager(
plotting_thread_pool_core_indices
.into_iter()
Expand All @@ -278,10 +287,12 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
piece_getter: piece_getter.clone(),
cache_percentage: CACHE_PERCENTAGE,
downloading_semaphore: Arc::clone(&downloading_semaphore),
record_encoding_concurrency,
farm_during_initial_plotting,
farming_thread_pool_size: recommended_number_of_farming_threads(),
plotting_thread_pool_manager: plotting_thread_pool_manager.clone(),
plotting_delay: Some(plotting_delay_receiver),
disable_farm_locking: false,
},
disk_farm_index,
);
Expand Down Expand Up @@ -322,7 +333,7 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu
single_disk_farms.push(single_disk_farm);
}

let cache_acknowledgement_receiver = piece_cache
let cache_acknowledgement_receiver = farmer_cache
.replace_backing_caches(
single_disk_farms
.iter()
Expand Down Expand Up @@ -387,12 +398,12 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu

let notifications = Arc::new(Notifications::default());

piece_cache
farmer_cache
.on_sync_progress(Arc::new({
let notifications = Arc::clone(&notifications);

move |progress| {
notifications.call_simple(&FarmerNotification::PieceCacheSyncProgress {
notifications.call_simple(&FarmerNotification::FarmerCacheSyncProgress {
progress: *progress,
});
}
Expand Down Expand Up @@ -508,7 +519,7 @@ pub(super) async fn create_farmer(farmer_options: FarmerOptions) -> anyhow::Resu

anyhow::Ok(Farmer {
farm_fut,
piece_cache_worker_fut,
farmer_cache_worker_fut,
initial_farm_states,
farm_during_initial_plotting,
notifications,
Expand Down
12 changes: 6 additions & 6 deletions src/backend/networking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ use parking_lot::Mutex;
use std::collections::HashSet;
use std::path::Path;
use std::sync::{Arc, Weak};
use subspace_farmer::farmer_cache::FarmerCache;
use subspace_farmer::node_client::NodeClientExt;
use subspace_farmer::piece_cache::PieceCache;
use subspace_farmer::utils::plotted_pieces::PlottedPieces;
use subspace_farmer::KNOWN_PEERS_CACHE_SIZE;
use subspace_networking::libp2p::identity::ed25519::Keypair;
Expand Down Expand Up @@ -87,8 +87,8 @@ pub fn create_network<NC>(
}: NetworkOptions,
weak_plotted_pieces: Weak<Mutex<Option<PlottedPieces>>>,
node_client: NC,
piece_cache: PieceCache,
) -> Result<(Node, NodeRunner<PieceCache>), anyhow::Error>
farmer_cache: FarmerCache,
) -> Result<(Node, NodeRunner<FarmerCache>), anyhow::Error>
where
NC: NodeClientExt,
{
Expand All @@ -106,7 +106,7 @@ where
})
.map(Box::new)?;

let default_config = Config::new(protocol_prefix, keypair.into(), piece_cache.clone(), None);
let default_config = Config::new(protocol_prefix, keypair.into(), farmer_cache.clone(), None);
let config = Config {
reserved_peers,
listen_on,
Expand All @@ -117,11 +117,11 @@ where
debug!(?piece_index, "Piece request received. Trying cache...");

let weak_plotted_pieces = weak_plotted_pieces.clone();
let piece_cache = piece_cache.clone();
let farmer_cache = farmer_cache.clone();

async move {
let key = RecordKey::from(piece_index.to_multihash());
let piece_from_store = piece_cache.get_piece(key).await;
let piece_from_store = farmer_cache.get_piece(key).await;

if let Some(piece) = piece_from_store {
Some(PieceByIndexResponse { piece: Some(piece) })
Expand Down
Loading
Loading