Merge pull request paritytech#74 from subspace/farmer-lib

Farmer bin to lib transition - 1st step

crates/subspace-farmer/src/commands/farm.rs

@@ -1,7 +1,8 @@
 use crate::commitments::Commitments;
+use crate::common::{Salt, Tag};
+use crate::identity::Identity;
 use crate::object_mappings::ObjectMappings;
 use crate::plot::Plot;
-use crate::{Salt, Tag};
 use anyhow::{anyhow, Result};
 use futures::future;
 use futures::future::Either;
@@ -10,10 +11,7 @@ use jsonrpsee::types::v2::params::JsonRpcParams;
 use jsonrpsee::types::Subscription;
 use jsonrpsee::ws_client::{WsClient, WsClientBuilder};
 use log::{debug, error, info, trace};
-use schnorrkel::context::SigningContext;
-use schnorrkel::{Keypair, PublicKey};
 use serde::{Deserialize, Serialize};
-use std::fs;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
@@ -24,7 +22,7 @@ use subspace_core_primitives::objects::{
     BlockObjectMapping, GlobalObject, PieceObject, PieceObjectMapping,
 };
 use subspace_core_primitives::{crypto, Sha256Hash};
-use subspace_solving::{SubspaceCodec, SOLUTION_SIGNING_CONTEXT};
+use subspace_solving::SubspaceCodec;
 
 type SlotNumber = u64;
 
@@ -104,17 +102,7 @@ pub(crate) async fn farm(base_directory: PathBuf, ws_server: &str) -> Result<()>
     info!("Connecting to RPC server");
     let client = Arc::new(WsClientBuilder::default().build(ws_server).await?);
 
-    let identity_file = base_directory.join("identity.bin");
-    let keypair = if identity_file.exists() {
-        info!("Opening existing keypair");
-        Keypair::from_bytes(&fs::read(identity_file)?).map_err(anyhow::Error::msg)?
-    } else {
-        info!("Generating new keypair");
-        let keypair = Keypair::generate();
-        fs::write(identity_file, keypair.to_bytes())?;
-        keypair
-    };
-    let ctx = schnorrkel::context::signing_context(SOLUTION_SIGNING_CONTEXT);
+    let identity = Identity::open_or_create(&base_directory)?;
 
     // TODO: This doesn't account for the fact that node can have a completely different history to
     //  what farmer expects
@@ -136,15 +124,15 @@ pub(crate) async fn farm(base_directory: PathBuf, ws_server: &str) -> Result<()>
             let client = Arc::clone(&client);
             let plot = plot.clone();
             let commitments = commitments.clone();
-            let public_key = keypair.public;
+            let public_key = identity.public_key();
 
             Box::pin(async move {
                 background_plotting(client, plot, commitments, object_mappings, &public_key).await
             })
         },
-        Box::pin(async move {
-            subscribe_to_slot_info(&client, &plot, &commitments, &keypair, &ctx).await
-        }),
+        Box::pin(
+            async move { subscribe_to_slot_info(&client, &plot, &commitments, &identity).await },
+        ),
     )
     .await
     {
@@ -163,12 +151,12 @@ pub(crate) async fn farm(base_directory: PathBuf, ws_server: &str) -> Result<()>
 // TODO: Blocks that are coming form substrate node are fully trusted right now, which we probably
 //  don't want eventually
 /// Maintains plot in up to date state plotting new pieces as they are produced on the network.
-async fn background_plotting(
+async fn background_plotting<P: AsRef<[u8]>>(
     client: Arc<WsClient>,
     plot: Plot,
     commitments: Commitments,
     object_mappings: ObjectMappings,
-    public_key: &PublicKey,
+    public_key: &P,
 ) -> Result<()> {
     let weak_plot = plot.downgrade();
     let FarmerMetadata {
@@ -494,10 +482,9 @@ async fn subscribe_to_slot_info(
     client: &WsClient,
     plot: &Plot,
     commitments: &Commitments,
-    keypair: &Keypair,
-    ctx: &SigningContext,
+    identity: &Identity,
 ) -> Result<()> {
-    let farmer_public_key_hash = crypto::sha256_hash(&keypair.public);
+    let farmer_public_key_hash = crypto::sha256_hash(&identity.public_key());
 
     info!("Subscribing to slot info notifications");
     let mut subscription: Subscription<SlotInfo> = client
@@ -525,10 +512,10 @@ async fn subscribe_to_slot_info(
             Some((tag, piece_index)) => {
                 let encoding = plot.read(piece_index).await?;
                 let solution = Solution {
-                    public_key: keypair.public.to_bytes(),
+                    public_key: identity.public_key().to_bytes(),
                     piece_index,
                     encoding: encoding.to_vec(),
-                    signature: keypair.sign(ctx.bytes(&tag)).to_bytes().to_vec(),
+                    signature: identity.sign(&tag).to_bytes().to_vec(),
                     tag,
                 };
 
@@ -550,7 +537,7 @@ async fn subscribe_to_slot_info(
                     &ProposedProofOfReplicationResponse {
                         slot_number: slot_info.slot_number,
                         solution,
-                        secret_key: keypair.secret.to_bytes().into(),
+                        secret_key: identity.secret_key().to_bytes().into(),
                     },
                 )?]),
             )

crates/subspace-farmer/src/commitments.rs

@@ -1,8 +1,8 @@
 #[cfg(test)]
 mod tests;
 
+use crate::common::{Salt, Tag, BATCH_SIZE};
 use crate::plot::Plot;
-use crate::{Salt, Tag, BATCH_SIZE};
 use async_lock::Mutex;
 use async_std::io;
 use async_std::path::PathBuf;

crates/subspace-farmer/src/commitments/tests.rs

@@ -1,10 +1,10 @@
 use crate::commitments::Commitments;
+use crate::common::{Salt, Tag};
 use crate::plot::Plot;
-use crate::{Salt, Tag};
 use rand::prelude::*;
 use rand::rngs::StdRng;
 use std::sync::Arc;
-use subspace_core_primitives::Piece;
+use subspace_core_primitives::{Piece, PIECE_SIZE};
 use tempfile::TempDir;
 
 fn init() {
@@ -59,7 +59,7 @@ async fn find_by_tag() {
         Arc::new(
             (0..1024_usize)
                 .map(|_| {
-                    let mut bytes = [0u8; crate::PIECE_SIZE];
+                    let mut bytes = [0u8; PIECE_SIZE];
                     rng.fill(&mut bytes[..]);
                     bytes
                 })

crates/subspace-farmer/src/common.rs

@@ -0,0 +1,5 @@
+use subspace_core_primitives::PIECE_SIZE;
+
+pub(crate) type Tag = [u8; 8];
+pub(crate) type Salt = [u8; 8];
+pub(crate) const BATCH_SIZE: u64 = (16 * 1024 * 1024 / PIECE_SIZE) as u64;

crates/subspace-farmer/src/identity.rs

@@ -0,0 +1,42 @@
+use anyhow::Error;
+use log::info;
+use schnorrkel::{context::SigningContext, Keypair, PublicKey, SecretKey, Signature};
+use std::fs;
+use std::path::Path;
+use subspace_solving::SOLUTION_SIGNING_CONTEXT;
+
+pub struct Identity {
+    keypair: Keypair,
+    ctx: SigningContext,
+}
+
+impl Identity {
+    pub fn open_or_create(path: &Path) -> Result<Identity, Error> {
+        let identity_file = path.join("identity.bin");
+        let keypair = if identity_file.exists() {
+            info!("Opening existing keypair"); // TODO: turn this into a channel
+            Keypair::from_bytes(&fs::read(identity_file)?).map_err(Error::msg)?
+        } else {
+            info!("Generating new keypair"); // TODO: turn this into a channel
+            let new_keypair = Keypair::generate();
+            fs::write(identity_file, new_keypair.to_bytes())?;
+            new_keypair
+        };
+        Ok(Identity {
+            keypair,
+            ctx: schnorrkel::context::signing_context(SOLUTION_SIGNING_CONTEXT),
+        })
+    }
+
+    pub fn public_key(&self) -> PublicKey {
+        self.keypair.public
+    }
+
+    pub fn secret_key(&self) -> SecretKey {
+        self.keypair.secret.clone()
+    }
+
+    pub fn sign(&self, data: &[u8]) -> Signature {
+        self.keypair.sign(self.ctx.bytes(data))
+    }
+}

crates/subspace-farmer/src/lib.rs

@@ -0,0 +1,21 @@
+//! subspace-farmer implementation overview
+//!
+//! The application typically runs two processes in parallel: plotting and farming.
+//!
+//! During plotting we create a binary plot file, which contains subspace-encoded pieces one
+//! after another as well as RocksDB key-value database with tags, where key is tag (first 8 bytes
+//! of `hmac(encoding, salt)`) and value is an offset of corresponding encoded piece in the plot (we
+//! can do this because all pieces have the same size). So for every 4096 bytes we also store a
+//! record with 8-bytes tag and 8-bytes index (+some overhead of RocksDB itself).
+//!
+//! During farming we receive a global challenge and need to find a solution, given target and
+//! solution range. In order to find solution we derive local challenge as our target and do range
+//! query in RocksDB. For that we interpret target as 64-bit unsigned integer, and find all of the
+//! keys in tags database that are `target ± solution range` (while also handing overflow/underflow)
+//! converted back to bytes.
+#![feature(try_blocks)]
+#![feature(hash_drain_filter)]
+
+pub(crate) mod identity;
+
+pub use identity::Identity;

crates/subspace-farmer/src/main.rs

@@ -11,13 +11,15 @@
 //! During farming we receive a global challenge and need to find a solution, given target and
 //! solution range. In order to find solution we derive local challenge as our target and do range
 //! query in RocksDB. For that we interpret target as 64-bit unsigned integer, and find all of the
-//! keys in tags database that are `target ± solution range` (while also handing overflow/underlow)
+//! keys in tags database that are `target ± solution range` (while also handing overflow/underflow)
 //! converted back to bytes.
 #![feature(try_blocks)]
 #![feature(hash_drain_filter)]
 
 mod commands;
 mod commitments;
+mod common;
+mod identity;
 mod object_mappings;
 mod plot;
 mod utils;
@@ -28,14 +30,6 @@ use env_logger::Env;
 use log::info;
 use std::fs;
 use std::path::{Path, PathBuf};
-use subspace_core_primitives::PIECE_SIZE;
-
-type Tag = [u8; 8];
-type Salt = [u8; 8];
-
-const BATCH_SIZE: u64 = (16 * 1024 * 1024 / PIECE_SIZE) as u64;
-// TODO: Move to codec
-// const CUDA_BATCH_SIZE: u64 = (32 * 1024) as u64;
 
 // TODO: Separate commands for erasing the plot and wiping everyting
 #[derive(Debug, Clap)]

crates/subspace-farmer/src/plot/tests.rs

@@ -1,15 +1,15 @@
 use crate::plot::Plot;
 use rand::prelude::*;
 use std::sync::Arc;
-use subspace_core_primitives::{LastArchivedBlock, Piece, RootBlock};
+use subspace_core_primitives::{LastArchivedBlock, Piece, RootBlock, PIECE_SIZE};
 use tempfile::TempDir;
 
 fn init() {
     let _ = env_logger::builder().is_test(true).try_init();
 }
 
 fn generate_random_piece() -> Piece {
-    let mut bytes = [0u8; crate::PIECE_SIZE];
+    let mut bytes = [0u8; PIECE_SIZE];
     rand::thread_rng().fill(&mut bytes[..]);
     bytes
 }