Separate BN for block proposals (#4182)

It is a well-known fact that IP addresses for beacon nodes used by specific validators can be de-anonymized. There is an assumed risk that a malicious user may attempt to DOS validators when producing blocks to prevent chain growth/liveness.

Although there are a number of ideas put forward to address this, there a few simple approaches we can take to mitigate this risk.

Currently, a Lighthouse user is able to set a number of beacon-nodes that their validator client can connect to. If one beacon node is taken offline, it can fallback to another. Different beacon nodes can use VPNs or rotate IPs in order to mask their IPs.

This PR provides an additional setup option which further mitigates attacks of this kind.

This PR introduces a CLI flag --proposer-only to the beacon node. Setting this flag will configure the beacon node to run with minimal peers and crucially will not subscribe to subnets or sync committees. Therefore nodes of this kind should not be identified as nodes connected to validators of any kind.

It also introduces a CLI flag --proposer-nodes to the validator client. Users can then provide a number of beacon nodes (which may or may not run the --proposer-only flag) that the Validator client will use for block production and propagation only. If these nodes fail, the validator client will fallback to the default list of beacon nodes.

Users are then able to set up a number of beacon nodes dedicated to block proposals (which are unlikely to be identified as validator nodes) and point their validator clients to produce blocks on these nodes and attest on other beacon nodes. An attack attempting to prevent liveness on the eth2 network would then need to preemptively find and attack the proposer nodes which is significantly more difficult than the default setup.

This is a follow on from: #3328 

Co-authored-by: Michael Sproul <michael@sigmaprime.io>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

beacon_node/lighthouse_network/src/config.rs

@@ -134,6 +134,9 @@ pub struct Config {
     /// List of extra topics to initially subscribe to as strings.
     pub topics: Vec<GossipKind>,
 
+    /// Whether we are running a block proposer only node.
+    pub proposer_only: bool,
+
     /// Whether metrics are enabled.
     pub metrics_enabled: bool,
 
@@ -322,6 +325,7 @@ impl Default for Config {
             import_all_attestations: false,
             shutdown_after_sync: false,
             topics: Vec::new(),
+            proposer_only: false,
             metrics_enabled: false,
             enable_light_client_server: false,
             outbound_rate_limiter_config: None,

beacon_node/network/src/subnet_service/attestation_subnets.rs

@@ -112,6 +112,9 @@ pub struct AttestationService<T: BeaconChainTypes> {
     #[cfg(feature = "deterministic_long_lived_attnets")]
     next_long_lived_subscription_event: Pin<Box<tokio::time::Sleep>>,
 
+    /// Whether this node is a block proposer-only node.
+    proposer_only: bool,
+
     /// The logger for the attestation service.
     log: slog::Logger,
 }
@@ -155,6 +158,7 @@ impl<T: BeaconChainTypes> AttestationService<T> {
             known_validators: HashSetDelay::new(last_seen_val_timeout),
             waker: None,
             discovery_disabled: config.disable_discovery,
+            proposer_only: config.proposer_only,
             subscribe_all_subnets: config.subscribe_all_subnets,
             long_lived_subnet_subscription_slots,
             log,
@@ -256,6 +260,11 @@ impl<T: BeaconChainTypes> AttestationService<T> {
         &mut self,
         subscriptions: Vec<ValidatorSubscription>,
     ) -> Result<(), String> {
+        // If the node is in a proposer-only state, we ignore all subnet subscriptions.
+        if self.proposer_only {
+            return Ok(());
+        }
+
         // Maps each subnet_id subscription to it's highest slot
         let mut subnets_to_discover: HashMap<SubnetId, Slot> = HashMap::new();
         for subscription in subscriptions {
@@ -450,6 +459,10 @@ impl<T: BeaconChainTypes> AttestationService<T> {
         subnet: SubnetId,
         attestation: &Attestation<T::EthSpec>,
     ) -> bool {
+        // Proposer-only mode does not need to process attestations
+        if self.proposer_only {
+            return false;
+        }
         self.aggregate_validators_on_subnet
             .as_ref()
             .map(|tracked_vals| {

beacon_node/network/src/subnet_service/sync_subnets.rs

@@ -54,6 +54,9 @@ pub struct SyncCommitteeService<T: BeaconChainTypes> {
     /// We are always subscribed to all subnets.
     subscribe_all_subnets: bool,
 
+    /// Whether this node is a block proposer-only node.
+    proposer_only: bool,
+
     /// The logger for the attestation service.
     log: slog::Logger,
 }
@@ -82,6 +85,7 @@ impl<T: BeaconChainTypes> SyncCommitteeService<T> {
             waker: None,
             subscribe_all_subnets: config.subscribe_all_subnets,
             discovery_disabled: config.disable_discovery,
+            proposer_only: config.proposer_only,
             log,
         }
     }
@@ -110,6 +114,11 @@ impl<T: BeaconChainTypes> SyncCommitteeService<T> {
         &mut self,
         subscriptions: Vec<SyncCommitteeSubscription>,
     ) -> Result<(), String> {
+        // A proposer-only node does not subscribe to any sync-committees
+        if self.proposer_only {
+            return Ok(());
+        }
+
         let mut subnets_to_discover = Vec::new();
         for subscription in subscriptions {
             metrics::inc_counter(&metrics::SYNC_COMMITTEE_SUBSCRIPTION_REQUESTS);

beacon_node/src/cli.rs

@@ -123,7 +123,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
             Arg::with_name("target-peers")
                 .long("target-peers")
                 .help("The target number of peers.")
-                .default_value("80")
                 .takes_value(true),
         )
         .arg(
@@ -269,6 +268,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
             .min_values(0)
             .hidden(true)
         )
+        .arg(
+            Arg::with_name("proposer-only")
+                .long("proposer-only")
+                .help("Sets this beacon node at be a block proposer only node. \
+                       This will run the beacon node in a minimal configuration that is sufficient for block publishing only. This flag should be used \
+                       for a beacon node being referenced by validator client using the --proposer-node flag. This configuration is for enabling more secure setups.")
+                .takes_value(false),
+        )
+
         .arg(
             Arg::with_name("disable-backfill-rate-limiting")
                 .long("disable-backfill-rate-limiting")

beacon_node/src/config.rs

@@ -979,10 +979,13 @@ pub fn set_network_config(
 
     config.set_listening_addr(parse_listening_addresses(cli_args, log)?);
 
+    // A custom target-peers command will overwrite the --proposer-only default.
     if let Some(target_peers_str) = cli_args.value_of("target-peers") {
         config.target_peers = target_peers_str
             .parse::<usize>()
             .map_err(|_| format!("Invalid number of target peers: {}", target_peers_str))?;
+    } else {
+        config.target_peers = 80; // default value
     }
 
     if let Some(value) = cli_args.value_of("network-load") {
@@ -1218,6 +1221,20 @@ pub fn set_network_config(
         config.outbound_rate_limiter_config = Some(Default::default());
     }
 
+    // Proposer-only mode overrides a number of previous configuration parameters.
+    // Specifically, we avoid subscribing to long-lived subnets and wish to maintain a minimal set
+    // of peers.
+    if cli_args.is_present("proposer-only") {
+        config.subscribe_all_subnets = false;
+
+        if cli_args.value_of("target-peers").is_none() {
+            // If a custom value is not set, change the default to 15
+            config.target_peers = 15;
+        }
+        config.proposer_only = true;
+        warn!(log, "Proposer-only mode enabled"; "info"=> "Do not connect a validator client to this node unless via the --proposer-nodes flag");
+    }
+
     Ok(())
 }
 

book/src/SUMMARY.md

@@ -41,6 +41,7 @@
     * [Checkpoint Sync](./checkpoint-sync.md)
     * [Custom Data Directories](./advanced-datadir.md)
     * [Validator Graffiti](./graffiti.md)
+    * [Proposer Only Beacon Nodes](./advanced-proposer-only.md)
     * [Remote Signing with Web3Signer](./validator-web3signer.md)
     * [Database Configuration](./advanced_database.md)
     * [Database Migrations](./database-migrations.md)

book/src/advanced-proposer-only.md

@@ -0,0 +1,71 @@
+# Advanced Proposer-Only Beacon Nodes
+
+Lighthouse allows for more exotic setups that can minimize attack vectors by
+adding redundant beacon nodes and dividing the roles of attesting and block
+production between them. 
+
+The purpose of this is to minimize attack vectors
+where malicious users obtain the network identities (IP addresses) of beacon
+nodes corresponding to individual validators and subsequently perform Denial Of Service
+attacks on the beacon nodes when they are due to produce a block on the
+network. By splitting the duties of attestation and block production across
+different beacon nodes, an attacker may not know which node is the block
+production node, especially if the user rotates IP addresses of the block
+production beacon node in between block proposals (this is in-frequent with
+networks with large validator counts).
+
+## The Beacon Node
+
+A Lighthouse beacon node can be configured with the `--proposer-only` flag
+(i.e. `lighthouse bn --proposer-only`).
+Setting a beacon node with this flag will limit its use as a beacon node for
+normal activities such as performing attestations, but it will make the node
+harder to identify as a potential node to attack and will also consume less
+resources.
+
+Specifically, this flag reduces the default peer count (to a safe minimal
+number as maintaining peers on attestation subnets do not need to be considered), 
+prevents the node from subscribing to any attestation-subnets or
+sync-committees which is a primary way for attackers to de-anonymize
+validators.
+
+> Note: Beacon nodes that have set the `--proposer-only` flag should not be connected
+> to validator clients unless via the `--proposer-nodes` flag. If connected as a
+> normal beacon node, the validator may fail to handle its duties correctly and
+> result in a loss of income.
+
+
+## The Validator Client
+
+The validator client can be given a list of HTTP API endpoints representing
+beacon nodes that will be solely used for block propagation on the network, via
+the CLI flag `--proposer-nodes`. These nodes can be any working beacon nodes
+and do not specifically have to be proposer-only beacon nodes that have been
+executed with the  `--proposer-only` (although we do recommend this flag for
+these nodes for added security).
+
+> Note: The validator client still requires at least one other beacon node to
+> perform its duties and must be specified in the usual `--beacon-nodes` flag.
+
+> Note: The validator client will attempt to get a block to propose from the
+> beacon nodes specified in `--beacon-nodes` before trying `--proposer-nodes`.
+> This is because the nodes subscribed to subnets have a higher chance of
+> producing a more profitable block. Any block builders should therefore be
+> attached to the `--beacon-nodes` and not necessarily the `--proposer-nodes`.
+
+
+## Setup Overview
+
+The intended set-up to take advantage of this mechanism is to run one (or more)
+normal beacon nodes in conjunction with one (or more) proposer-only beacon
+nodes. See the [Redundancy](./redundancy.md) section for more information about
+setting up redundant beacon nodes. The proposer-only beacon nodes should be
+setup to use a different IP address than the primary (non proposer-only) nodes.
+For added security, the IP addresses of the proposer-only nodes should be
+rotated occasionally such that a new IP-address is used per block proposal.
+
+A single validator client can then connect to all of the above nodes via the
+`--beacon-nodes` and `--proposer-nodes` flags. The resulting setup will allow
+the validator client to perform its regular duties on the standard beacon nodes
+and when the time comes to propose a block, it will send this block via the
+specified proposer-only nodes.

testing/simulator/src/cli.rs

@@ -24,6 +24,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .takes_value(true)
                         .default_value("4")
                         .help("Number of beacon nodes"))
+                    .arg(Arg::with_name("proposer-nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .default_value("2")
+                        .help("Number of proposer-only beacon nodes"))
                     .arg(Arg::with_name("validators_per_node")
                         .short("v")
                         .long("validators_per_node")
@@ -57,6 +63,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .takes_value(true)
                         .default_value("4")
                         .help("Number of beacon nodes"))
+                    .arg(Arg::with_name("proposer-nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .default_value("2")
+                        .help("Number of proposer-only beacon nodes"))
                     .arg(Arg::with_name("validators_per_node")
                         .short("v")
                         .long("validators_per_node")

testing/simulator/src/eth1_sim.rs

@@ -27,6 +27,8 @@ const SUGGESTED_FEE_RECIPIENT: [u8; 20] =
 
 pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
     let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default");
+    let proposer_nodes = value_t!(matches, "proposer-nodes", usize).unwrap_or(0);
+    println!("PROPOSER-NODES: {}", proposer_nodes);
     let validators_per_node = value_t!(matches, "validators_per_node", usize)
         .expect("missing validators_per_node default");
     let speed_up_factor =
@@ -35,7 +37,8 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
     let post_merge_sim = matches.is_present("post-merge");
 
     println!("Beacon Chain Simulator:");
-    println!(" nodes:{}", node_count);
+    println!(" nodes:{}, proposer_nodes: {}", node_count, proposer_nodes);
+
     println!(" validators_per_node:{}", validators_per_node);
     println!(" post merge simulation:{}", post_merge_sim);
     println!(" continue_after_checks:{}", continue_after_checks);
@@ -147,7 +150,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
         beacon_config.sync_eth1_chain = true;
         beacon_config.eth1.auto_update_interval_millis = eth1_block_time.as_millis() as u64;
         beacon_config.eth1.chain_id = Eth1Id::from(chain_id);
-        beacon_config.network.target_peers = node_count - 1;
+        beacon_config.network.target_peers = node_count + proposer_nodes - 1;
 
         beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None);
 
@@ -173,7 +176,17 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
          * One by one, add beacon nodes to the network.
          */
         for _ in 0..node_count - 1 {
-            network.add_beacon_node(beacon_config.clone()).await?;
+            network
+                .add_beacon_node(beacon_config.clone(), false)
+                .await?;
+        }
+
+        /*
+         * One by one, add proposer nodes to the network.
+         */
+        for _ in 0..proposer_nodes - 1 {
+            println!("Adding a proposer node");
+            network.add_beacon_node(beacon_config.clone(), true).await?;
         }
 
         /*
@@ -310,7 +323,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
          */
         println!(
             "Simulation complete. Finished with {} beacon nodes and {} validator clients",
-            network.beacon_node_count(),
+            network.beacon_node_count() + network.proposer_node_count(),
             network.validator_client_count()
         );