validator: Added --health-check-slot-distance (bp #10324) (#10330)

automerge

core/src/rpc.rs

@@ -64,6 +64,7 @@ pub struct JsonRpcConfig {
     pub enable_rpc_transaction_history: bool,
     pub identity_pubkey: Pubkey,
     pub faucet_addr: Option<SocketAddr>,
+    pub health_check_slot_distance: u64,
 }
 
 #[derive(Clone)]

core/src/rpc_service.rs

@@ -26,10 +26,6 @@ use std::{
 };
 use tokio::prelude::Future;
 
-// If trusted validators are specified, consider this validator healthy if its latest account hash
-// is no further behind than this distance from the latest trusted validator account hash
-const HEALTH_CHECK_SLOT_DISTANCE: u64 = 150;
-
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
 
@@ -46,6 +42,7 @@ struct RpcRequestMiddleware {
     cluster_info: Arc<ClusterInfo>,
     trusted_validators: Option<HashSet<Pubkey>>,
     bank_forks: Arc<RwLock<BankForks>>,
+    health_check_slot_distance: u64,
     override_health_check: Arc<AtomicBool>,
 }
 
@@ -56,6 +53,7 @@ impl RpcRequestMiddleware {
         cluster_info: Arc<ClusterInfo>,
         trusted_validators: Option<HashSet<Pubkey>>,
         bank_forks: Arc<RwLock<BankForks>>,
+        health_check_slot_distance: u64,
         override_health_check: Arc<AtomicBool>,
     ) -> Self {
         Self {
@@ -66,6 +64,7 @@ impl RpcRequestMiddleware {
             cluster_info,
             trusted_validators,
             bank_forks,
+            health_check_slot_distance,
             override_health_check,
         }
     }
@@ -170,12 +169,12 @@ impl RpcRequestMiddleware {
             };
 
             // This validator is considered healthy if its latest account hash slot is within
-            // `HEALTH_CHECK_SLOT_DISTANCE` of the latest trusted validator's account hash slot
+            // `health_check_slot_distance` of the latest trusted validator's account hash slot
             if latest_account_hash_slot > 0
                 && latest_trusted_validator_account_hash_slot > 0
                 && latest_account_hash_slot
                     > latest_trusted_validator_account_hash_slot
-                        .saturating_sub(HEALTH_CHECK_SLOT_DISTANCE)
+                        .saturating_sub(self.health_check_slot_distance)
             {
                 "ok"
             } else {
@@ -300,6 +299,7 @@ impl JsonRpcService {
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
+        let health_check_slot_distance = config.health_check_slot_distance;
         let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(
             config,
             bank_forks.clone(),
@@ -328,6 +328,7 @@ impl JsonRpcService {
                     cluster_info.clone(),
                     trusted_validators,
                     bank_forks.clone(),
+                    health_check_slot_distance,
                     override_health_check,
                 );
                 let server = ServerBuilder::with_meta_extractor(
@@ -490,6 +491,7 @@ mod tests {
             cluster_info.clone(),
             None,
             bank_forks.clone(),
+            42,
             Arc::new(AtomicBool::new(false)),
         );
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
@@ -502,6 +504,7 @@ mod tests {
             cluster_info,
             None,
             bank_forks,
+            42,
             Arc::new(AtomicBool::new(false)),
         );
 
@@ -536,6 +539,7 @@ mod tests {
             cluster_info.clone(),
             None,
             create_bank_forks(),
+            42,
             Arc::new(AtomicBool::new(false)),
         );
         assert_eq!(rm.health_check(), "ok");
@@ -545,6 +549,8 @@ mod tests {
     fn test_health_check_with_trusted_validators() {
         let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
 
+        let health_check_slot_distance = 123;
+
         let override_health_check = Arc::new(AtomicBool::new(false));
         let trusted_validators = vec![Pubkey::new_rand(), Pubkey::new_rand(), Pubkey::new_rand()];
         let rm = RpcRequestMiddleware::new(
@@ -553,6 +559,7 @@ mod tests {
             cluster_info.clone(),
             Some(trusted_validators.clone().into_iter().collect()),
             create_bank_forks(),
+            health_check_slot_distance,
             override_health_check.clone(),
         );
 
@@ -595,7 +602,7 @@ mod tests {
             .insert(
                 CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
                     trusted_validators[1].clone(),
-                    vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE - 1, Hash::default())],
+                    vec![(1000 + health_check_slot_distance - 1, Hash::default())],
                 ))),
                 1,
             )
@@ -611,7 +618,7 @@ mod tests {
             .insert(
                 CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
                     trusted_validators[2].clone(),
-                    vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE, Hash::default())],
+                    vec![(1000 + health_check_slot_distance, Hash::default())],
                 ))),
                 1,
             )

validator/src/main.rs

@@ -507,9 +507,10 @@ pub fn main() {
                 .takes_value(true)
                 .validator(is_pubkey_or_keypair)
                 .requires("identity")
-                .help("Validator vote account public key.  If unspecified voting will be disabled. \
-                       The authorized voter for the account must either be the --identity keypair \
-                       or with the --authorized-voter argument")
+                .help("Validator vote account public key.  \
+                       If unspecified voting will be disabled. \
+                       The authorized voter for the account must either be the \
+                       --identity keypair or with the --authorized-voter argument")
         )
         .arg(
             Arg::with_name("storage_keypair")
@@ -525,7 +526,8 @@ pub fn main() {
                 .long("init-complete-file")
                 .value_name("FILE")
                 .takes_value(true)
-                .help("Create this file, if it doesn't already exist, once node initialization is complete"),
+                .help("Create this file if it doesn't already exist \
+                       once node initialization is complete"),
         )
         .arg(
             Arg::with_name("ledger_path")
@@ -550,7 +552,8 @@ pub fn main() {
                 .long("no-snapshot-fetch")
                 .takes_value(false)
                 .requires("entrypoint")
-                .help("Do not attempt to fetch a snapshot from the cluster, start from a local snapshot if present"),
+                .help("Do not attempt to fetch a snapshot from the cluster, \
+                      start from a local snapshot if present"),
         )
         .arg(
             Arg::with_name("no_genesis_fetch")
@@ -605,19 +608,35 @@ pub fn main() {
             Arg::with_name("enable_rpc_exit")
                 .long("enable-rpc-exit")
                 .takes_value(false)
-                .help("Enable the JSON RPC 'validatorExit' API.  Only enable in a debug environment"),
+                .help("Enable the JSON RPC 'validatorExit' API. \
+                       Only enable in a debug environment"),
         )
         .arg(
             Arg::with_name("enable_rpc_set_log_filter")
                 .long("enable-rpc-set-log-filter")
                 .takes_value(false)
-                .help("Enable the JSON RPC 'setLogFilter' API.  Only enable in a debug environment"),
+                .help("Enable the JSON RPC 'setLogFilter' API. \
+                       Only enable in a debug environment"),
         )
         .arg(
             Arg::with_name("enable_rpc_transaction_history")
                 .long("enable-rpc-transaction-history")
                 .takes_value(false)
-                .help("Enable historical transaction info over JSON RPC, including the 'getConfirmedBlock' API.  This will cause an increase in disk usage and IOPS"),
+                .help("Enable historical transaction info over JSON RPC, \
+                       including the 'getConfirmedBlock' API.  \
+                       This will cause an increase in disk usage  and IOPS"),
+        )
+        .arg(
+            Arg::with_name("health_check_slot_distance")
+                .long("health-check-slot-distance")
+                .value_name("SLOT_DISTANCE")
+                .takes_value(true)
+                .default_value("150")
+                .help("If --trusted-validators are specified, report this validator healthy \
+                       if its latest account hash is no further behind than this number of \
+                       slots from the latest trusted validator account hash. \
+                       If no --trusted-validators are specified, the validator will always \
+                       report itself to be healthy")
         )
         .arg(
             Arg::with_name("rpc_faucet_addr")
@@ -657,7 +676,8 @@ pub fn main() {
                 .takes_value(true)
                 .conflicts_with("entrypoint")
                 .validator(solana_net_utils::is_host)
-                .help("IP address for the node to advertise in gossip when --entrypoint is not provided [default: 127.0.0.1]"),
+                .help("IP address for the node to advertise in gossip when \
+                      --entrypoint is not provided [default: 127.0.0.1]"),
         )
         .arg(
             Arg::with_name("dynamic_port_range")
@@ -674,7 +694,8 @@ pub fn main() {
                 .value_name("SNAPSHOT_INTERVAL_SLOTS")
                 .takes_value(true)
                 .default_value("100")
-                .help("Number of slots between generating snapshots, 0 to disable snapshots"),
+                .help("Number of slots between generating snapshots, \
+                      0 to disable snapshots"),
         )
         .arg(
             Arg::with_name("limit_ledger_size")
@@ -867,6 +888,11 @@ pub fn main() {
             faucet_addr: matches.value_of("rpc_faucet_addr").map(|address| {
                 solana_net_utils::parse_host_port(address).expect("failed to parse faucet address")
             }),
+            health_check_slot_distance: value_t_or_exit!(
+                matches,
+                "health_check_slot_distance",
+                u64
+            ),
         },
         rpc_ports: value_t!(matches, "rpc_port", u16)
             .ok()