Feat(stateless-validation): Dynamically compute mandate price from target number of mandates per shard

chain/epoch-manager/src/test_utils.rs

@@ -107,11 +107,12 @@ pub fn epoch_info_with_num_seats(
     };
     let all_validators = account_to_validators(accounts);
     let validator_mandates = {
-        // TODO(#10014) determine required stake per mandate instead of reusing seat price.
-        // TODO(#10014) determine `min_mandates_per_shard`
         let num_shards = chunk_producers_settlement.len();
-        let min_mandates_per_shard = 0;
-        let config = ValidatorMandatesConfig::new(seat_price, min_mandates_per_shard, num_shards);
+        let total_stake =
+            all_validators.iter().fold(0_u128, |acc, v| acc.saturating_add(v.stake()));
+        // For tests we estimate the target number of seats based on the seat price of the old algorithm.
+        let target_mandates_per_shard = (total_stake / seat_price) as usize;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
         ValidatorMandates::new(config, &all_validators)
     };
     EpochInfo::new(

chain/epoch-manager/src/validator_selection.rs

@@ -187,11 +187,13 @@ pub fn proposals_to_epoch_info(
     };
 
     let validator_mandates = if checked_feature!("stable", StatelessValidationV0, next_version) {
-        // TODO(#10014) determine required stake per mandate instead of reusing seat price.
-        // TODO(#10014) determine `min_mandates_per_shard`
-        let min_mandates_per_shard = 0;
+        // Value chosen based on calculations for the security of the protocol.
+        // With this number of mandates per shard and 6 shards, the theory calculations predict the
+        // protocol is secure for 40 years (at 90% confidence).
+        let target_mandates_per_shard = 68;
+        let num_shards = shard_ids.len();
         let validator_mandates_config =
-            ValidatorMandatesConfig::new(threshold, min_mandates_per_shard, shard_ids.len());
+            ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
         // We can use `all_validators` to construct mandates Since a validator's position in
         // `all_validators` corresponds to its `ValidatorId`
         ValidatorMandates::new(validator_mandates_config, &all_validators)
@@ -836,10 +838,10 @@ mod tests {
         // Given `epoch_info` and `proposals` above, the sample at a given height is deterministic.
         let height = 42;
         let expected_assignments = vec![
-            vec![(1, 300), (0, 300), (2, 300), (3, 60)],
-            vec![(0, 600), (2, 200), (1, 200)],
-            vec![(3, 200), (2, 300), (1, 100), (0, 400)],
-            vec![(2, 200), (4, 140), (1, 400), (0, 200)],
+            vec![(4, 56), (1, 168), (2, 300), (3, 84), (0, 364)],
+            vec![(3, 70), (1, 300), (4, 42), (2, 266), (0, 308)],
+            vec![(4, 42), (1, 238), (3, 42), (0, 450), (2, 196)],
+            vec![(2, 238), (1, 294), (3, 64), (0, 378)],
         ];
         assert_eq!(epoch_info.sample_chunk_validators(height), expected_assignments);
     }

core/primitives/src/validator_mandates/compute_price.rs

@@ -0,0 +1,252 @@
+use {
+    super::ValidatorMandatesConfig,
+    near_primitives_core::types::Balance,
+    std::cmp::{min, Ordering},
+};
+
+/// Given the stakes for the validators and the target number of mandates to have,
+/// this function computes the mandate price to use. It works by using a binary search.
+pub fn compute_mandate_price(config: ValidatorMandatesConfig, stakes: &[Balance]) -> Balance {
+    let ValidatorMandatesConfig { target_mandates_per_shard, num_shards } = config;
+    let total_stake = saturating_sum(stakes.iter().copied());
+
+    // The target number of mandates cannot be larger than the total amount of stake.
+    // In production the total stake is _much_ higher than
+    // `num_shards * target_mandates_per_shard`, but in tests validators are given
+    // low staked numbers, so we need to have this condition in place.
+    let target_mandates: u128 =
+        min(num_shards.saturating_mul(target_mandates_per_shard) as u128, total_stake);
+
+    // Note: the reason to have the binary search look for the largest mandate price
+    // which obtains the target number of whole mandates is because the largest value
+    // minimizes the partial mandates. This can be seen as follows:
+    // Let `s_i` be the ith stake, `T` be the total stake and `m` be the mandate price.
+    // T / m = \sum (s_i / m) = \sum q_i + \sum r_i
+    // ==> \sum q_i = (T / m) - \sum r_i     [Eq. (1)]
+    // where `s_i = m * q_i + r_i` is obtained by the Euclidean algorithm.
+    // Notice that the LHS of (1) is the number of whole mandates, which we
+    // are assuming is equal to our target value for some range of `m` values.
+    // When we use a larger `m` value, `T / m` decreases but we need the LHS
+    // to remain constant, therefore `\sum r_i` must also decrease.
+    binary_search(1, total_stake, target_mandates, |mandate_price| {
+        saturating_sum(stakes.iter().map(|s| *s / mandate_price))
+    })
+}
+
+/// Assume `f` is a non-increasing function (f(x) <= f(y) if x > y) and `low < high`.
+/// This function uses a binary search to attempt to find the largest input, `x` such that
+/// `f(x) == target`, `low <= x` and `x <= high`.
+/// If there is no such `x` then it will return the unique input `x` such that
+/// `f(x) > target`, `f(x + 1) < target`, `low <= x` and `x <= high`.
+fn binary_search<F>(low: Balance, high: Balance, target: u128, f: F) -> Balance
+where
+    F: Fn(Balance) -> u128,
+{
+    debug_assert!(low < high);
+
+    let mut low = low;
+    let mut high = high;
+
+    if f(low) == target {
+        return highest_exact(low, high, target, f);
+    } else if f(high) == target {
+        // No need to use `highest_exact` here because we are already at the upper bound.
+        return high;
+    }
+
+    while high - low > 1 {
+        let mid = low + (high - low) / 2;
+        let f_mid = f(mid);
+
+        match f_mid.cmp(&target) {
+            Ordering::Equal => return highest_exact(mid, high, target, f),
+            Ordering::Less => high = mid,
+            Ordering::Greater => low = mid,
+        }
+    }
+
+    // No exact answer, return best price which gives an answer greater than
+    // `target_mandates` (which is `low` because `count_whole_mandates` is a non-increasing function).
+    low
+}
+
+/// Assume `f` is a non-increasing function (f(x) <= f(y) if x > y), `f(low) == target`
+/// and `f(high) < target`. This function uses a binary search to find the largest input, `x`
+/// such that `f(x) == target`.
+fn highest_exact<F>(low: Balance, high: Balance, target: u128, f: F) -> Balance
+where
+    F: Fn(Balance) -> u128,
+{
+    debug_assert!(low < high);
+    debug_assert_eq!(f(low), target);
+    debug_assert!(f(high) < target);
+
+    let mut low = low;
+    let mut high = high;
+
+    while high - low > 1 {
+        let mid = low + (high - low) / 2;
+        let f_mid = f(mid);
+
+        match f_mid.cmp(&target) {
+            Ordering::Equal => low = mid,
+            Ordering::Less => high = mid,
+            Ordering::Greater => unreachable!("Given function must be non-increasing"),
+        }
+    }
+
+    low
+}
+
+fn saturating_sum<I: Iterator<Item = u128>>(iter: I) -> u128 {
+    iter.fold(0, |acc, x| acc.saturating_add(x))
+}
+
+#[cfg(test)]
+mod tests {
+    use rand::{Rng, SeedableRng};
+
+    use super::*;
+
+    // Test case where the target number of mandates is larger than the total stake.
+    // This should never happen in production, but nearcore tests sometimes have
+    // low stake.
+    #[test]
+    fn test_small_total_stake() {
+        let stakes = [100_u128; 1];
+        let num_shards = 1;
+        let target_mandates_per_shard = 1000;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        assert_eq!(compute_mandate_price(config, &stakes), 1);
+    }
+
+    // Test cases where all stakes are equal.
+    #[test]
+    fn test_constant_dist() {
+        let stakes = [11_u128; 13];
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // There are enough validators to have 1:1 correspondence with mandates.
+        assert_eq!(compute_mandate_price(config, &stakes), stakes[0]);
+
+        let target_mandates_per_shard = 2 * stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // Now each validator needs to take two mandates.
+        assert_eq!(compute_mandate_price(config, &stakes), stakes[0] / 2);
+
+        let target_mandates_per_shard = stakes.len() - 1;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // Now there are more validators than we need, but
+        // the mandate price still doesn't go below the common stake.
+        assert_eq!(compute_mandate_price(config, &stakes), stakes[0]);
+    }
+
+    // Test cases where the stake distribution is a step function.
+    #[test]
+    fn test_step_dist() {
+        let stakes = {
+            let mut buf = [11_u128; 13];
+            let n = buf.len() / 2;
+            for s in buf.iter_mut().take(n) {
+                *s *= 5;
+            }
+            buf
+        };
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // Computed price gives whole number of seats close to the target number
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard + 5);
+
+        let target_mandates_per_shard = 2 * stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard + 11);
+
+        let target_mandates_per_shard = stakes.len() / 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+    }
+
+    // Test cases where the stake distribution is exponential.
+    #[test]
+    fn test_exp_dist() {
+        let stakes = {
+            let mut buf = vec![1_000_000_000_u128; 210];
+            let mut last_stake = buf[0];
+            for s in buf.iter_mut().skip(1) {
+                last_stake = last_stake * 97 / 100;
+                *s = last_stake;
+            }
+            buf
+        };
+
+        // This case is similar to the mainnet data.
+        let num_shards = 6;
+        let target_mandates_per_shard = 68;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard * num_shards);
+
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+
+        let target_mandates_per_shard = stakes.len() * 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+
+        let target_mandates_per_shard = stakes.len() / 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+    }
+
+    // Test cases where the stakes are chosen uniformly at random.
+    #[test]
+    fn test_rand_dist() {
+        let stakes = {
+            let mut stakes = vec![0_u128; 1000];
+            let mut rng = rand::rngs::StdRng::seed_from_u64(0xdeadbeef);
+            for s in stakes.iter_mut() {
+                *s = rng.gen_range(1_u128..10_000u128);
+            }
+            stakes
+        };
+
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        // In this case it was not possible to find a seat price that exactly results
+        // in the target number of mandates. This is simply due to the discrete nature
+        // of the problem. But the algorithm still gets very close (3 out of 1000 is
+        // 0.3% off the target).
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard + 3);
+
+        let target_mandates_per_shard = 2 * stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+
+        let target_mandates_per_shard = stakes.len() / 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+    }
+
+    fn count_whole_mandates(stakes: &[u128], mandate_price: u128) -> usize {
+        saturating_sum(stakes.iter().map(|s| *s / mandate_price)) as usize
+    }
+}