Feat(stateless-validation): Dynamically compute mandate price from ta…

…rget number of mandates per shard (#11044)

This PR completes the remaining tasks in #10014 though in a slightly
different way that initially conceived. This PR adds a function to
compute the mandate price from a target number of mandates and the
distribution of stakes. The function works by looking for the fixed
point of a particular function, where this fixed point should exist
because it arises from calculating the same value in two different ways.
More details on this are found in the comments on the new code.

This is a little different than the initial design where we define a
minimum number of mandates per shard. Instead we give a target number
for the function to attempt to achieve. For many stake distributions it
will exactly hit the target, but it is possible that the target cannot
be exactly achieved due to the discrete nature of the problem. In this
case the function will get as close as possible to the target; even if
that means getting a value slightly lower. Importantly, the mandate
counting done by the function does not include partial mandates,
therefore if the number of whole mandates ends up a little below the
target the partial mandates should make up for it to keep the security
of the protocol high enough.

As part of this PR, the target number of mandates per shard in
production is set to 68. This number was chosen based on [some theory
calculations](https://near.zulipchat.com/#narrow/stream/407237-core.2Fstateless-validation/topic/validator.20seat.20assignment/near/430080139).
In a future PR this number should probably be extracted out as a genesis
config parameter, but I think it's ok for now (stateless validation
MVP).

chain/epoch-manager/src/test_utils.rs

@@ -107,11 +107,12 @@ pub fn epoch_info_with_num_seats(
     };
     let all_validators = account_to_validators(accounts);
     let validator_mandates = {
-        // TODO(#10014) determine required stake per mandate instead of reusing seat price.
-        // TODO(#10014) determine `min_mandates_per_shard`
         let num_shards = chunk_producers_settlement.len();
-        let min_mandates_per_shard = 0;
-        let config = ValidatorMandatesConfig::new(seat_price, min_mandates_per_shard, num_shards);
+        let total_stake =
+            all_validators.iter().fold(0_u128, |acc, v| acc.saturating_add(v.stake()));
+        // For tests we estimate the target number of seats based on the seat price of the old algorithm.
+        let target_mandates_per_shard = (total_stake / seat_price) as usize;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
         ValidatorMandates::new(config, &all_validators)
     };
     EpochInfo::new(

chain/epoch-manager/src/validator_selection.rs

@@ -187,11 +187,13 @@ pub fn proposals_to_epoch_info(
     };
 
     let validator_mandates = if checked_feature!("stable", StatelessValidationV0, next_version) {
-        // TODO(#10014) determine required stake per mandate instead of reusing seat price.
-        // TODO(#10014) determine `min_mandates_per_shard`
-        let min_mandates_per_shard = 0;
+        // Value chosen based on calculations for the security of the protocol.
+        // With this number of mandates per shard and 6 shards, the theory calculations predict the
+        // protocol is secure for 40 years (at 90% confidence).
+        let target_mandates_per_shard = 68;
+        let num_shards = shard_ids.len();
         let validator_mandates_config =
-            ValidatorMandatesConfig::new(threshold, min_mandates_per_shard, shard_ids.len());
+            ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
         // We can use `all_validators` to construct mandates Since a validator's position in
         // `all_validators` corresponds to its `ValidatorId`
         ValidatorMandates::new(validator_mandates_config, &all_validators)
@@ -836,10 +838,10 @@ mod tests {
         // Given `epoch_info` and `proposals` above, the sample at a given height is deterministic.
         let height = 42;
         let expected_assignments = vec![
-            vec![(1, 300), (0, 300), (2, 300), (3, 60)],
-            vec![(0, 600), (2, 200), (1, 200)],
-            vec![(3, 200), (2, 300), (1, 100), (0, 400)],
-            vec![(2, 200), (4, 140), (1, 400), (0, 200)],
+            vec![(4, 56), (1, 168), (2, 300), (3, 84), (0, 364)],
+            vec![(3, 70), (1, 300), (4, 42), (2, 266), (0, 308)],
+            vec![(4, 42), (1, 238), (3, 42), (0, 450), (2, 196)],
+            vec![(2, 238), (1, 294), (3, 64), (0, 378)],
         ];
         assert_eq!(epoch_info.sample_chunk_validators(height), expected_assignments);
     }

core/primitives/src/validator_mandates/compute_price.rs

@@ -0,0 +1,252 @@
+use {
+    super::ValidatorMandatesConfig,
+    near_primitives_core::types::Balance,
+    std::cmp::{min, Ordering},
+};
+
+/// Given the stakes for the validators and the target number of mandates to have,
+/// this function computes the mandate price to use. It works by using a binary search.
+pub fn compute_mandate_price(config: ValidatorMandatesConfig, stakes: &[Balance]) -> Balance {
+    let ValidatorMandatesConfig { target_mandates_per_shard, num_shards } = config;
+    let total_stake = saturating_sum(stakes.iter().copied());
+
+    // The target number of mandates cannot be larger than the total amount of stake.
+    // In production the total stake is _much_ higher than
+    // `num_shards * target_mandates_per_shard`, but in tests validators are given
+    // low staked numbers, so we need to have this condition in place.
+    let target_mandates: u128 =
+        min(num_shards.saturating_mul(target_mandates_per_shard) as u128, total_stake);
+
+    // Note: the reason to have the binary search look for the largest mandate price
+    // which obtains the target number of whole mandates is because the largest value
+    // minimizes the partial mandates. This can be seen as follows:
+    // Let `s_i` be the ith stake, `T` be the total stake and `m` be the mandate price.
+    // T / m = \sum (s_i / m) = \sum q_i + \sum r_i
+    // ==> \sum q_i = (T / m) - \sum r_i     [Eq. (1)]
+    // where `s_i = m * q_i + r_i` is obtained by the Euclidean algorithm.
+    // Notice that the LHS of (1) is the number of whole mandates, which we
+    // are assuming is equal to our target value for some range of `m` values.
+    // When we use a larger `m` value, `T / m` decreases but we need the LHS
+    // to remain constant, therefore `\sum r_i` must also decrease.
+    binary_search(1, total_stake, target_mandates, |mandate_price| {
+        saturating_sum(stakes.iter().map(|s| *s / mandate_price))
+    })
+}
+
+/// Assume `f` is a non-increasing function (f(x) <= f(y) if x > y) and `low < high`.
+/// This function uses a binary search to attempt to find the largest input, `x` such that
+/// `f(x) == target`, `low <= x` and `x <= high`.
+/// If there is no such `x` then it will return the unique input `x` such that
+/// `f(x) > target`, `f(x + 1) < target`, `low <= x` and `x <= high`.
+fn binary_search<F>(low: Balance, high: Balance, target: u128, f: F) -> Balance
+where
+    F: Fn(Balance) -> u128,
+{
+    debug_assert!(low < high);
+
+    let mut low = low;
+    let mut high = high;
+
+    if f(low) == target {
+        return highest_exact(low, high, target, f);
+    } else if f(high) == target {
+        // No need to use `highest_exact` here because we are already at the upper bound.
+        return high;
+    }
+
+    while high - low > 1 {
+        let mid = low + (high - low) / 2;
+        let f_mid = f(mid);
+
+        match f_mid.cmp(&target) {
+            Ordering::Equal => return highest_exact(mid, high, target, f),
+            Ordering::Less => high = mid,
+            Ordering::Greater => low = mid,
+        }
+    }
+
+    // No exact answer, return best price which gives an answer greater than
+    // `target_mandates` (which is `low` because `count_whole_mandates` is a non-increasing function).
+    low
+}
+
+/// Assume `f` is a non-increasing function (f(x) <= f(y) if x > y), `f(low) == target`
+/// and `f(high) < target`. This function uses a binary search to find the largest input, `x`
+/// such that `f(x) == target`.
+fn highest_exact<F>(low: Balance, high: Balance, target: u128, f: F) -> Balance
+where
+    F: Fn(Balance) -> u128,
+{
+    debug_assert!(low < high);
+    debug_assert_eq!(f(low), target);
+    debug_assert!(f(high) < target);
+
+    let mut low = low;
+    let mut high = high;
+
+    while high - low > 1 {
+        let mid = low + (high - low) / 2;
+        let f_mid = f(mid);
+
+        match f_mid.cmp(&target) {
+            Ordering::Equal => low = mid,
+            Ordering::Less => high = mid,
+            Ordering::Greater => unreachable!("Given function must be non-increasing"),
+        }
+    }
+
+    low
+}
+
+fn saturating_sum<I: Iterator<Item = u128>>(iter: I) -> u128 {
+    iter.fold(0, |acc, x| acc.saturating_add(x))
+}
+
+#[cfg(test)]
+mod tests {
+    use rand::{Rng, SeedableRng};
+
+    use super::*;
+
+    // Test case where the target number of mandates is larger than the total stake.
+    // This should never happen in production, but nearcore tests sometimes have
+    // low stake.
+    #[test]
+    fn test_small_total_stake() {
+        let stakes = [100_u128; 1];
+        let num_shards = 1;
+        let target_mandates_per_shard = 1000;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        assert_eq!(compute_mandate_price(config, &stakes), 1);
+    }
+
+    // Test cases where all stakes are equal.
+    #[test]
+    fn test_constant_dist() {
+        let stakes = [11_u128; 13];
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // There are enough validators to have 1:1 correspondence with mandates.
+        assert_eq!(compute_mandate_price(config, &stakes), stakes[0]);
+
+        let target_mandates_per_shard = 2 * stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // Now each validator needs to take two mandates.
+        assert_eq!(compute_mandate_price(config, &stakes), stakes[0] / 2);
+
+        let target_mandates_per_shard = stakes.len() - 1;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // Now there are more validators than we need, but
+        // the mandate price still doesn't go below the common stake.
+        assert_eq!(compute_mandate_price(config, &stakes), stakes[0]);
+    }
+
+    // Test cases where the stake distribution is a step function.
+    #[test]
+    fn test_step_dist() {
+        let stakes = {
+            let mut buf = [11_u128; 13];
+            let n = buf.len() / 2;
+            for s in buf.iter_mut().take(n) {
+                *s *= 5;
+            }
+            buf
+        };
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+
+        // Computed price gives whole number of seats close to the target number
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard + 5);
+
+        let target_mandates_per_shard = 2 * stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard + 11);
+
+        let target_mandates_per_shard = stakes.len() / 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+    }
+
+    // Test cases where the stake distribution is exponential.
+    #[test]
+    fn test_exp_dist() {
+        let stakes = {
+            let mut buf = vec![1_000_000_000_u128; 210];
+            let mut last_stake = buf[0];
+            for s in buf.iter_mut().skip(1) {
+                last_stake = last_stake * 97 / 100;
+                *s = last_stake;
+            }
+            buf
+        };
+
+        // This case is similar to the mainnet data.
+        let num_shards = 6;
+        let target_mandates_per_shard = 68;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard * num_shards);
+
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+
+        let target_mandates_per_shard = stakes.len() * 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+
+        let target_mandates_per_shard = stakes.len() / 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+    }
+
+    // Test cases where the stakes are chosen uniformly at random.
+    #[test]
+    fn test_rand_dist() {
+        let stakes = {
+            let mut stakes = vec![0_u128; 1000];
+            let mut rng = rand::rngs::StdRng::seed_from_u64(0xdeadbeef);
+            for s in stakes.iter_mut() {
+                *s = rng.gen_range(1_u128..10_000u128);
+            }
+            stakes
+        };
+
+        let num_shards = 1;
+        let target_mandates_per_shard = stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        // In this case it was not possible to find a seat price that exactly results
+        // in the target number of mandates. This is simply due to the discrete nature
+        // of the problem. But the algorithm still gets very close (3 out of 1000 is
+        // 0.3% off the target).
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard + 3);
+
+        let target_mandates_per_shard = 2 * stakes.len();
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+
+        let target_mandates_per_shard = stakes.len() / 2;
+        let config = ValidatorMandatesConfig::new(target_mandates_per_shard, num_shards);
+        let price = compute_mandate_price(config, &stakes);
+        assert_eq!(count_whole_mandates(&stakes, price), target_mandates_per_shard);
+    }
+
+    fn count_whole_mandates(stakes: &[u128], mandate_price: u128) -> usize {
+        saturating_sum(stakes.iter().map(|s| *s / mandate_price)) as usize
+    }
+}