From d4eb66ff0de0d79edcbb6394f559c2f6dd55d741 Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Thu, 9 May 2024 12:17:32 -0700 Subject: [PATCH] feat: Move shard-assignment for chunk producers to nightly (#11203) Per #11190, move the shard-assignment shuffling feature for chunk producers to nightly. Also enable this feature for mocknet together with stateless validation to exercise the state sync code path more often. Note that the guarding for the feature in nightly/prod is different from mocknet, as we use mocknet to exercise (stateless validation) features in between prod and nightly. After this change, shard shuffling will be activated in the following cases: 1) In "nightly", it will be active by default, since we move the feature among the set of nightly features and update the protocol version accordingly (143). 2) In "mocknet", it will be active by default, as we enable it directly when chain id == "mocknet". 3) In all other networks (including "mainnet", "testnet", and "statelessnet"), it will be disabled by default and will only be enabled (later) explicitly by moving the feature from nightly to stable protocol version. Compiling the binary with or without "statelessnet_protocol" will not have any effect on the enablement of the feature (instead need to compile with "nightly"). --- chain/chain/src/tests/simple_chain.rs | 4 +- core/primitives-core/src/chains.rs | 3 ++ core/primitives-core/src/version.rs | 11 ++--- core/primitives/src/epoch_manager.rs | 59 +++++++++++++++++---------- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/chain/chain/src/tests/simple_chain.rs b/chain/chain/src/tests/simple_chain.rs index 55afe8ada7d..3243f9c98a4 100644 --- a/chain/chain/src/tests/simple_chain.rs +++ b/chain/chain/src/tests/simple_chain.rs @@ -32,7 +32,7 @@ fn build_chain() { // cargo insta test --accept -p near-chain --features nightly -- tests::simple_chain::build_chain let hash = chain.head().unwrap().last_block_hash; if cfg!(feature = "nightly") { - insta::assert_snapshot!(hash, @"CJaRGRZy7GE3KkSp55HE8VheYHzPr11nRpsh9rK9F1ag"); + insta::assert_snapshot!(hash, @"C3zeKRZubVungxfrSdq379TSCYnuz2YzjEkcJTdm3pU4"); } else { insta::assert_snapshot!(hash, @"2WHohfYksQnwKwSEoTKpkseu2RWthbGf9kmGetgHgfQQ"); } @@ -50,7 +50,7 @@ fn build_chain() { let hash = chain.head().unwrap().last_block_hash; if cfg!(feature = "nightly") { - insta::assert_snapshot!(hash, @"HoQty43QCe2RPp3iZWv61TaJWW18pTqaHu2t5hWtnVir"); + insta::assert_snapshot!(hash, @"EjLaoHRiAdRp2NcDqwbMcAYYxGfcv5R7GuYUNfRpaJvB"); } else { insta::assert_snapshot!(hash, @"HJuuENeSwwikoR9BZA7cSonxAPZgY5mKQWL2pSXwjAwZ"); } diff --git a/core/primitives-core/src/chains.rs b/core/primitives-core/src/chains.rs index 93d039f28db..5ef7252bb83 100644 --- a/core/primitives-core/src/chains.rs +++ b/core/primitives-core/src/chains.rs @@ -8,3 +8,6 @@ pub const TESTNET: &str = "testnet"; /// Temporary StatelessNet active from 2024-02-01. pub const STATELESSNET: &str = "statelessnet"; + +/// Pre-release testing environment. +pub const MOCKNET: &str = "mocknet"; diff --git a/core/primitives-core/src/version.rs b/core/primitives-core/src/version.rs index 45b8961518c..36d18036e47 100644 --- a/core/primitives-core/src/version.rs +++ b/core/primitives-core/src/version.rs @@ -149,9 +149,8 @@ pub enum ProtocolFeature { SingleShardTracking, // Stateless validation: state witness size limits. StateWitnessSizeLimit, - // Stateless validation: in statelessnet, shuffle shard assignments for chunk producers every - // epoch. - StatelessnetShuffleShardAssignmentsForChunkProducers, + // Shuffle shard assignments for chunk producers at every epoch. + ShuffleShardAssignments, // Stateless validation: limit the size of storage proof generated by a single receipt. // Receipts which generate storage proofs larger than this limit will be rejected. // Protocol 85 also decreased the soft per-chunk storage proof limit to 3MB. @@ -222,7 +221,6 @@ impl ProtocolFeature { ProtocolFeature::LowerValidatorKickoutPercentForDebugging => 81, ProtocolFeature::SingleShardTracking => 82, ProtocolFeature::StateWitnessSizeLimit => 83, - ProtocolFeature::StatelessnetShuffleShardAssignmentsForChunkProducers => 84, ProtocolFeature::PerReceiptHardStorageProofLimit => 85, ProtocolFeature::PartialEncodedStateWitness => 86, @@ -237,6 +235,9 @@ impl ProtocolFeature { #[cfg(feature = "protocol_feature_nonrefundable_transfer_nep491")] ProtocolFeature::NonrefundableStorage => 140, ProtocolFeature::CongestionControl => 142, + // TODO(#11201): When stabilizing this feature in mainnet, also remove the temporary code + // that always enables this for mocknet (see config_mocknet function). + ProtocolFeature::ShuffleShardAssignments => 143, } } @@ -256,7 +257,7 @@ pub const PROTOCOL_VERSION: ProtocolVersion = if cfg!(feature = "statelessnet_pr 86 } else if cfg!(feature = "nightly_protocol") { // On nightly, pick big enough version to support all features. - 142 + 143 } else { // Enable all stable features. STABLE_PROTOCOL_VERSION diff --git a/core/primitives/src/epoch_manager.rs b/core/primitives/src/epoch_manager.rs index 8ecc3a80d31..fe36145dcc0 100644 --- a/core/primitives/src/epoch_manager.rs +++ b/core/primitives/src/epoch_manager.rs @@ -132,12 +132,16 @@ impl AllEpochConfig { pub fn for_protocol_version(&self, protocol_version: ProtocolVersion) -> EpochConfig { let mut config = self.genesis_epoch_config.clone(); + Self::config_mocknet(&mut config, &self.chain_id); + Self::config_stateless_net(&mut config, &self.chain_id, protocol_version); if !self.use_production_config { return config; } + Self::config_validator_selection(&mut config, protocol_version); + Self::config_nightshade(&mut config, protocol_version); Self::config_chunk_only_producers(&mut config, &self.chain_id, protocol_version); @@ -153,33 +157,44 @@ impl AllEpochConfig { &self.chain_id } + /// Configures mocknet-specific features only. + fn config_mocknet(config: &mut EpochConfig, chain_id: &str) { + if chain_id != near_primitives_core::chains::MOCKNET { + return; + } + // In production (mainnet/testnet) and nightly environments this setting is guarded by + // ProtocolFeature::ShuffleShardAssignments. (see config_validator_selection function). + // For pre-release environment such as mocknet, which uses features between production and nightly + // (eg. stateless validation) we enable it by default with stateless validation in order to exercise + // the codepaths for state sync more often. + // TODO(#11201): When stabilizing "ShuffleShardAssignments" in mainnet, + // also remove this temporary code and always rely on ShuffleShardAssignments. + config.validator_selection_config.shuffle_shard_assignment_for_chunk_producers = true; + } + + /// Configures statelessnet-specific features only. fn config_stateless_net( config: &mut EpochConfig, chain_id: &str, protocol_version: ProtocolVersion, ) { - // StatelessNet only. - if chain_id == near_primitives_core::chains::STATELESSNET { - // Lower the kickout threshold so the network is more stable while - // we figure out issues with block and chunk production. - if checked_feature!( - "stable", - LowerValidatorKickoutPercentForDebugging, - protocol_version - ) { - config.block_producer_kickout_threshold = 50; - config.chunk_producer_kickout_threshold = 50; - } - // Shuffle shard assignments every epoch, to trigger state sync more - // frequently to exercise that code path. - if checked_feature!( - "stable", - StatelessnetShuffleShardAssignmentsForChunkProducers, - protocol_version - ) { - config.validator_selection_config.shuffle_shard_assignment_for_chunk_producers = - true; - } + if chain_id != near_primitives_core::chains::STATELESSNET { + return; + } + // Lower the kickout threshold so the network is more stable while + // we figure out issues with block and chunk production. + if checked_feature!("stable", LowerValidatorKickoutPercentForDebugging, protocol_version) { + config.block_producer_kickout_threshold = 50; + config.chunk_producer_kickout_threshold = 50; + } + } + + /// Configures validator-selection related features. + fn config_validator_selection(config: &mut EpochConfig, protocol_version: ProtocolVersion) { + // Shuffle shard assignments every epoch, to trigger state sync more + // frequently to exercise that code path. + if checked_feature!("stable", ShuffleShardAssignments, protocol_version) { + config.validator_selection_config.shuffle_shard_assignment_for_chunk_producers = true; } }