Merge branch 'master' into key-in-parts

CHANGELOG.md

@@ -20,7 +20,6 @@
 * undo-block tool to reset the chain head from current head to its prev block. Use the tool by running: `./target/release/neard undo-block`. [#8681](https://github.com/near/nearcore/pull/8681)
 * Add prometheus metrics for expected number of blocks/chunks at the end of the epoch. [#8759](https://github.com/near/nearcore/pull/8759)
 * Node can sync State from S3. [#8789](https://github.com/near/nearcore/pull/8789)
-* The contract runtime switched to using our fork of wasmer, with various improvements. [#8912](https://github.com/near/nearcore/pull/8912)
 * Node can sync State from local filesystem. [#8913](https://github.com/near/nearcore/pull/8913)
 * Add per shard granularity for chunks in validator info metric. [#8934](https://github.com/near/nearcore/pull/8934)
 
@@ -67,14 +66,14 @@ to pay for the storage of their accounts.
 [Stabilization #8601](https://github.com/near/nearcore/pull/8601)
 
 ### Non-protocol Changes
-* Config validation can be done by following command: 
-  `./target/debug/neard --home {path_to_config_files} validate-config`. 
-  This will show error if there are file issues or semantic issues in `config.json`, `genesis.json`, `records.json`, `node_key.json` and `validator_key.json`. 
+* Config validation can be done by following command:
+  `./target/debug/neard --home {path_to_config_files} validate-config`.
+  This will show error if there are file issues or semantic issues in `config.json`, `genesis.json`, `records.json`, `node_key.json` and `validator_key.json`.
   [#8485](https://github.com/near/nearcore/pull/8485)
 * Comments are allowed in configs. This includes
   `config.json`, `genesis.json`, `node_key.json` and `validator_key.json`. You can use `//`, `#` and `/*...*/` for comments.
   [#8423](https://github.com/near/nearcore/pull/8423)
-* `/debug` page now has client_config linked. 
+* `/debug` page now has client_config linked.
   You can also check your client_config directly at /debug/client_config
   [#8400](https://github.com/near/nearcore/pull/8400)
 * Added cold store loop - a background thread that copies data from hot to cold storage and a new json rpc endpoing - split_storage_info - that

chain/chain/src/flat_storage_creator.rs

@@ -14,16 +14,18 @@ use assert_matches::assert_matches;
 use crossbeam_channel::{unbounded, Receiver, Sender};
 use near_chain_primitives::Error;
 use near_primitives::shard_layout::ShardUId;
+use near_primitives::state::ValueRef;
 use near_primitives::state_part::PartId;
 use near_primitives::types::{AccountId, BlockHeight, StateRoot};
 use near_store::flat::{
-    store_helper, BlockInfo, FetchingStateStatus, FlatStateChanges, FlatStateValue,
-    FlatStorageCreationMetrics, FlatStorageCreationStatus, FlatStorageReadyStatus,
-    FlatStorageStatus, NUM_PARTS_IN_ONE_STEP, STATE_PART_MEMORY_LIMIT,
+    store_helper, BlockInfo, FetchingStateStatus, FlatStateChanges, FlatStorageCreationMetrics,
+    FlatStorageCreationStatus, FlatStorageReadyStatus, FlatStorageStatus, NUM_PARTS_IN_ONE_STEP,
+    STATE_PART_MEMORY_LIMIT,
 };
 use near_store::Store;
 use near_store::{Trie, TrieDBStorage, TrieTraversalItem};
 use std::collections::HashMap;
+use std::rc::Rc;
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
 use tracing::{debug, info};
@@ -89,7 +91,7 @@ impl FlatStorageShardCreator {
         result_sender: Sender<u64>,
     ) {
         let trie_storage = TrieDBStorage::new(store.clone(), shard_uid);
-        let trie = Trie::new(Box::new(trie_storage), state_root, None);
+        let trie = Trie::new(Rc::new(trie_storage), state_root, None);
         let path_begin = trie.find_state_part_boundary(part_id.idx, part_id.total).unwrap();
         let path_end = trie.find_state_part_boundary(part_id.idx + 1, part_id.total).unwrap();
         let hex_path_begin = Self::nibbles_to_hex(&path_begin);
@@ -103,13 +105,9 @@ impl FlatStorageShardCreator {
         {
             if let Some(key) = key {
                 let value = trie.storage.retrieve_raw_bytes(&hash).unwrap();
-                store_helper::set_flat_state_value(
-                    &mut store_update,
-                    shard_uid,
-                    key,
-                    Some(FlatStateValue::value_ref(&value)),
-                )
-                .expect("Failed to put value in FlatState");
+                let value_ref = ValueRef::new(&value);
+                store_helper::set_ref(&mut store_update, shard_uid, key, Some(value_ref))
+                    .expect("Failed to put value in FlatState");
                 num_items += 1;
             }
         }
@@ -186,7 +184,7 @@ impl FlatStorageShardCreator {
                     let trie_storage = TrieDBStorage::new(store, shard_uid);
                     let state_root =
                         *chain_store.get_chunk_extra(&block_hash, &shard_uid)?.state_root();
-                    let trie = Trie::new(Box::new(trie_storage), state_root, None);
+                    let trie = Trie::new(Rc::new(trie_storage), state_root, None);
                     let root_node = trie.retrieve_root_node().unwrap();
                     let num_state_parts =
                         root_node.memory_usage / STATE_PART_MEMORY_LIMIT.as_u64() + 1;

core/primitives-core/src/config.rs

@@ -210,8 +210,8 @@ impl VMLimitConfig {
 
             // NOTE: Stack height has to be 16K, otherwise Wasmer produces non-deterministic results.
             // For experimentation try `test_stack_overflow`.
-            max_stack_height: 256 * 1024, // 256kiB of stack.
-            contract_prepare_version: ContractPrepareVersion::V2,
+            max_stack_height: 16 * 1024, // 16Kib of stack.
+            contract_prepare_version: ContractPrepareVersion::V1,
             initial_memory_pages: 2u32.pow(10), // 64Mib of memory.
             max_memory_pages: 2u32.pow(11),     // 128Mib of memory.
 

core/primitives/res/runtime_configs/61.yaml

@@ -1,6 +1,3 @@
-max_stack_height: { old: 16384, new: 262144 }
-contract_prepare_version: { old: 1, new: 2 }
-
 # Compute costs to allow for flat storage read-only MVP.
 # See https://github.com/near/nearcore/issues/8006
 wasm_touching_trie_node:    { old: 16_101_955_926, new: { gas: 16_101_955_926, compute: 110_000_000_000 } }

core/primitives/res/runtime_configs/parameters.snap

@@ -138,8 +138,8 @@ wasm_alt_bn128_g1_sum_base                     3_000_000_000
 wasm_alt_bn128_g1_sum_element                  5_000_000_000
 max_gas_burnt                            300_000_000_000_000
 max_gas_burnt_view                       300_000_000_000_000
-max_stack_height                                     262_144
-contract_prepare_version                                   2
+max_stack_height                                      16_384
+contract_prepare_version                                   1
 initial_memory_pages                                   1_024
 max_memory_pages                                       2_048
 registers_memory_limit                         1_073_741_824

core/primitives/src/runtime/snapshots/near_primitives__runtime__config_store__tests__61.json.snap

@@ -174,8 +174,8 @@ expression: config_view
     "regular_op_cost": 822756,
     "limit_config": {
       "max_gas_burnt": 300000000000000,
-      "max_stack_height": 262144,
-      "contract_prepare_version": 2,
+      "max_stack_height": 16384,
+      "contract_prepare_version": 1,
       "initial_memory_pages": 1024,
       "max_memory_pages": 2048,
       "registers_memory_limit": 1073741824,

core/primitives/src/runtime/snapshots/near_primitives__runtime__config_store__tests__testnet_61.json.snap

@@ -174,8 +174,8 @@ expression: config_view
     "regular_op_cost": 822756,
     "limit_config": {
       "max_gas_burnt": 300000000000000,
-      "max_stack_height": 262144,
-      "contract_prepare_version": 2,
+      "max_stack_height": 16384,
+      "contract_prepare_version": 1,
       "initial_memory_pages": 1024,
       "max_memory_pages": 2048,
       "registers_memory_limit": 1073741824,

core/primitives/src/snapshots/near_primitives__views__tests__runtime_config_view.snap

@@ -174,8 +174,8 @@ expression: "&view"
     "regular_op_cost": 3856371,
     "limit_config": {
       "max_gas_burnt": 200000000000000,
-      "max_stack_height": 262144,
-      "contract_prepare_version": 2,
+      "max_stack_height": 16384,
+      "contract_prepare_version": 1,
       "initial_memory_pages": 1024,
       "max_memory_pages": 2048,
       "registers_memory_limit": 1073741824,

core/primitives/src/version.rs

@@ -89,12 +89,6 @@ pub enum ProtocolFeature {
     /// Although wasmer2 is faster, we don't change fees with this protocol
     /// version -- we can safely do that in a separate step.
     Wasmer2,
-    /// This feature switch our WASM engine implementation from wasmer 2.* to
-    /// near-vm, bringing better performance and reliability.
-    ///
-    /// Although near-vm is faster, we don't change fees with this protocol
-    /// version -- we can safely do that in a separate step.
-    NearVm,
     SimpleNightshade,
     LowerDataReceiptAndEcrecoverBaseCost,
     /// Lowers the cost of wasm instruction due to switch to wasmer2.
@@ -250,9 +244,7 @@ impl ProtocolFeature {
             ProtocolFeature::Ed25519Verify
             | ProtocolFeature::ZeroBalanceAccount
             | ProtocolFeature::DelegateAction => 59,
-            ProtocolFeature::ComputeCosts
-            | ProtocolFeature::NearVm
-            | ProtocolFeature::FlatStorageReads => 61,
+            ProtocolFeature::ComputeCosts | ProtocolFeature::FlatStorageReads => 61,
 
             // Nightly features
             #[cfg(feature = "protocol_feature_fix_staking_threshold")]

core/store/src/flat/chunk_view.rs

@@ -1,8 +1,9 @@
 use near_primitives::hash::CryptoHash;
+use near_primitives::state::ValueRef;
 
 use crate::Store;
 
-use super::{FlatStateValue, FlatStorage};
+use super::FlatStorage;
 
 /// Struct for getting value references from the flat storage, corresponding
 /// to some block defined in `blocks_to_head`.
@@ -38,7 +39,7 @@ impl FlatStorageChunkView {
     /// they are stored in `DBCol::State`. Also the separation is done so we
     /// could charge users for the value length before loading the value.
     // TODO (#7327): consider inlining small values, so we could use only one db access.
-    pub fn get_value(&self, key: &[u8]) -> Result<Option<FlatStateValue>, crate::StorageError> {
-        self.flat_storage.get_value(&self.block_hash, key)
+    pub fn get_ref(&self, key: &[u8]) -> Result<Option<ValueRef>, crate::StorageError> {
+        self.flat_storage.get_ref(&self.block_hash, key)
     }
 }

core/store/src/flat/delta.rs

@@ -7,8 +7,7 @@ use near_primitives::types::RawStateChangesWithTrieKey;
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use super::types::INLINE_DISK_VALUE_THRESHOLD;
-use super::{store_helper, BlockInfo, FlatStateValue};
+use super::{store_helper, BlockInfo};
 use crate::{CryptoHash, StoreUpdate};
 
 #[derive(Debug)]
@@ -35,14 +34,14 @@ impl KeyForFlatStateDelta {
         res
     }
 }
-/// Delta of the state for some shard and block, stores mapping from keys to values
-/// or None, if key was removed in this block.
+/// Delta of the state for some shard and block, stores mapping from keys to value refs or None, if key was removed in
+/// this block.
 #[derive(BorshSerialize, BorshDeserialize, Clone, Default, PartialEq, Eq)]
-pub struct FlatStateChanges(pub(crate) HashMap<Vec<u8>, Option<FlatStateValue>>);
+pub struct FlatStateChanges(pub(crate) HashMap<Vec<u8>, Option<ValueRef>>);
 
 impl<T> From<T> for FlatStateChanges
 where
-    T: IntoIterator<Item = (Vec<u8>, Option<FlatStateValue>)>,
+    T: IntoIterator<Item = (Vec<u8>, Option<ValueRef>)>,
 {
     fn from(iter: T) -> Self {
         Self(HashMap::from_iter(iter))
@@ -58,17 +57,13 @@ impl std::fmt::Debug for FlatStateChanges {
 }
 
 impl FlatStateChanges {
-    /// Returns `Some(Option<FlatStateValue>)` from delta for the given key. If key is not present, returns None.
-    pub fn get(&self, key: &[u8]) -> Option<Option<FlatStateValue>> {
+    /// Returns `Some(Option<ValueRef>)` from delta for the given key. If key is not present, returns None.
+    pub fn get(&self, key: &[u8]) -> Option<Option<ValueRef>> {
         self.0.get(key).cloned()
     }
 
     /// Inserts a key-value pair to delta.
-    pub fn insert(
-        &mut self,
-        key: Vec<u8>,
-        value: Option<FlatStateValue>,
-    ) -> Option<Option<FlatStateValue>> {
+    pub fn insert(&mut self, key: Vec<u8>, value: Option<ValueRef>) -> Option<Option<ValueRef>> {
         self.0.insert(key, value)
     }
 
@@ -93,23 +88,20 @@ impl FlatStateChanges {
                 .last()
                 .expect("Committed entry should have at least one change")
                 .data;
-            let flat_state_value = last_change.as_ref().map(|value| {
-                if value.len() <= INLINE_DISK_VALUE_THRESHOLD {
-                    FlatStateValue::inlined(value)
-                } else {
-                    FlatStateValue::value_ref(value)
+            match last_change {
+                Some(value) => {
+                    delta.insert(key, Some(near_primitives::state::ValueRef::new(value)))
                 }
-            });
-            delta.insert(key, flat_state_value);
+                None => delta.insert(key, None),
+            };
         }
         Self(delta)
     }
 
     /// Applies delta to the flat state.
     pub fn apply_to_flat_state(self, store_update: &mut StoreUpdate, shard_uid: ShardUId) {
         for (key, value) in self.0.into_iter() {
-            store_helper::set_flat_state_value(store_update, shard_uid, key, value)
-                .expect("Borsh cannot fail");
+            store_helper::set_ref(store_update, shard_uid, key, value).expect("Borsh cannot fail");
         }
     }
 }
@@ -125,13 +117,7 @@ pub struct CachedFlatStateDelta {
 
 impl From<FlatStateChanges> for CachedFlatStateChanges {
     fn from(delta: FlatStateChanges) -> Self {
-        Self(
-            delta
-                .0
-                .into_iter()
-                .map(|(key, value)| (hash(&key), value.map(|v| v.to_value_ref())))
-                .collect(),
-        )
+        Self(delta.0.into_iter().map(|(key, value)| (hash(&key), value)).collect())
     }
 }
 
@@ -158,9 +144,8 @@ impl CachedFlatStateChanges {
 
 #[cfg(test)]
 mod tests {
-    use crate::flat::FlatStateValue;
-
     use super::FlatStateChanges;
+    use near_primitives::state::ValueRef;
     use near_primitives::trie_key::TrieKey;
     use near_primitives::types::{RawStateChange, RawStateChangesWithTrieKey, StateChangeCause};
 
@@ -223,17 +208,17 @@ mod tests {
         let flat_state_changes = FlatStateChanges::from_state_changes(&state_changes);
         assert_eq!(
             flat_state_changes.get(&alice_trie_key.to_vec()),
-            Some(Some(FlatStateValue::inlined(&[3, 4])))
+            Some(Some(ValueRef::new(&[3, 4])))
         );
         assert_eq!(flat_state_changes.get(&bob_trie_key.to_vec()), Some(None));
         assert_eq!(flat_state_changes.get(&carol_trie_key.to_vec()), None);
         assert_eq!(
             flat_state_changes.get(&delayed_trie_key.to_vec()),
-            Some(Some(FlatStateValue::inlined(&[1])))
+            Some(Some(ValueRef::new(&[1])))
         );
         assert_eq!(
             flat_state_changes.get(&delayed_receipt_trie_key.to_vec()),
-            Some(Some(FlatStateValue::inlined(&[2])))
+            Some(Some(ValueRef::new(&[2])))
         );
     }
 
@@ -242,23 +227,23 @@ mod tests {
     #[test]
     fn flat_state_changes_merge() {
         let mut changes = FlatStateChanges::from([
-            (vec![1], Some(FlatStateValue::value_ref(&[4]))),
-            (vec![2], Some(FlatStateValue::value_ref(&[5]))),
+            (vec![1], Some(ValueRef::new(&[4]))),
+            (vec![2], Some(ValueRef::new(&[5]))),
             (vec![3], None),
-            (vec![4], Some(FlatStateValue::value_ref(&[6]))),
+            (vec![4], Some(ValueRef::new(&[6]))),
         ]);
         let changes_new = FlatStateChanges::from([
-            (vec![2], Some(FlatStateValue::value_ref(&[7]))),
-            (vec![3], Some(FlatStateValue::value_ref(&[8]))),
+            (vec![2], Some(ValueRef::new(&[7]))),
+            (vec![3], Some(ValueRef::new(&[8]))),
             (vec![4], None),
-            (vec![5], Some(FlatStateValue::value_ref(&[9]))),
+            (vec![5], Some(ValueRef::new(&[9]))),
         ]);
         changes.merge(changes_new);
 
-        assert_eq!(changes.get(&[1]), Some(Some(FlatStateValue::value_ref(&[4]))));
-        assert_eq!(changes.get(&[2]), Some(Some(FlatStateValue::value_ref(&[7]))));
-        assert_eq!(changes.get(&[3]), Some(Some(FlatStateValue::value_ref(&[8]))));
+        assert_eq!(changes.get(&[1]), Some(Some(ValueRef::new(&[4]))));
+        assert_eq!(changes.get(&[2]), Some(Some(ValueRef::new(&[7]))));
+        assert_eq!(changes.get(&[3]), Some(Some(ValueRef::new(&[8]))));
         assert_eq!(changes.get(&[4]), Some(None));
-        assert_eq!(changes.get(&[5]), Some(Some(FlatStateValue::value_ref(&[9]))));
+        assert_eq!(changes.get(&[5]), Some(Some(ValueRef::new(&[9]))));
     }
 }

core/store/src/flat/mod.rs

@@ -39,7 +39,7 @@ pub use manager::FlatStorageManager;
 pub use metrics::FlatStorageCreationMetrics;
 pub use storage::FlatStorage;
 pub use types::{
-    BlockInfo, FetchingStateStatus, FlatStateValue, FlatStorageCreationStatus, FlatStorageError,
+    BlockInfo, FetchingStateStatus, FlatStorageCreationStatus, FlatStorageError,
     FlatStorageReadyStatus, FlatStorageStatus,
 };