feature: Implement compute usage aggregation and limiting

chain/chain/src/test_utils/kv_runtime.rs

@@ -1183,6 +1183,7 @@ impl RuntimeAdapter for KeyValueRuntime {
                         logs: vec![],
                         receipt_ids: new_receipt_hashes,
                         gas_burnt: 0,
+                        compute_usage: Some(0),
                         tokens_burnt: 0,
                         executor_id: to.clone(),
                         metadata: ExecutionMetadata::V1,

chain/chain/src/types.rs

@@ -637,6 +637,7 @@ mod tests {
                 logs: vec!["outcome1".to_string()],
                 receipt_ids: vec![hash(&[1])],
                 gas_burnt: 100,
+                compute_usage: Some(200),
                 tokens_burnt: 10000,
                 executor_id: "alice".parse().unwrap(),
                 metadata: ExecutionMetadata::V1,
@@ -649,6 +650,7 @@ mod tests {
                 logs: vec!["outcome2".to_string()],
                 receipt_ids: vec![],
                 gas_burnt: 0,
+                compute_usage: Some(0),
                 tokens_burnt: 0,
                 executor_id: "bob".parse().unwrap(),
                 metadata: ExecutionMetadata::V1,

chain/rosetta-rpc/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
-name = "near-rosetta-rpc"
-version = "0.0.0"
 authors.workspace = true
-publish = false
 edition.workspace = true
+name = "near-rosetta-rpc"
+publish = false
+version = "0.0.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -24,15 +24,15 @@ thiserror.workspace = true
 tokio.workspace = true
 validator.workspace = true
 
-near-primitives.workspace = true
 near-account-id.workspace = true
-near-crypto.workspace = true
 near-chain-configs.workspace = true
-near-client.workspace = true
 near-client-primitives.workspace = true
+near-client.workspace = true
+near-crypto.workspace = true
 near-network.workspace = true
 near-o11y.workspace = true
-
+near-primitives.workspace = true
+node-runtime.workspace = true
 
 [dev-dependencies]
 insta.workspace = true

chain/rosetta-rpc/src/utils.rs

@@ -271,7 +271,7 @@ where
 
 /// Zero-balance account (NEP-448)
 fn is_zero_balance_account(account: &near_primitives::account::Account) -> bool {
-    account.storage_usage() <= 770
+    account.storage_usage() <= node_runtime::ZERO_BALANCE_ACCOUNT_STORAGE_LIMIT
 }
 
 /// Tokens not locked due to staking (=liquid) but reserved for state.

core/primitives/src/runtime/parameter_table.rs

@@ -96,6 +96,10 @@ impl TryFrom<&ParameterValue> for ParameterCost {
     fn try_from(value: &ParameterValue) -> Result<Self, Self::Error> {
         match value {
             ParameterValue::ParameterCost { gas, compute } => {
+                if !cfg!(feature = "protocol_feature_compute_costs") {
+                    assert_eq!(compute, gas, "Compute cost must match gas cost");
+                }
+
                 Ok(ParameterCost { gas: *gas, compute: *compute })
             }
             // If not specified, compute costs default to gas costs.

core/primitives/src/transaction.rs

@@ -9,6 +9,7 @@ use borsh::{BorshDeserialize, BorshSerialize};
 use near_crypto::{PublicKey, Signature};
 use near_o11y::pretty;
 use near_primitives_core::profile::{ProfileDataV2, ProfileDataV3};
+use near_primitives_core::types::Compute;
 use std::borrow::Borrow;
 use std::fmt;
 use std::hash::{Hash, Hasher};
@@ -415,6 +416,13 @@ pub struct ExecutionOutcome {
     pub receipt_ids: Vec<CryptoHash>,
     /// The amount of the gas burnt by the given transaction or receipt.
     pub gas_burnt: Gas,
+    /// The amount of compute time spent by the given transaction or receipt.
+    // TODO(#8859): Treat this field in the same way as `gas_burnt`.
+    // At the moment this field is only set at runtime and is not persisted in the database.
+    // This means that when execution outcomes are read from the database, this value will not be
+    // set and any code that attempts to use it will crash.
+    #[borsh_skip]
+    pub compute_usage: Option<Compute>,
     /// The amount of tokens burnt corresponding to the burnt gas amount.
     /// This value doesn't always equal to the `gas_burnt` multiplied by the gas price, because
     /// the prepaid gas price might be lower than the actual gas price and it creates a deficit.
@@ -437,7 +445,7 @@ pub enum ExecutionMetadata {
     V1,
     /// V2: With ProfileData by legacy `Cost` enum
     V2(ProfileDataV2),
-    // V3: With ProfileData by gas parameters
+    /// V3: With ProfileData by gas parameters
     V3(ProfileDataV3),
 }
 
@@ -453,6 +461,7 @@ impl fmt::Debug for ExecutionOutcome {
             .field("logs", &pretty::Slice(&self.logs))
             .field("receipt_ids", &pretty::Slice(&self.receipt_ids))
             .field("burnt_gas", &self.gas_burnt)
+            .field("compute_usage", &self.compute_usage.unwrap_or_default())
             .field("tokens_burnt", &self.tokens_burnt)
             .field("status", &self.status)
             .field("metadata", &self.metadata)
@@ -598,6 +607,7 @@ mod tests {
             logs: vec!["123".to_string(), "321".to_string()],
             receipt_ids: vec![],
             gas_burnt: 123,
+            compute_usage: Some(456),
             tokens_burnt: 1234000,
             executor_id: "alice".parse().unwrap(),
             metadata: ExecutionMetadata::V1,

core/store/src/db.rs

@@ -224,6 +224,9 @@ pub trait Database: Sync + Send {
 
     /// Returns statistics about the database if available.
     fn get_store_statistics(&self) -> Option<StoreStatistics>;
+
+    /// Create checkpoint in provided path
+    fn create_checkpoint(&self, path: &std::path::Path) -> anyhow::Result<()>;
 }
 
 fn assert_no_overwrite(col: DBCol, key: &[u8], value: &[u8], old_value: &[u8]) {

core/store/src/db/colddb.rs

@@ -110,6 +110,10 @@ impl Database for ColdDB {
     fn get_store_statistics(&self) -> Option<crate::StoreStatistics> {
         self.cold.get_store_statistics()
     }
+
+    fn create_checkpoint(&self, path: &std::path::Path) -> anyhow::Result<()> {
+        self.cold.create_checkpoint(path)
+    }
 }
 
 /// Adjust database operation to be performed on cold storage.

core/store/src/db/rocksdb.rs

@@ -385,6 +385,12 @@ impl Database for RocksDB {
             Some(result)
         }
     }
+
+    fn create_checkpoint(&self, path: &std::path::Path) -> anyhow::Result<()> {
+        let cp = ::rocksdb::checkpoint::Checkpoint::new(&self.db)?;
+        cp.create_checkpoint(path)?;
+        Ok(())
+    }
 }
 
 /// DB level options

core/store/src/db/splitdb.rs

@@ -197,6 +197,11 @@ impl Database for SplitDB {
         log_assert_fail!("get_store_statistics is not allowed - the split storage has two stores");
         None
     }
+
+    fn create_checkpoint(&self, _path: &std::path::Path) -> anyhow::Result<()> {
+        log_assert_fail!("create_checkpoint is not allowed - the split storage has two stores");
+        Ok(())
+    }
 }
 
 #[cfg(test)]

core/store/src/db/testdb.rs

@@ -127,4 +127,8 @@ impl Database for TestDB {
     fn get_store_statistics(&self) -> Option<StoreStatistics> {
         self.stats.read().unwrap().clone()
     }
+
+    fn create_checkpoint(&self, _path: &std::path::Path) -> anyhow::Result<()> {
+        Ok(())
+    }
 }

core/store/src/lib.rs

@@ -54,7 +54,9 @@ pub mod test_utils;
 mod trie;
 
 pub use crate::config::{Mode, StoreConfig};
-pub use crate::opener::{StoreMigrator, StoreOpener, StoreOpenerError};
+pub use crate::opener::{
+    checkpoint_hot_storage_and_cleanup_columns, StoreMigrator, StoreOpener, StoreOpenerError,
+};
 
 /// Specifies temperature of a storage.
 ///

core/store/src/opener.rs

@@ -1,9 +1,10 @@
 use std::sync::Arc;
+use strum::IntoEnumIterator;
 
 use crate::db::rocksdb::snapshot::{Snapshot, SnapshotError, SnapshotRemoveError};
 use crate::db::rocksdb::RocksDB;
 use crate::metadata::{DbKind, DbMetadata, DbVersion, DB_VERSION};
-use crate::{Mode, NodeStorage, Store, StoreConfig, Temperature};
+use crate::{DBCol, DBTransaction, Mode, NodeStorage, Store, StoreConfig, Temperature};
 
 #[derive(Debug, thiserror::Error)]
 pub enum StoreOpenerError {
@@ -569,3 +570,121 @@ pub trait StoreMigrator {
     /// equal to [`DB_VERSION`].
     fn migrate(&self, store: &Store, version: DbVersion) -> anyhow::Result<()>;
 }
+
+/// Creates checkpoint of hot storage in `home_dir.join(checkpoint_relative_path)`
+///
+/// If `columns_to_keep` is None doesn't cleanup columns.
+/// Otherwise deletes all columns that are not in `columns_to_keep`.
+///
+/// Returns NodeStorage of checkpoint db.
+/// `archive` -- is hot storage archival (needed to open checkpoint).
+#[allow(dead_code)]
+pub fn checkpoint_hot_storage_and_cleanup_columns(
+    db_storage: &NodeStorage,
+    home_dir: &std::path::Path,
+    checkpoint_relative_path: std::path::PathBuf,
+    columns_to_keep: Option<Vec<DBCol>>,
+    archive: bool,
+) -> anyhow::Result<NodeStorage> {
+    let checkpoint_path = home_dir.join(checkpoint_relative_path);
+
+    db_storage.hot_storage.create_checkpoint(&checkpoint_path)?;
+
+    // As only path from config is used in StoreOpener, default config with custom path will do.
+    let mut config = StoreConfig::default();
+    config.path = Some(checkpoint_path);
+    let opener = StoreOpener::new(home_dir, archive, &config, None);
+    let node_storage = opener.open()?;
+
+    if let Some(columns_to_keep) = columns_to_keep {
+        let columns_to_keep_set: std::collections::HashSet<DBCol> =
+            std::collections::HashSet::from_iter(columns_to_keep.into_iter());
+        let mut transaction = DBTransaction::new();
+
+        for col in DBCol::iter() {
+            if !columns_to_keep_set.contains(&col) {
+                transaction.delete_all(col);
+            }
+        }
+
+        node_storage.hot_storage.write(transaction)?;
+    }
+
+    Ok(node_storage)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn check_keys_existence(store: &Store, column: &DBCol, keys: &Vec<Vec<u8>>, expected: bool) {
+        for key in keys {
+            assert_eq!(store.exists(*column, &key).unwrap(), expected, "Column {:?}", column);
+        }
+    }
+
+    #[test]
+    fn test_checkpoint_hot_storage_and_cleanup_columns() {
+        let (home_dir, opener) = NodeStorage::test_opener();
+        let node_storage = opener.open().unwrap();
+
+        let keys = vec![vec![0], vec![1], vec![2], vec![3]];
+        let columns = vec![DBCol::Block, DBCol::Chunks, DBCol::BlockHeader];
+
+        let mut store_update = node_storage.get_hot_store().store_update();
+        for column in columns {
+            for key in &keys {
+                store_update.insert(column, key, &vec![42]);
+            }
+        }
+        store_update.commit().unwrap();
+
+        let store = checkpoint_hot_storage_and_cleanup_columns(
+            &node_storage,
+            &home_dir.path(),
+            std::path::PathBuf::from("checkpoint_none"),
+            None,
+            false,
+        )
+        .unwrap();
+        check_keys_existence(&store.get_hot_store(), &DBCol::Block, &keys, true);
+        check_keys_existence(&store.get_hot_store(), &DBCol::Chunks, &keys, true);
+        check_keys_existence(&store.get_hot_store(), &DBCol::BlockHeader, &keys, true);
+
+        let store = checkpoint_hot_storage_and_cleanup_columns(
+            &node_storage,
+            &home_dir.path(),
+            std::path::PathBuf::from("checkpoint_some"),
+            Some(vec![DBCol::Block]),
+            false,
+        )
+        .unwrap();
+        check_keys_existence(&store.get_hot_store(), &DBCol::Block, &keys, true);
+        check_keys_existence(&store.get_hot_store(), &DBCol::Chunks, &keys, false);
+        check_keys_existence(&store.get_hot_store(), &DBCol::BlockHeader, &keys, false);
+
+        let store = checkpoint_hot_storage_and_cleanup_columns(
+            &node_storage,
+            &home_dir.path(),
+            std::path::PathBuf::from("checkpoint_all"),
+            Some(vec![DBCol::Block, DBCol::Chunks, DBCol::BlockHeader]),
+            false,
+        )
+        .unwrap();
+        check_keys_existence(&store.get_hot_store(), &DBCol::Block, &keys, true);
+        check_keys_existence(&store.get_hot_store(), &DBCol::Chunks, &keys, true);
+        check_keys_existence(&store.get_hot_store(), &DBCol::BlockHeader, &keys, true);
+
+        let store = checkpoint_hot_storage_and_cleanup_columns(
+            &node_storage,
+            &home_dir.path(),
+            std::path::PathBuf::from("checkpoint_empty"),
+            Some(vec![]),
+            false,
+        )
+        .unwrap();
+        check_keys_existence(&store.get_hot_store(), &DBCol::Block, &keys, false);
+        check_keys_existence(&store.get_hot_store(), &DBCol::Chunks, &keys, false);
+        check_keys_existence(&store.get_hot_store(), &DBCol::BlockHeader, &keys, false);
+    }
+}