refactor: Prepare (re)genesis for off chain tables (#1771)

Mostly refactoring to allow for #1619 . Not final but close. Some cleanup remains here or there. That will be bundled with the actual feature itself. The `SnapshotReader` and `SnapshotWriter` (previously the `StateReader` and `StateWriter`) now: * have a single `read` (or `write`) method generic over the table type * no longer use `*Config` structs in the interface -- all reading and writing is to be done over tables * `ChainConfig` is now written and read by the `SnapshotWriter` and `SnapshotReader` (since it is actually part of the snapshot). The genesis progress is now a generic `String` -> `u64` mapping with no enumerations for the key (previously we had an enum with a variant for each table). Every table gets its own parquet file. Json is still in a single file with the `StateConfig` schema. Depending on whether we want the offchain tables in the `StateConfig` we can either drop them or include them in the next PR. I avoided enumerating the tables as much as possible to lessen coupling. A new table should ideally require: 1. a call to write::<NewTable> when generating the snapshot 2. an implementation of `ProcessState<NewTable>` to describe how to import it on regenesis 3. an implementation of `AsTable` to describe how (if at all) this table can be extracted from a in-memory `StateConfig` 4. a call to `workers.spawn::<NewTable>` to run a worker to import it from the snapshot. After this PR we'll add in the off-chain tables and identify and regenerate dependent tables (ideally with batching + resumability). --------- Co-authored-by: xgreenx <xgreenx9999@gmail.com>
FuelLabs · Mar 29, 2024 · 099853c · 099853c
1 parent 951308e
commit 099853c
Show file tree

Hide file tree

Showing 75 changed files with 121,154 additions and 150,555 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,8 @@ Description of the upcoming release here.
 ### Changed
 
 #### Breaking
+
+- [#1771](https://github.com/FuelLabs/fuel-core/pull/1771): Contract 'states' and 'balances' brought back into `ContractConfig`. Parquet now writes a file per table.
 - [1779](https://github.com/FuelLabs/fuel-core/pull/1779): Modify Relayer service to order Events from L1 by block index
 - [#1783](https://github.com/FuelLabs/fuel-core/pull/1783): The PR upgrade `fuel-vm` to `0.48.0` release. Because of some breaking changes, we also adapted our codebase to follow them: 
   - Implementation of `Default` for configs was moved under the `test-helpers` feature. The `fuel-core` binary uses testnet configuration instead of `Default::default`(for cases when `ChainConfig` was not provided by the user).
@@ -469,4 +471,4 @@ FuelVM received a lot of safety and stability improvements:
 - [#1484](https://github.com/FuelLabs/fuel-core/pull/1484): Removed `--network` CLI argument. Now the name of the network is fetched form chain configuration.
 - [#1399](https://github.com/FuelLabs/fuel-core/pull/1399): Removed `relayer-da-finalization` parameter from the relayer CLI.
 - [#1338](https://github.com/FuelLabs/fuel-core/pull/1338): Updated GraphQL client to use `DependentCost` for `k256`, `mcpi`, `s256`, `scwq`, `swwq` opcodes.
-- [#1322](https://github.com/FuelLabs/fuel-core/pull/1322): The `manual_blocks_enabled` flag is removed from the CLI. The analog is a `debug` flag.
+- [#1322](https://github.com/FuelLabs/fuel-core/pull/1322): The `manual_blocks_enabled` flag is removed from the CLI. The analog is a `debug` flag.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs
@@ -33,9 +33,10 @@ use fuel_core_benches::{
     *,
 };
 use fuel_core_chain_config::{
+    ChainConfig,
     ContractConfig,
+    SnapshotReader,
     StateConfig,
-    StateReader,
 };
 use fuel_core_services::Service;
 use fuel_core_storage::{
@@ -265,28 +266,21 @@ fn service_with_many_contracts(
     let _drop = rt.enter();
     let mut database = Database::rocksdb_temp();
 
-    let mut config = Config::local_node();
+    let mut chain_config = ChainConfig::local_testnet();
 
-    config.chain_config.consensus_parameters.set_tx_params(
+    chain_config.consensus_parameters.set_tx_params(
         TxParameters::default().with_max_gas_per_tx(TARGET_BLOCK_GAS_LIMIT),
     );
-    config
-        .chain_config
-        .consensus_parameters
-        .set_predicate_params(
-            PredicateParameters::default()
-                .with_max_gas_per_predicate(TARGET_BLOCK_GAS_LIMIT),
-        );
-    config
-        .chain_config
+    chain_config.consensus_parameters.set_predicate_params(
+        PredicateParameters::default().with_max_gas_per_predicate(TARGET_BLOCK_GAS_LIMIT),
+    );
+    chain_config
         .consensus_parameters
         .set_fee_params(FeeParameters::default().with_gas_per_byte(0));
-    config
-        .chain_config
+    chain_config
         .consensus_parameters
         .set_block_gas_limit(TARGET_BLOCK_GAS_LIMIT);
-    config
-        .chain_config
+    chain_config
         .consensus_parameters
         .set_gas_costs(GasCosts::new(default_gas_costs()));
 
@@ -297,12 +291,15 @@ fn service_with_many_contracts(
             ..Default::default()
         })
         .collect::<Vec<_>>();
+
     let state_config = StateConfig {
         contracts: contract_configs,
         ..Default::default()
     };
-    config.state_reader = StateReader::in_memory(state_config);
-
+    let mut config = Config {
+        snapshot_reader: SnapshotReader::new_in_memory(chain_config, state_config),
+        ..Config::local_node()
+    };
     config.utxo_validation = false;
     config.block_production = Trigger::Instant;
 
@@ -413,18 +410,11 @@ fn run_with_service_with_extra_inputs(
                 tx_builder.add_output(*output);
             }
             let mut tx = tx_builder.finalize_as_transaction();
-            tx.estimate_predicates(
-                &shared
-                    .config
-                    .chain_config
-                    .consensus_parameters
-                    .clone()
-                    .into(),
-            )
-            .unwrap();
+            let chain_config = shared.config.snapshot_reader.chain_config().clone();
+            tx.estimate_predicates(&chain_config.consensus_parameters.clone().into())
+                .unwrap();
             async move {
-                let tx_id =
-                    tx.id(&shared.config.chain_config.consensus_parameters.chain_id());
+                let tx_id = tx.id(&chain_config.consensus_parameters.chain_id());
 
                 let mut sub = shared.block_importer.block_importer.subscribe();
                 shared

diff --git a/benches/benches/vm_set/blockchain.rs b/benches/benches/vm_set/blockchain.rs
@@ -106,11 +106,12 @@ impl BenchDb {
         // Adds a genesis block to the database.
         let config = Config::local_node();
         let block = fuel_core::service::genesis::create_genesis_block(&config);
+        let chain_config = config.snapshot_reader.chain_config();
         database
             .storage::<FuelBlocks>()
             .insert(
                 &0u32.into(),
-                &block.compress(&config.chain_config.consensus_parameters.chain_id()),
+                &block.compress(&chain_config.consensus_parameters.chain_id()),
             )
             .unwrap();
 

diff --git a/bin/e2e-test-client/src/test_context.rs b/bin/e2e-test-client/src/test_context.rs
@@ -30,7 +30,6 @@ use fuel_core_types::{
         Finalizable,
         Input,
         Output,
-        StorageSlot,
         Transaction,
         TransactionBuilder,
         TxId,
@@ -45,6 +44,7 @@ use fuel_core_types::{
     },
     fuel_vm::SecretKey,
 };
+use itertools::Itertools;
 
 use crate::config::{
     ClientConfig,
@@ -302,7 +302,6 @@ impl Wallet {
         &self,
         config: ContractConfig,
         salt: Salt,
-        slots: Vec<StorageSlot>,
     ) -> anyhow::Result<()> {
         let asset_id = AssetId::BASE;
         let total_amount = BASE_AMOUNT;
@@ -315,10 +314,17 @@ impl Wallet {
         let ContractConfig {
             contract_id,
             code: bytes,
+            states,
             ..
         } = config;
-        let state_root = Contract::initial_state_root(slots.iter());
-        let mut tx = TransactionBuilder::create(bytes.into(), salt, slots);
+
+        let state: Vec<_> = states
+            .into_iter()
+            .map(|entry| entry.try_into())
+            .try_collect()?;
+
+        let state_root = Contract::initial_state_root(state.iter());
+        let mut tx = TransactionBuilder::create(bytes.into(), salt, state);
 
         for coin in coins {
             if let CoinType::Coin(coin) = coin {
@@ -331,6 +337,7 @@ impl Wallet {
                 );
             }
         }
+
         tx.add_output(Output::ContractCreated {
             contract_id,
             state_root,

diff --git a/bin/e2e-test-client/src/tests/contracts.rs b/bin/e2e-test-client/src/tests/contracts.rs
@@ -20,12 +20,9 @@ pub async fn deploy_large_contract(ctx: &TestContext) -> Result<(), Failed> {
         ..Default::default()
     };
     let salt = Default::default();
-    let storage_slots = vec![];
-    contract_config.update_contract_id(salt, &storage_slots);
+    contract_config.update_contract_id(salt);
 
-    let deployment_request =
-        ctx.bob
-            .deploy_contract(contract_config, salt, storage_slots);
+    let deployment_request = ctx.bob.deploy_contract(contract_config, salt);
 
     // wait for contract to deploy in 5 minutes, because 16mb takes a lot of time.
     timeout(Duration::from_secs(300), deployment_request).await??;

diff --git a/bin/e2e-test-client/src/tests/script.rs b/bin/e2e-test-client/src/tests/script.rs
@@ -12,7 +12,6 @@ use fuel_core_types::{
         field::ScriptGasLimit,
         Receipt,
         ScriptExecutionResult,
-        StorageSlot,
         Transaction,
         UniqueIdentifier,
     },
@@ -22,7 +21,7 @@ use fuel_core_types::{
     },
     services::executor::TransactionExecutionResult,
 };
-use itertools::Itertools;
+
 use libtest_mimic::Failed;
 use std::{
     path::Path,
@@ -103,51 +102,25 @@ pub async fn dry_run_multiple_txs(ctx: &TestContext) -> Result<(), Failed> {
     .await
 }
 
-fn load_contract(
-    salt: Salt,
-    path: impl AsRef<Path>,
-) -> Result<(ContractConfig, Vec<StorageSlot>), Failed> {
+fn load_contract(salt: Salt, path: impl AsRef<Path>) -> Result<ContractConfig, Failed> {
     let snapshot = SnapshotMetadata::read(path)?;
     let state_config = StateConfig::from_snapshot_metadata(snapshot)?;
-
-    let state = state_config
-        .contract_state
+    let mut contract_config = state_config
+        .contracts
         .into_iter()
-        .map(|entry| {
-            Ok::<_, core::array::TryFromSliceError>(StorageSlot::new(
-                entry.key,
-                entry.value.as_slice().try_into()?,
-            ))
-        })
-        .try_collect()?;
-
-    let contract_config = {
-        let contracts = state_config.contracts;
-
-        if contracts.len() != 1 {
-            return Err(format!(
-                "Expected to find only one contract, but found {}",
-                contracts.len()
-            )
-            .into());
-        }
-        let mut contract_config = contracts[0].clone();
-
-        contract_config.update_contract_id(salt, &state);
-
-        contract_config
-    };
+        .next()
+        .ok_or("No contract found in the state")?;
 
-    Ok((contract_config, state))
+    contract_config.update_contract_id(salt);
+    Ok(contract_config)
 }
 
 // Maybe deploy a contract with large state and execute the script
 pub async fn run_contract_large_state(ctx: &TestContext) -> Result<(), Failed> {
     let salt: Salt = "0x3b91bab936e4f3db9453046b34c142514e78b64374bf61a04ab45afbd6bca83e"
         .parse()
         .expect("Should be able to parse the salt");
-    let (contract_config, state) =
-        load_contract(salt, "./src/tests/test_data/large_state")?;
+    let contract_config = load_contract(salt, "./src/tests/test_data/large_state")?;
     let dry_run = include_bytes!("test_data/large_state/tx.json");
     let dry_run: Transaction = serde_json::from_slice(dry_run.as_ref())
         .expect("Should be able do decode the Transaction");
@@ -157,12 +130,11 @@ pub async fn run_contract_large_state(ctx: &TestContext) -> Result<(), Failed> {
     // `test_data/large_state/state_config.json` together with:
     // 244, 41, 47, 229, 13, 33, 102, 142, 20, 6, 54, 171, 105, 199, 212, 179, 208, 105, 246, 110, 185, 239, 61, 164, 176, 163, 36, 64, 156, 195, 107, 140,
     let contract_id = contract_config.contract_id;
-    println!("\nThe `contract_id` of the contract with large state: {contract_id}");
 
     // if the contract is not deployed yet, let's deploy it
     let result = ctx.bob.client.contract(&contract_id).await;
     if result?.is_none() {
-        let deployment_request = ctx.bob.deploy_contract(contract_config, salt, state);
+        let deployment_request = ctx.bob.deploy_contract(contract_config, salt);
 
         timeout(Duration::from_secs(20), deployment_request).await??;
     }

diff --git a/bin/e2e-test-client/src/tests/test_data/large_state/metadata.json b/bin/e2e-test-client/src/tests/test_data/large_state/metadata.json
@@ -1,8 +1,8 @@
 {
   "chain_config": "chain_config.json",
-  "state_encoding": {
+  "table_encoding": {
     "Json": {
       "filepath": "state_config.json"
     }
   }
-}
+}