Move updating of the owned messages and coins to off-chain worker (#1635

)

Almost closes #1583

The change moves the `OwnedCoins` and `OwnedMessageIds` tables and their
implementation to the GraphQL module. Now, the off-chain worker is
responsible for tracking owners of coins and messages. For that purpose,
`Executor` generates execution events to track which messages/coins are
consumed and created during execution. The worker uses these events and
updates corresponding tables.

It is not required to produce these events from the executor's
perspective since anyone can calculate them based on the imported block
and events from the relayer(knowing the execution rules of the state
transition). However, I decided to embed that logic into the executor to
simplify support for it. In the future, if we decide to change how coins
and messages are created/spent, we don't need to update a bunch of
places.

Events from the `ExecutionResult` are insufficient to complete the
change since we also need to support coins and messages from the genesis
block. I added a new function into the `genesis` module to perform an
update of the `OwnedCoins` and `OwnedMessageIds` tables from the
`StateConfig`(in the future, it will be done by an off-chain regenesis
process). The initial idea was to emit events, but after reviewing the
#1519 I realized that we can
have so many events that it will be hard to fit them into the memory.
So, I decided to implement a separate function that later can work with
batches and be parallelizable.

---------

Co-authored-by: Mitchell Turner <james.mitchell.turner@gmail.com>
Co-authored-by: Brandon Kite <brandonkite92@gmail.com>

CHANGELOG.md

@@ -11,6 +11,8 @@ Description of the upcoming release here.
 ### Changed
 
 - [#1658](https://github.com/FuelLabs/fuel-core/pull/1658): Removed `Receipts` table. Instead, receipts are part of the `TransactionStatuses` table.
+- [#1640](https://github.com/FuelLabs/fuel-core/pull/1640): Upgrade to fuel-vm 0.45.0.
+- [#1635](https://github.com/FuelLabs/fuel-core/pull/1635): Move updating of the owned messages and coins to off-chain worker.
 - [#1650](https://github.com/FuelLabs/fuel-core/pull/1650): Add api endpoint for getting estimates for future gas prices
 - [#1649](https://github.com/FuelLabs/fuel-core/pull/1649): Add api endpoint for getting latest gas price
 - [#1600](https://github.com/FuelLabs/fuel-core/pull/1640): Upgrade to fuel-vm 0.45.0

crates/chain-config/src/config/coin.rs

@@ -27,7 +27,7 @@ use serde_with::{
 
 #[skip_serializing_none]
 #[serde_as]
-#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
+#[derive(Default, Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
 pub struct CoinConfig {
     /// auto-generated if None
     #[serde_as(as = "Option<HexType>")]

crates/chain-config/src/config/state.rs

@@ -21,7 +21,7 @@ use super::{
 // TODO: do streaming deserialization to handle large state configs
 #[serde_as]
 #[skip_serializing_none]
-#[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq)]
+#[derive(Default, Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
 pub struct StateConfig {
     /// Spendable coins
     pub coins: Option<Vec<CoinConfig>>,

crates/fuel-core/src/coins_query.rs

@@ -229,7 +229,19 @@ mod tests {
             SpendQuery,
         },
         combined_database::CombinedDatabase,
-        fuel_core_graphql_api::api_service::ReadDatabase as ServiceDatabase,
+        fuel_core_graphql_api::{
+            api_service::ReadDatabase as ServiceDatabase,
+            storage::{
+                coins::{
+                    owner_coin_id_key,
+                    OwnedCoins,
+                },
+                messages::{
+                    OwnedMessageIds,
+                    OwnedMessageKey,
+                },
+            },
+        },
         query::asset_query::{
             AssetQuery,
             AssetSpendTarget,
@@ -921,6 +933,7 @@ mod tests {
         coin_result
     }
 
+    // TODO: Should use any mock database instead of the `fuel_core::CombinedDatabase`.
     pub struct TestDatabase {
         database: CombinedDatabase,
         last_coin_index: u64,
@@ -961,6 +974,9 @@ mod tests {
 
             let db = self.database.on_chain_mut();
             StorageMutate::<Coins>::insert(db, &id, &coin).unwrap();
+            let db = self.database.off_chain_mut();
+            let coin_by_owner = owner_coin_id_key(&owner, &id);
+            StorageMutate::<OwnedCoins>::insert(db, &coin_by_owner, &()).unwrap();
 
             coin.uncompress(id)
         }
@@ -981,6 +997,10 @@ mod tests {
 
             let db = self.database.on_chain_mut();
             StorageMutate::<Messages>::insert(db, message.id(), &message).unwrap();
+            let db = self.database.off_chain_mut();
+            let owned_message_key = OwnedMessageKey::new(&owner, &nonce);
+            StorageMutate::<OwnedMessageIds>::insert(db, &owned_message_key, &())
+                .unwrap();
 
             message
         }

crates/fuel-core/src/combined_database.rs

@@ -79,6 +79,11 @@ impl CombinedDatabase {
         &self.off_chain
     }
 
+    #[cfg(any(feature = "test-helpers", test))]
+    pub fn off_chain_mut(&mut self) -> &mut Database<OffChain> {
+        &mut self.off_chain
+    }
+
     pub fn relayer(&self) -> &Database<Relayer> {
         &self.relayer
     }

crates/fuel-core/src/database/coin.rs

@@ -1,26 +1,20 @@
-use crate::database::{
-    database_description::on_chain::OnChain,
-    Database,
+use crate::{
+    database::{
+        database_description::off_chain::OffChain,
+        Database,
+    },
+    fuel_core_graphql_api::storage::coins::{
+        owner_coin_id_key,
+        OwnedCoins,
+    },
 };
 use fuel_core_chain_config::CoinConfig;
 use fuel_core_storage::{
-    blueprint::plain::Plain,
-    codec::{
-        postcard::Postcard,
-        primitive::utxo_id_to_bytes,
-        raw::Raw,
-    },
     iter::IterDirection,
     not_found,
-    structured_storage::TableWithBlueprint,
     tables::Coins,
-    Error as StorageError,
-    Mappable,
     Result as StorageResult,
-    StorageAsMut,
     StorageAsRef,
-    StorageInspect,
-    StorageMutate,
 };
 use fuel_core_txpool::types::TxId;
 use fuel_core_types::{
@@ -30,78 +24,8 @@ use fuel_core_types::{
         UtxoId,
     },
 };
-use std::borrow::Cow;
-
-// TODO: Reuse `fuel_vm::storage::double_key` macro.
-pub fn owner_coin_id_key(owner: &Address, coin_id: &UtxoId) -> OwnedCoinKey {
-    let mut default = [0u8; Address::LEN + TxId::LEN + 1];
-    default[0..Address::LEN].copy_from_slice(owner.as_ref());
-    default[Address::LEN..].copy_from_slice(utxo_id_to_bytes(coin_id).as_ref());
-    default
-}
-
-/// The storage table of owned coin ids. Maps addresses to owned coins.
-pub struct OwnedCoins;
-/// The storage key for owned coins: `Address ++ UtxoId`
-pub type OwnedCoinKey = [u8; Address::LEN + TxId::LEN + 1];
-
-impl Mappable for OwnedCoins {
-    type Key = Self::OwnedKey;
-    type OwnedKey = OwnedCoinKey;
-    type Value = Self::OwnedValue;
-    type OwnedValue = ();
-}
 
-impl TableWithBlueprint for OwnedCoins {
-    type Blueprint = Plain<Raw, Postcard>;
-    type Column = fuel_core_storage::column::Column;
-
-    fn column() -> Self::Column {
-        Self::Column::OwnedCoins
-    }
-}
-
-impl StorageInspect<Coins> for Database {
-    type Error = StorageError;
-
-    fn get(&self, key: &UtxoId) -> Result<Option<Cow<CompressedCoin>>, Self::Error> {
-        self.data.storage::<Coins>().get(key)
-    }
-
-    fn contains_key(&self, key: &UtxoId) -> Result<bool, Self::Error> {
-        self.data.storage::<Coins>().contains_key(key)
-    }
-}
-
-impl StorageMutate<Coins> for Database {
-    fn insert(
-        &mut self,
-        key: &UtxoId,
-        value: &CompressedCoin,
-    ) -> Result<Option<CompressedCoin>, Self::Error> {
-        let coin_by_owner = owner_coin_id_key(value.owner(), key);
-        // insert primary record
-        let insert = self.data.storage_as_mut::<Coins>().insert(key, value)?;
-        // insert secondary index by owner
-        self.storage_as_mut::<OwnedCoins>()
-            .insert(&coin_by_owner, &())?;
-        Ok(insert)
-    }
-
-    fn remove(&mut self, key: &UtxoId) -> Result<Option<CompressedCoin>, Self::Error> {
-        let coin = self.data.storage_as_mut::<Coins>().remove(key)?;
-
-        // cleanup secondary index
-        if let Some(coin) = &coin {
-            let key = owner_coin_id_key(coin.owner(), key);
-            self.storage_as_mut::<OwnedCoins>().remove(&key)?;
-        }
-
-        Ok(coin)
-    }
-}
-
-impl Database<OnChain> {
+impl Database<OffChain> {
     pub fn owned_coins_ids(
         &self,
         owner: &Address,
@@ -158,22 +82,3 @@ impl Database {
         Ok(Some(configs))
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    fn generate_key(rng: &mut impl rand::Rng) -> <OwnedCoins as Mappable>::Key {
-        let mut bytes = [0u8; 65];
-        rng.fill(bytes.as_mut());
-        bytes
-    }
-
-    fuel_core_storage::basic_storage_tests!(
-        OwnedCoins,
-        [0u8; 65],
-        <OwnedCoins as Mappable>::Value::default(),
-        <OwnedCoins as Mappable>::Value::default(),
-        generate_key
-    );
-}

crates/fuel-core/src/database/database_description/on_chain.rs

@@ -22,10 +22,7 @@ impl DatabaseDescription for OnChain {
 
     fn prefix(column: &Self::Column) -> Option<usize> {
         match column {
-            Self::Column::OwnedCoins
-            | Self::Column::OwnedMessageIds
-            | Self::Column::ContractsAssets
-            | Self::Column::ContractsState => {
+            Self::Column::ContractsAssets | Self::Column::ContractsState => {
                 // prefix is address length
                 Some(32)
             }