Added handling of incorrect shutdown of the off-chain GraphQL worker

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## [Unreleased]
 
 ### Added
+- [#2004](https://github.com/FuelLabs/fuel-core/pull/2004): Added new CLI argument `continue-services-on-error` to control internal flow of services.
+- [#2004](https://github.com/FuelLabs/fuel-core/pull/2004): Added handling of incorrect shutdown of the off-chain GraphQL worker by using state rewind feature.
 - [#2007](https://github.com/FuelLabs/fuel-core/pull/2007): Improved metrics:
   - Added database metrics per column.
   - Added statistic about commit time of each database.

bin/fuel-core/src/cli/run.rs

@@ -127,7 +127,7 @@ pub struct Command {
     /// The default value is 7 days = 604800 blocks.
     ///
     /// The `BlockHeight` is `u32`, meaning the maximum possible number of blocks
-    /// is less than 137 years. `2^32 / 24 / 60 / 60/ 365` = `136.1925195332` years.
+    /// is less than 137 years. `2^32 / 24 / 60 / 60 / 365` = `136.1925195332` years.
     /// If the value is 136 years or more, the rewind feature is enabled for all blocks.
     #[clap(long = "state-rewind-duration", default_value = "7d", env)]
     pub state_rewind_duration: humantime::Duration,
@@ -141,6 +141,10 @@ pub struct Command {
     #[arg(name = "DB_PRUNE", long = "db-prune", env, default_value = "false")]
     pub db_prune: bool,
 
+    /// The determines whether to continue the services on internal error or not.
+    #[clap(long = "continue-services-on-error", default_value = "false", env)]
+    pub continue_on_error: bool,
+
     /// Should be used for local development only. Enabling debug mode:
     /// - Allows GraphQL Endpoints to arbitrarily advance blocks.
     /// - Enables debugger GraphQL Endpoints.
@@ -236,6 +240,7 @@ impl Command {
             state_rewind_duration,
             db_prune,
             snapshot,
+            continue_on_error,
             vm_backtrace,
             debug,
             utxo_validation,
@@ -384,6 +389,7 @@ impl Command {
             snapshot_reader,
             debug,
             native_executor_version,
+            continue_on_error,
             utxo_validation,
             block_production: trigger,
             vm: VMConfig {

crates/fuel-core/src/database.rs

@@ -278,7 +278,9 @@ where
             );
         };
         self.inner_storage().data.rollback_last_block()?;
-        *lock = height.rollback_height();
+        let new_height = height.rollback_height();
+        *lock = new_height;
+        tracing::info!("Rollback to the height {:?} was successful", new_height);
 
         Ok(())
     }

crates/fuel-core/src/database/sealed_block.rs

@@ -51,6 +51,14 @@ impl OnChainIterableKeyValueView {
         }
     }
 
+    pub fn genesis_height(&self) -> StorageResult<Option<BlockHeight>> {
+        Ok(self
+            .iter_all::<FuelBlocks>(Some(IterDirection::Forward))
+            .next()
+            .transpose()?
+            .map(|(height, _)| height))
+    }
+
     pub fn genesis_block(&self) -> StorageResult<Option<CompressedBlock>> {
         Ok(self
             .iter_all::<FuelBlocks>(Some(IterDirection::Forward))

crates/fuel-core/src/graphql_api/ports.rs

@@ -286,16 +286,24 @@ pub mod worker {
         },
     };
 
-    pub trait Transactional: Send + Sync {
-        type Transaction<'a>: OffChainDatabase
+    pub trait OnChainDatabase: Send + Sync {
+        /// Returns the latest block height.
+        fn latest_height(&self) -> StorageResult<Option<BlockHeight>>;
+    }
+
+    pub trait OffChainDatabase: Send + Sync {
+        type Transaction<'a>: OffChainDatabaseTransaction
         where
             Self: 'a;
 
+        /// Returns the latest block height.
+        fn latest_height(&self) -> StorageResult<Option<BlockHeight>>;
+
         /// Creates a write database transaction.
         fn transaction(&mut self) -> Self::Transaction<'_>;
     }
 
-    pub trait OffChainDatabase:
+    pub trait OffChainDatabaseTransaction:
         StorageMutate<OwnedMessageIds, Error = StorageError>
         + StorageMutate<OwnedCoins, Error = StorageError>
         + StorageMutate<FuelBlockIdsToHeights, Error = StorageError>
@@ -331,9 +339,15 @@ pub mod worker {
         fn commit(self) -> StorageResult<()>;
     }
 
-    pub trait BlockImporter {
+    pub trait BlockImporter: Send + Sync {
         /// Returns a stream of imported block.
         fn block_events(&self) -> BoxStream<SharedImportResult>;
+
+        /// Return the import result at the given height.
+        fn block_event_at_height(
+            &self,
+            height: Option<BlockHeight>,
+        ) -> anyhow::Result<SharedImportResult>;
     }
 
     pub trait TxPool: Send + Sync {

crates/fuel-core/src/graphql_api/storage.rs

@@ -9,7 +9,7 @@ use crate::{
             TransactionStatuses,
         },
     },
-    graphql_api::ports::worker::OffChainDatabase,
+    graphql_api::ports::worker::OffChainDatabaseTransaction,
 };
 use fuel_core_storage::{
     kv_store::{
@@ -116,7 +116,7 @@ impl StorageColumn for Column {
     }
 }
 
-impl<S> OffChainDatabase for StorageTransaction<S>
+impl<S> OffChainDatabaseTransaction for StorageTransaction<S>
 where
     S: KeyValueInspect<Column = Column> + Modifiable,
     StorageTransaction<S>: StorageMutate<OwnedMessageIds, Error = StorageError>