From 9a6efa1bc05607ab0de245186e86bffecd874c4c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 3 Jul 2024 15:48:50 +0200 Subject: [PATCH 1/4] fix: avoid tokio executor blocked in aggregator when importing Cardano transactions This is happening when sync database code is blocking the executor, and thus making the node completely unavailable (REST API included). --- .../services/cardano_transactions_importer.rs | 23 +++++++++++++++---- .../tests/create_certificate.rs | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mithril-aggregator/src/services/cardano_transactions_importer.rs b/mithril-aggregator/src/services/cardano_transactions_importer.rs index ba73188ca40..dc81c1b94fe 100644 --- a/mithril-aggregator/src/services/cardano_transactions_importer.rs +++ b/mithril-aggregator/src/services/cardano_transactions_importer.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; use slog::{debug, Logger}; +use tokio::{runtime::Handle, task}; use mithril_common::cardano_block_scanner::{BlockScanner, ChainScannedBlocks}; use mithril_common::crypto_helper::{MKTree, MKTreeNode}; @@ -174,13 +175,25 @@ impl CardanoTransactionsImporter { .store_block_range_roots(block_ranges_with_merkle_root) .await } + + async fn import_transactions_and_block_ranges( + &self, + up_to_beacon: BlockNumber, + ) -> StdResult<()> { + self.import_transactions(up_to_beacon).await?; + self.import_block_ranges().await + } } #[async_trait] impl TransactionsImporter for CardanoTransactionsImporter { async fn import(&self, up_to_beacon: BlockNumber) -> StdResult<()> { - self.import_transactions(up_to_beacon).await?; - self.import_block_ranges().await + task::block_in_place(move || { + Handle::current().block_on(async move { + self.import_transactions_and_block_ranges(up_to_beacon) + .await + }) + }) } } @@ -639,7 +652,7 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn importing_twice_starting_with_nothing_in_a_real_db_should_yield_transactions_in_same_order( ) { let blocks = vec![ @@ -676,7 +689,7 @@ mod tests { assert_eq!(cold_imported_transactions, warm_imported_transactions); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn when_rollbackward_should_remove_transactions() { let connection = cardano_tx_db_connection().unwrap(); let repository = Arc::new(CardanoTransactionRepository::new(Arc::new( @@ -719,7 +732,7 @@ mod tests { assert_eq!(expected_remaining_transactions, stored_transactions); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn when_rollbackward_should_remove_block_ranges() { let connection = cardano_tx_db_connection().unwrap(); let repository = Arc::new(CardanoTransactionRepository::new(Arc::new( diff --git a/mithril-aggregator/tests/create_certificate.rs b/mithril-aggregator/tests/create_certificate.rs index c7e9b93d8b1..324a1aba678 100644 --- a/mithril-aggregator/tests/create_certificate.rs +++ b/mithril-aggregator/tests/create_certificate.rs @@ -10,7 +10,7 @@ use mithril_common::{ }; use test_extensions::{utilities::get_test_dir, ExpectedCertificate, RuntimeTester}; -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn create_certificate() { let protocol_parameters = ProtocolParameters { k: 5, From c17c72d2b2d920cff1e52aaf8f2f4a0570a58ed5 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 3 Jul 2024 15:54:18 +0200 Subject: [PATCH 2/4] fix: avoid tokio executor blocked in signer when importing Cardano transactions This is happening when sync database code is blocking the executor, and thus making the node completely unavailable. --- .../src/cardano_transactions_importer.rs | 23 +++++++++++++++---- ...te_cardano_transaction_single_signature.rs | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mithril-signer/src/cardano_transactions_importer.rs b/mithril-signer/src/cardano_transactions_importer.rs index ba73188ca40..dc81c1b94fe 100644 --- a/mithril-signer/src/cardano_transactions_importer.rs +++ b/mithril-signer/src/cardano_transactions_importer.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; use slog::{debug, Logger}; +use tokio::{runtime::Handle, task}; use mithril_common::cardano_block_scanner::{BlockScanner, ChainScannedBlocks}; use mithril_common::crypto_helper::{MKTree, MKTreeNode}; @@ -174,13 +175,25 @@ impl CardanoTransactionsImporter { .store_block_range_roots(block_ranges_with_merkle_root) .await } + + async fn import_transactions_and_block_ranges( + &self, + up_to_beacon: BlockNumber, + ) -> StdResult<()> { + self.import_transactions(up_to_beacon).await?; + self.import_block_ranges().await + } } #[async_trait] impl TransactionsImporter for CardanoTransactionsImporter { async fn import(&self, up_to_beacon: BlockNumber) -> StdResult<()> { - self.import_transactions(up_to_beacon).await?; - self.import_block_ranges().await + task::block_in_place(move || { + Handle::current().block_on(async move { + self.import_transactions_and_block_ranges(up_to_beacon) + .await + }) + }) } } @@ -639,7 +652,7 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn importing_twice_starting_with_nothing_in_a_real_db_should_yield_transactions_in_same_order( ) { let blocks = vec![ @@ -676,7 +689,7 @@ mod tests { assert_eq!(cold_imported_transactions, warm_imported_transactions); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn when_rollbackward_should_remove_transactions() { let connection = cardano_tx_db_connection().unwrap(); let repository = Arc::new(CardanoTransactionRepository::new(Arc::new( @@ -719,7 +732,7 @@ mod tests { assert_eq!(expected_remaining_transactions, stored_transactions); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn when_rollbackward_should_remove_block_ranges() { let connection = cardano_tx_db_connection().unwrap(); let repository = Arc::new(CardanoTransactionRepository::new(Arc::new( diff --git a/mithril-signer/tests/create_cardano_transaction_single_signature.rs b/mithril-signer/tests/create_cardano_transaction_single_signature.rs index 42b19245755..52cb5710b02 100644 --- a/mithril-signer/tests/create_cardano_transaction_single_signature.rs +++ b/mithril-signer/tests/create_cardano_transaction_single_signature.rs @@ -9,7 +9,7 @@ use mithril_common::{ use test_extensions::StateMachineTester; #[rustfmt::skip] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_create_cardano_transaction_single_signature() { let protocol_parameters = tests_setup::setup_protocol_parameters(); let fixture = MithrilFixtureBuilder::default() From 1ce35cbf4a25f0820e7c1f51ae55342e15b1aee1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 3 Jul 2024 15:56:21 +0200 Subject: [PATCH 3/4] docs: update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d963f159a6a..30bca31a34a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ As a minor extension, we have adopted a slightly different versioning convention - Chunk the Cardano transactions import in `mithril-signer` to reduce disk footprint by running the pruning process more frequently. - Add a database connection pool on the Cardano transaction repository for increased performances of the prover. - Import Cardano transactions with Chain Sync mini protocol and Pallas chain reader. + - Avoid aggregator and signer being blocked when importing the Cardano transactions. - Crates versions: From 217f80c7e49f8f71577c6b3ad00baeb5c8cb11f5 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 3 Jul 2024 15:58:19 +0200 Subject: [PATCH 4/4] chore: bump crates versions - 'mithril-aggregator' from '0.5.33' to '0.5.34' - 'mithril-signer' from '0.2.156' to '0.2.157'. --- Cargo.lock | 4 ++-- mithril-aggregator/Cargo.toml | 2 +- mithril-signer/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6acb427097..aa3107d3995 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3547,7 +3547,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.5.33" +version = "0.5.34" dependencies = [ "anyhow", "async-trait", @@ -3848,7 +3848,7 @@ dependencies = [ [[package]] name = "mithril-signer" -version = "0.2.156" +version = "0.2.157" dependencies = [ "anyhow", "async-trait", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 77b1ae7a8cb..c3fcf928e70 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.5.33" +version = "0.5.34" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml index b890b1d168f..34ae8565bec 100644 --- a/mithril-signer/Cargo.toml +++ b/mithril-signer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-signer" -version = "0.2.156" +version = "0.2.157" description = "A Mithril Signer" authors = { workspace = true } edition = { workspace = true }