From 35a85861080634fe076b22a0c2cdb13382d71ad3 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Mon, 23 Sep 2019 14:22:59 +0300 Subject: [PATCH 01/22] Split bootstrap core_test --- nano/core_test/CMakeLists.txt | 1 + nano/core_test/bootstrap.cpp | 715 ++++++++++++++++++++++++++++++++++ nano/core_test/network.cpp | 715 +--------------------------------- 3 files changed, 719 insertions(+), 712 deletions(-) create mode 100644 nano/core_test/bootstrap.cpp diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 802b96ac26..38991e67b0 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable (core_test active_transactions.cpp block.cpp block_store.cpp + bootstrap.cpp conflicts.cpp difficulty.cpp distributed_work.cpp diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp new file mode 100644 index 0000000000..a011beda8b --- /dev/null +++ b/nano/core_test/bootstrap.cpp @@ -0,0 +1,715 @@ +#include +#include + +#include + +using namespace std::chrono_literals; + +// If the account doesn't exist, current == end so there's no iteration +TEST (bulk_pull, no_address) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::bulk_pull); + req->start = 1; + req->end = 2; + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (request->current, request->request->end); + ASSERT_TRUE (request->current.is_zero ()); +} + +TEST (bulk_pull, genesis_to_end) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::bulk_pull{}); + req->start = nano::test_genesis_key.pub; + req->end.clear (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (system.nodes[0]->latest (nano::test_genesis_key.pub), request->current); + ASSERT_EQ (request->request->end, request->request->end); +} + +// If we can't find the end block, send everything +TEST (bulk_pull, no_end) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::bulk_pull{}); + req->start = nano::test_genesis_key.pub; + req->end = 1; + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (system.nodes[0]->latest (nano::test_genesis_key.pub), request->current); + ASSERT_TRUE (request->request->end.is_zero ()); +} + +TEST (bulk_pull, end_not_owned) +{ + nano::system system (24000, 1); + nano::keypair key2; + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, 100)); + nano::block_hash latest (system.nodes[0]->latest (nano::test_genesis_key.pub)); + nano::open_block open (0, 1, 2, nano::keypair ().prv, 4, 5); + open.hashables.account = key2.pub; + open.hashables.representative = key2.pub; + open.hashables.source = latest; + open.signature = nano::sign_message (key2.prv, key2.pub, open.hash ()); + system.nodes[0]->work_generate_blocking (open); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (open).code); + auto connection (std::make_shared (nullptr, system.nodes[0])); + nano::genesis genesis; + std::unique_ptr req (new nano::bulk_pull{}); + req->start = key2.pub; + req->end = genesis.hash (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (request->current, request->request->end); +} + +TEST (bulk_pull, none) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + nano::genesis genesis; + std::unique_ptr req (new nano::bulk_pull{}); + req->start = nano::test_genesis_key.pub; + req->end = genesis.hash (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + auto block (request->get_next ()); + ASSERT_EQ (nullptr, block); +} + +TEST (bulk_pull, get_next_on_open) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::bulk_pull{}); + req->start = nano::test_genesis_key.pub; + req->end.clear (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + auto block (request->get_next ()); + ASSERT_NE (nullptr, block); + ASSERT_TRUE (block->previous ().is_zero ()); + ASSERT_FALSE (connection->requests.empty ()); + ASSERT_EQ (request->current, request->request->end); +} + +TEST (bulk_pull, by_block) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + nano::genesis genesis; + std::unique_ptr req (new nano::bulk_pull{}); + req->start = genesis.hash (); + req->end.clear (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + auto block (request->get_next ()); + ASSERT_NE (nullptr, block); + ASSERT_EQ (block->hash (), genesis.hash ()); + + block = request->get_next (); + ASSERT_EQ (nullptr, block); +} + +TEST (bulk_pull, by_block_single) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + nano::genesis genesis; + std::unique_ptr req (new nano::bulk_pull{}); + req->start = genesis.hash (); + req->end = genesis.hash (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + auto block (request->get_next ()); + ASSERT_NE (nullptr, block); + ASSERT_EQ (block->hash (), genesis.hash ()); + + block = request->get_next (); + ASSERT_EQ (nullptr, block); +} + +TEST (bulk_pull, count_limit) +{ + nano::system system (24000, 1); + nano::genesis genesis; + + auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), nano::test_genesis_key.pub, 1, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send1).code); + auto receive1 (std::make_shared (send1->hash (), send1->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*receive1).code); + + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::bulk_pull{}); + req->start = receive1->hash (); + req->set_count_present (true); + req->count = 2; + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + + ASSERT_EQ (request->max_count, 2); + ASSERT_EQ (request->sent_count, 0); + + auto block (request->get_next ()); + ASSERT_EQ (receive1->hash (), block->hash ()); + + block = request->get_next (); + ASSERT_EQ (send1->hash (), block->hash ()); + + block = request->get_next (); + ASSERT_EQ (nullptr, block); +} + +TEST (bootstrap_processor, DISABLED_process_none) +{ + nano::system system (24000, 1); + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + auto done (false); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + while (!done) + { + system.io_ctx.run_one (); + } + node1->stop (); +} + +// Bootstrap can pull one basic block +TEST (bootstrap_processor, process_one) +{ + nano::system system (24000, 1); + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, 100)); + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + nano::block_hash hash1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); + nano::block_hash hash2 (node1->latest (nano::test_genesis_key.pub)); + ASSERT_NE (hash1, hash2); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + ASSERT_NE (node1->latest (nano::test_genesis_key.pub), system.nodes[0]->latest (nano::test_genesis_key.pub)); + system.deadline_set (10s); + while (node1->latest (nano::test_genesis_key.pub) != system.nodes[0]->latest (nano::test_genesis_key.pub)) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (0, node1->active.size ()); + node1->stop (); +} + +TEST (bootstrap_processor, process_two) +{ + nano::system system (24000, 1); + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + nano::block_hash hash1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, 50)); + nano::block_hash hash2 (system.nodes[0]->latest (nano::test_genesis_key.pub)); + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, 50)); + nano::block_hash hash3 (system.nodes[0]->latest (nano::test_genesis_key.pub)); + ASSERT_NE (hash1, hash2); + ASSERT_NE (hash1, hash3); + ASSERT_NE (hash2, hash3); + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + ASSERT_NE (node1->latest (nano::test_genesis_key.pub), system.nodes[0]->latest (nano::test_genesis_key.pub)); + system.deadline_set (10s); + while (node1->latest (nano::test_genesis_key.pub) != system.nodes[0]->latest (nano::test_genesis_key.pub)) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +// Bootstrap can pull universal blocks +TEST (bootstrap_processor, process_state) +{ + nano::system system (24000, 1); + nano::genesis genesis; + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + auto node0 (system.nodes[0]); + auto block1 (std::make_shared (nano::test_genesis_key.pub, node0->latest (nano::test_genesis_key.pub), nano::test_genesis_key.pub, nano::genesis_amount - 100, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0)); + auto block2 (std::make_shared (nano::test_genesis_key.pub, block1->hash (), nano::test_genesis_key.pub, nano::genesis_amount, block1->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0)); + node0->work_generate_blocking (*block1); + node0->work_generate_blocking (*block2); + node0->process (*block1); + node0->process (*block2); + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_EQ (node0->latest (nano::test_genesis_key.pub), block2->hash ()); + ASSERT_NE (node1->latest (nano::test_genesis_key.pub), block2->hash ()); + node1->bootstrap_initiator.bootstrap (node0->network.endpoint ()); + ASSERT_NE (node1->latest (nano::test_genesis_key.pub), node0->latest (nano::test_genesis_key.pub)); + system.deadline_set (10s); + while (node1->latest (nano::test_genesis_key.pub) != node0->latest (nano::test_genesis_key.pub)) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (0, node1->active.size ()); + node1->stop (); +} + +TEST (bootstrap_processor, process_new) +{ + nano::system system (24000, 2); + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + nano::keypair key2; + system.wallet (1)->insert_adhoc (key2.prv); + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ())); + system.deadline_set (10s); + while (system.nodes[0]->balance (key2.pub).is_zero ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + nano::uint128_t balance1 (system.nodes[0]->balance (nano::test_genesis_key.pub)); + nano::uint128_t balance2 (system.nodes[0]->balance (key2.pub)); + auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + system.deadline_set (10s); + while (node1->balance (key2.pub) != balance2) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (balance1, node1->balance (nano::test_genesis_key.pub)); + node1->stop (); +} + +TEST (bootstrap_processor, pull_diamond) +{ + nano::system system (24000, 1); + nano::keypair key; + auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send1).code); + auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*open).code); + auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, system.work.generate (open->hash ()))); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send2).code); + auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*receive).code); + auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + system.deadline_set (10s); + while (node1->balance (nano::test_genesis_key.pub) != 100) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (100, node1->balance (nano::test_genesis_key.pub)); + node1->stop (); +} + +TEST (bootstrap_processor, push_diamond) +{ + nano::system system (24000, 1); + nano::keypair key; + auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + auto wallet1 (node1->wallets.create (100)); + wallet1->insert_adhoc (nano::test_genesis_key.prv); + wallet1->insert_adhoc (key.prv); + auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); + auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); + ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); + auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, system.work.generate (open->hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); + auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*receive).code); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + system.deadline_set (10s); + while (system.nodes[0]->balance (nano::test_genesis_key.pub) != 100) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (100, system.nodes[0]->balance (nano::test_genesis_key.pub)); + node1->stop (); +} + +TEST (bootstrap_processor, push_one) +{ + nano::system system (24000, 1); + nano::keypair key1; + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + auto wallet (node1->wallets.create (nano::uint256_union ())); + ASSERT_NE (nullptr, wallet); + wallet->insert_adhoc (nano::test_genesis_key.prv); + nano::uint128_t balance1 (node1->balance (nano::test_genesis_key.pub)); + ASSERT_NE (nullptr, wallet->send_action (nano::test_genesis_key.pub, key1.pub, 100)); + ASSERT_NE (balance1, node1->balance (nano::test_genesis_key.pub)); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + system.deadline_set (10s); + while (system.nodes[0]->balance (nano::test_genesis_key.pub) == balance1) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +TEST (bootstrap_processor, lazy_hash) +{ + nano::system system (24000, 1); + nano::genesis genesis; + nano::keypair key1; + nano::keypair key2; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); + auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); + auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); + auto receive2 (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (key2.pub))); + // Processing test chain + system.nodes[0]->block_processor.add (send1); + system.nodes[0]->block_processor.add (receive1); + system.nodes[0]->block_processor.add (send2); + system.nodes[0]->block_processor.add (receive2); + system.nodes[0]->block_processor.flush (); + // Start lazy bootstrap with last block in chain known + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); + node1->bootstrap_initiator.bootstrap_lazy (receive2->hash ()); + // Check processed blocks + system.deadline_set (10s); + while (node1->balance (key2.pub) == 0) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +TEST (bootstrap_processor, lazy_max_pull_count) +{ + nano::system system (24000, 1); + nano::genesis genesis; + nano::keypair key1; + nano::keypair key2; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); + auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); + auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); + auto receive2 (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (key2.pub))); + auto change1 (std::make_shared (key2.pub, receive2->hash (), key1.pub, nano::Gxrb_ratio, 0, key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (receive2->hash ()))); + auto change2 (std::make_shared (key2.pub, change1->hash (), nano::test_genesis_key.pub, nano::Gxrb_ratio, 0, key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (change1->hash ()))); + auto change3 (std::make_shared (key2.pub, change2->hash (), key2.pub, nano::Gxrb_ratio, 0, key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (change2->hash ()))); + // Processing test chain + system.nodes[0]->block_processor.add (send1); + system.nodes[0]->block_processor.add (receive1); + system.nodes[0]->block_processor.add (send2); + system.nodes[0]->block_processor.add (receive2); + system.nodes[0]->block_processor.add (change1); + system.nodes[0]->block_processor.add (change2); + system.nodes[0]->block_processor.add (change3); + system.nodes[0]->block_processor.flush (); + // Start lazy bootstrap with last block in chain known + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); + node1->bootstrap_initiator.bootstrap_lazy (change3->hash ()); + // Check processed blocks + system.deadline_set (10s); + while (node1->block (change3->hash ()) == nullptr) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +TEST (bootstrap_processor, wallet_lazy_frontier) +{ + nano::system system (24000, 1); + nano::genesis genesis; + nano::keypair key1; + nano::keypair key2; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); + auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); + auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); + auto receive2 (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (key2.pub))); + // Processing test chain + system.nodes[0]->block_processor.add (send1); + system.nodes[0]->block_processor.add (receive1); + system.nodes[0]->block_processor.add (send2); + system.nodes[0]->block_processor.add (receive2); + system.nodes[0]->block_processor.flush (); + // Start wallet lazy bootstrap + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); + auto wallet (node1->wallets.create (nano::uint256_union ())); + ASSERT_NE (nullptr, wallet); + wallet->insert_adhoc (key2.prv); + node1->bootstrap_wallet (); + // Check processed blocks + system.deadline_set (10s); + while (!node1->ledger.block_exists (receive2->hash ())) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +TEST (bootstrap_processor, wallet_lazy_pending) +{ + nano::system system (24000, 1); + nano::genesis genesis; + nano::keypair key1; + nano::keypair key2; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); + auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); + auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); + // Processing test chain + system.nodes[0]->block_processor.add (send1); + system.nodes[0]->block_processor.add (receive1); + system.nodes[0]->block_processor.add (send2); + system.nodes[0]->block_processor.flush (); + // Start wallet lazy bootstrap + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); + auto wallet (node1->wallets.create (nano::uint256_union ())); + ASSERT_NE (nullptr, wallet); + wallet->insert_adhoc (key2.prv); + node1->bootstrap_wallet (); + // Check processed blocks + system.deadline_set (10s); + while (!node1->ledger.block_exists (send2->hash ())) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +TEST (frontier_req_response, DISABLED_destruction) +{ + { + std::shared_ptr hold; // Destructing tcp acceptor on non-existent io_context + { + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::frontier_req); + req->start.clear (); + req->age = std::numeric_limitsage)>::max (); + req->count = std::numeric_limitscount)>::max (); + connection->requests.push (std::unique_ptr{}); + hold = std::make_shared (connection, std::move (req)); + } + } + ASSERT_TRUE (true); +} + +TEST (frontier_req, begin) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::frontier_req); + req->start.clear (); + req->age = std::numeric_limitsage)>::max (); + req->count = std::numeric_limitscount)>::max (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (nano::test_genesis_key.pub, request->current); + nano::genesis genesis; + ASSERT_EQ (genesis.hash (), request->frontier); +} + +TEST (frontier_req, end) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::frontier_req); + req->start = nano::test_genesis_key.pub.number () + 1; + req->age = std::numeric_limitsage)>::max (); + req->count = std::numeric_limitscount)>::max (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_TRUE (request->current.is_zero ()); +} + +TEST (frontier_req, count) +{ + nano::system system (24000, 1); + auto & node1 (*system.nodes[0]); + nano::genesis genesis; + // Public key FB93... after genesis in accounts table + nano::keypair key1 ("ED5AE0A6505B14B67435C29FD9FEEBC26F597D147BC92F6D795FFAD7AFD3D967"); + nano::state_block send1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); + node1.work_generate_blocking (send1); + ASSERT_EQ (nano::process_result::progress, node1.process (send1).code); + nano::state_block receive1 (key1.pub, 0, nano::test_genesis_key.pub, nano::Gxrb_ratio, send1.hash (), key1.prv, key1.pub, 0); + node1.work_generate_blocking (receive1); + ASSERT_EQ (nano::process_result::progress, node1.process (receive1).code); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::frontier_req); + req->start.clear (); + req->age = std::numeric_limitsage)>::max (); + req->count = 1; + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (nano::test_genesis_key.pub, request->current); + ASSERT_EQ (send1.hash (), request->frontier); +} + +TEST (frontier_req, time_bound) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::frontier_req); + req->start.clear (); + req->age = 1; + req->count = std::numeric_limitscount)>::max (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (nano::test_genesis_key.pub, request->current); + // Wait 2 seconds until age of account will be > 1 seconds + std::this_thread::sleep_for (std::chrono::milliseconds (2100)); + std::unique_ptr req2 (new nano::frontier_req); + req2->start.clear (); + req2->age = 1; + req2->count = std::numeric_limitscount)>::max (); + auto connection2 (std::make_shared (nullptr, system.nodes[0])); + connection2->requests.push (std::unique_ptr{}); + auto request2 (std::make_shared (connection, std::move (req2))); + ASSERT_TRUE (request2->current.is_zero ()); +} + +TEST (frontier_req, time_cutoff) +{ + nano::system system (24000, 1); + auto connection (std::make_shared (nullptr, system.nodes[0])); + std::unique_ptr req (new nano::frontier_req); + req->start.clear (); + req->age = 3; + req->count = std::numeric_limitscount)>::max (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_EQ (nano::test_genesis_key.pub, request->current); + nano::genesis genesis; + ASSERT_EQ (genesis.hash (), request->frontier); + // Wait 4 seconds until age of account will be > 3 seconds + std::this_thread::sleep_for (std::chrono::milliseconds (4100)); + std::unique_ptr req2 (new nano::frontier_req); + req2->start.clear (); + req2->age = 3; + req2->count = std::numeric_limitscount)>::max (); + auto connection2 (std::make_shared (nullptr, system.nodes[0])); + connection2->requests.push (std::unique_ptr{}); + auto request2 (std::make_shared (connection, std::move (req2))); + ASSERT_TRUE (request2->frontier.is_zero ()); +} + +TEST (bulk, genesis) +{ + nano::system system (24000, 1); + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + nano::block_hash latest1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); + nano::block_hash latest2 (node1->latest (nano::test_genesis_key.pub)); + ASSERT_EQ (latest1, latest2); + nano::keypair key2; + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, 100)); + nano::block_hash latest3 (system.nodes[0]->latest (nano::test_genesis_key.pub)); + ASSERT_NE (latest1, latest3); + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + system.deadline_set (10s); + while (node1->latest (nano::test_genesis_key.pub) != system.nodes[0]->latest (nano::test_genesis_key.pub)) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (node1->latest (nano::test_genesis_key.pub), system.nodes[0]->latest (nano::test_genesis_key.pub)); + node1->stop (); +} + +TEST (bulk, offline_send) +{ + nano::system system (24000, 1); + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); + ASSERT_FALSE (node1->init_error ()); + node1->start (); + system.nodes.push_back (node1); + nano::keypair key2; + auto wallet (node1->wallets.create (nano::uint256_union ())); + wallet->insert_adhoc (key2.prv); + ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ())); + ASSERT_NE (std::numeric_limits::max (), system.nodes[0]->balance (nano::test_genesis_key.pub)); + // Wait to finish election background tasks + system.deadline_set (10s); + while (!system.nodes[0]->active.empty ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + // Initiate bootstrap + node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); + // Nodes should find each other + do + { + ASSERT_NO_ERROR (system.poll ()); + } while (system.nodes[0]->network.empty () || node1->network.empty ()); + // Send block arrival via bootstrap + while (node1->balance (nano::test_genesis_key.pub) == std::numeric_limits::max ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + // Receiving send block + system.deadline_set (20s); + while (node1->balance (key2.pub) != system.nodes[0]->config.receive_minimum.number ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + node1->stop (); +} + +TEST (bulk_pull_account, basics) +{ + nano::system system (24000, 1); + system.nodes[0]->config.receive_minimum = nano::uint128_union (20); + nano::keypair key1; + system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); + system.wallet (0)->insert_adhoc (key1.prv); + auto send1 (system.wallet (0)->send_action (nano::genesis_account, key1.pub, 25)); + auto send2 (system.wallet (0)->send_action (nano::genesis_account, key1.pub, 10)); + auto send3 (system.wallet (0)->send_action (nano::genesis_account, key1.pub, 2)); + system.deadline_set (5s); + while (system.nodes[0]->balance (key1.pub) != 25) + { + ASSERT_NO_ERROR (system.poll ()); + } + auto connection (std::make_shared (nullptr, system.nodes[0])); + + { + std::unique_ptr req (new nano::bulk_pull_account{}); + req->account = key1.pub; + req->minimum_amount = 5; + req->flags = nano::bulk_pull_account_flags (); + connection->requests.push (std::unique_ptr{}); + auto request (std::make_shared (connection, std::move (req))); + ASSERT_FALSE (request->invalid_request); + ASSERT_FALSE (request->pending_include_address); + ASSERT_FALSE (request->pending_address_only); + ASSERT_EQ (request->current_key.account, key1.pub); + ASSERT_EQ (request->current_key.hash, 0); + auto block_data (request->get_next ()); + ASSERT_EQ (send2->hash (), block_data.first.get ()->hash); + ASSERT_EQ (nano::uint128_union (10), block_data.second.get ()->amount); + ASSERT_EQ (nano::genesis_account, block_data.second.get ()->source); + ASSERT_EQ (nullptr, request->get_next ().first.get ()); + } + + { + std::unique_ptr req (new nano::bulk_pull_account{}); + req->account = key1.pub; + req->minimum_amount = 0; + req->flags = nano::bulk_pull_account_flags::pending_address_only; + auto request (std::make_shared (connection, std::move (req))); + ASSERT_TRUE (request->pending_address_only); + auto block_data (request->get_next ()); + ASSERT_NE (nullptr, block_data.first.get ()); + ASSERT_NE (nullptr, block_data.second.get ()); + ASSERT_EQ (nano::genesis_account, block_data.second.get ()->source); + block_data = request->get_next (); + ASSERT_EQ (nullptr, block_data.first.get ()); + ASSERT_EQ (nullptr, block_data.second.get ()); + } +} diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index fa6e87d69e..ed9e523511 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -460,662 +460,6 @@ TEST (parse_endpoint, no_colon) ASSERT_TRUE (nano::parse_endpoint (string, endpoint)); } -// If the account doesn't exist, current == end so there's no iteration -TEST (bulk_pull, no_address) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::bulk_pull); - req->start = 1; - req->end = 2; - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (request->current, request->request->end); - ASSERT_TRUE (request->current.is_zero ()); -} - -TEST (bulk_pull, genesis_to_end) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::bulk_pull{}); - req->start = nano::test_genesis_key.pub; - req->end.clear (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (system.nodes[0]->latest (nano::test_genesis_key.pub), request->current); - ASSERT_EQ (request->request->end, request->request->end); -} - -// If we can't find the end block, send everything -TEST (bulk_pull, no_end) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::bulk_pull{}); - req->start = nano::test_genesis_key.pub; - req->end = 1; - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (system.nodes[0]->latest (nano::test_genesis_key.pub), request->current); - ASSERT_TRUE (request->request->end.is_zero ()); -} - -TEST (bulk_pull, end_not_owned) -{ - nano::system system (24000, 1); - nano::keypair key2; - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, 100)); - nano::block_hash latest (system.nodes[0]->latest (nano::test_genesis_key.pub)); - nano::open_block open (0, 1, 2, nano::keypair ().prv, 4, 5); - open.hashables.account = key2.pub; - open.hashables.representative = key2.pub; - open.hashables.source = latest; - open.signature = nano::sign_message (key2.prv, key2.pub, open.hash ()); - system.nodes[0]->work_generate_blocking (open); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (open).code); - auto connection (std::make_shared (nullptr, system.nodes[0])); - nano::genesis genesis; - std::unique_ptr req (new nano::bulk_pull{}); - req->start = key2.pub; - req->end = genesis.hash (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (request->current, request->request->end); -} - -TEST (bulk_pull, none) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - nano::genesis genesis; - std::unique_ptr req (new nano::bulk_pull{}); - req->start = nano::test_genesis_key.pub; - req->end = genesis.hash (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_EQ (nullptr, block); -} - -TEST (bulk_pull, get_next_on_open) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::bulk_pull{}); - req->start = nano::test_genesis_key.pub; - req->end.clear (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_TRUE (block->previous ().is_zero ()); - ASSERT_FALSE (connection->requests.empty ()); - ASSERT_EQ (request->current, request->request->end); -} - -TEST (bulk_pull, by_block) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - nano::genesis genesis; - std::unique_ptr req (new nano::bulk_pull{}); - req->start = genesis.hash (); - req->end.clear (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_EQ (block->hash (), genesis.hash ()); - - block = request->get_next (); - ASSERT_EQ (nullptr, block); -} - -TEST (bulk_pull, by_block_single) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - nano::genesis genesis; - std::unique_ptr req (new nano::bulk_pull{}); - req->start = genesis.hash (); - req->end = genesis.hash (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - auto block (request->get_next ()); - ASSERT_NE (nullptr, block); - ASSERT_EQ (block->hash (), genesis.hash ()); - - block = request->get_next (); - ASSERT_EQ (nullptr, block); -} - -TEST (bulk_pull, count_limit) -{ - nano::system system (24000, 1); - nano::genesis genesis; - - auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), nano::test_genesis_key.pub, 1, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send1).code); - auto receive1 (std::make_shared (send1->hash (), send1->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*receive1).code); - - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::bulk_pull{}); - req->start = receive1->hash (); - req->set_count_present (true); - req->count = 2; - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - - ASSERT_EQ (request->max_count, 2); - ASSERT_EQ (request->sent_count, 0); - - auto block (request->get_next ()); - ASSERT_EQ (receive1->hash (), block->hash ()); - - block = request->get_next (); - ASSERT_EQ (send1->hash (), block->hash ()); - - block = request->get_next (); - ASSERT_EQ (nullptr, block); -} - -TEST (bootstrap_processor, DISABLED_process_none) -{ - nano::system system (24000, 1); - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - auto done (false); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - while (!done) - { - system.io_ctx.run_one (); - } - node1->stop (); -} - -// Bootstrap can pull one basic block -TEST (bootstrap_processor, process_one) -{ - nano::system system (24000, 1); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, 100)); - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - nano::block_hash hash1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - nano::block_hash hash2 (node1->latest (nano::test_genesis_key.pub)); - ASSERT_NE (hash1, hash2); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - ASSERT_NE (node1->latest (nano::test_genesis_key.pub), system.nodes[0]->latest (nano::test_genesis_key.pub)); - system.deadline_set (10s); - while (node1->latest (nano::test_genesis_key.pub) != system.nodes[0]->latest (nano::test_genesis_key.pub)) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (0, node1->active.size ()); - node1->stop (); -} - -TEST (bootstrap_processor, process_two) -{ - nano::system system (24000, 1); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - nano::block_hash hash1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, 50)); - nano::block_hash hash2 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, 50)); - nano::block_hash hash3 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - ASSERT_NE (hash1, hash2); - ASSERT_NE (hash1, hash3); - ASSERT_NE (hash2, hash3); - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - ASSERT_NE (node1->latest (nano::test_genesis_key.pub), system.nodes[0]->latest (nano::test_genesis_key.pub)); - system.deadline_set (10s); - while (node1->latest (nano::test_genesis_key.pub) != system.nodes[0]->latest (nano::test_genesis_key.pub)) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - -// Bootstrap can pull universal blocks -TEST (bootstrap_processor, process_state) -{ - nano::system system (24000, 1); - nano::genesis genesis; - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - auto node0 (system.nodes[0]); - auto block1 (std::make_shared (nano::test_genesis_key.pub, node0->latest (nano::test_genesis_key.pub), nano::test_genesis_key.pub, nano::genesis_amount - 100, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0)); - auto block2 (std::make_shared (nano::test_genesis_key.pub, block1->hash (), nano::test_genesis_key.pub, nano::genesis_amount, block1->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0)); - node0->work_generate_blocking (*block1); - node0->work_generate_blocking (*block2); - node0->process (*block1); - node0->process (*block2); - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_EQ (node0->latest (nano::test_genesis_key.pub), block2->hash ()); - ASSERT_NE (node1->latest (nano::test_genesis_key.pub), block2->hash ()); - node1->bootstrap_initiator.bootstrap (node0->network.endpoint ()); - ASSERT_NE (node1->latest (nano::test_genesis_key.pub), node0->latest (nano::test_genesis_key.pub)); - system.deadline_set (10s); - while (node1->latest (nano::test_genesis_key.pub) != node0->latest (nano::test_genesis_key.pub)) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (0, node1->active.size ()); - node1->stop (); -} - -TEST (bootstrap_processor, process_new) -{ - nano::system system (24000, 2); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - nano::keypair key2; - system.wallet (1)->insert_adhoc (key2.prv); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ())); - system.deadline_set (10s); - while (system.nodes[0]->balance (key2.pub).is_zero ()) - { - ASSERT_NO_ERROR (system.poll ()); - } - nano::uint128_t balance1 (system.nodes[0]->balance (nano::test_genesis_key.pub)); - nano::uint128_t balance2 (system.nodes[0]->balance (key2.pub)); - auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - system.deadline_set (10s); - while (node1->balance (key2.pub) != balance2) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (balance1, node1->balance (nano::test_genesis_key.pub)); - node1->stop (); -} - -TEST (bootstrap_processor, pull_diamond) -{ - nano::system system (24000, 1); - nano::keypair key; - auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send1).code); - auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*open).code); - auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, system.work.generate (open->hash ()))); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send2).code); - auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); - ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*receive).code); - auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - system.deadline_set (10s); - while (node1->balance (nano::test_genesis_key.pub) != 100) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (100, node1->balance (nano::test_genesis_key.pub)); - node1->stop (); -} - -TEST (bootstrap_processor, push_diamond) -{ - nano::system system (24000, 1); - nano::keypair key; - auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - auto wallet1 (node1->wallets.create (100)); - wallet1->insert_adhoc (nano::test_genesis_key.prv); - wallet1->insert_adhoc (key.prv); - auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); - ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); - auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); - ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); - auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, system.work.generate (open->hash ()))); - ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); - auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); - ASSERT_EQ (nano::process_result::progress, node1->process (*receive).code); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - system.deadline_set (10s); - while (system.nodes[0]->balance (nano::test_genesis_key.pub) != 100) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (100, system.nodes[0]->balance (nano::test_genesis_key.pub)); - node1->stop (); -} - -TEST (bootstrap_processor, push_one) -{ - nano::system system (24000, 1); - nano::keypair key1; - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - auto wallet (node1->wallets.create (nano::uint256_union ())); - ASSERT_NE (nullptr, wallet); - wallet->insert_adhoc (nano::test_genesis_key.prv); - nano::uint128_t balance1 (node1->balance (nano::test_genesis_key.pub)); - ASSERT_NE (nullptr, wallet->send_action (nano::test_genesis_key.pub, key1.pub, 100)); - ASSERT_NE (balance1, node1->balance (nano::test_genesis_key.pub)); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - system.deadline_set (10s); - while (system.nodes[0]->balance (nano::test_genesis_key.pub) == balance1) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - -TEST (bootstrap_processor, lazy_hash) -{ - nano::system system (24000, 1); - nano::genesis genesis; - nano::keypair key1; - nano::keypair key2; - // Generating test chain - auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); - auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); - auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); - auto receive2 (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (key2.pub))); - // Processing test chain - system.nodes[0]->block_processor.add (send1); - system.nodes[0]->block_processor.add (receive1); - system.nodes[0]->block_processor.add (send2); - system.nodes[0]->block_processor.add (receive2); - system.nodes[0]->block_processor.flush (); - // Start lazy bootstrap with last block in chain known - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); - node1->bootstrap_initiator.bootstrap_lazy (receive2->hash ()); - // Check processed blocks - system.deadline_set (10s); - while (node1->balance (key2.pub) == 0) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - -TEST (bootstrap_processor, lazy_max_pull_count) -{ - nano::system system (24000, 1); - nano::genesis genesis; - nano::keypair key1; - nano::keypair key2; - // Generating test chain - auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); - auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); - auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); - auto receive2 (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (key2.pub))); - auto change1 (std::make_shared (key2.pub, receive2->hash (), key1.pub, nano::Gxrb_ratio, 0, key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (receive2->hash ()))); - auto change2 (std::make_shared (key2.pub, change1->hash (), nano::test_genesis_key.pub, nano::Gxrb_ratio, 0, key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (change1->hash ()))); - auto change3 (std::make_shared (key2.pub, change2->hash (), key2.pub, nano::Gxrb_ratio, 0, key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (change2->hash ()))); - // Processing test chain - system.nodes[0]->block_processor.add (send1); - system.nodes[0]->block_processor.add (receive1); - system.nodes[0]->block_processor.add (send2); - system.nodes[0]->block_processor.add (receive2); - system.nodes[0]->block_processor.add (change1); - system.nodes[0]->block_processor.add (change2); - system.nodes[0]->block_processor.add (change3); - system.nodes[0]->block_processor.flush (); - // Start lazy bootstrap with last block in chain known - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); - node1->bootstrap_initiator.bootstrap_lazy (change3->hash ()); - // Check processed blocks - system.deadline_set (10s); - while (node1->block (change3->hash ()) == nullptr) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - -TEST (bootstrap_processor, wallet_lazy_frontier) -{ - nano::system system (24000, 1); - nano::genesis genesis; - nano::keypair key1; - nano::keypair key2; - // Generating test chain - auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); - auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); - auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); - auto receive2 (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.nodes[0]->work_generate_blocking (key2.pub))); - // Processing test chain - system.nodes[0]->block_processor.add (send1); - system.nodes[0]->block_processor.add (receive1); - system.nodes[0]->block_processor.add (send2); - system.nodes[0]->block_processor.add (receive2); - system.nodes[0]->block_processor.flush (); - // Start wallet lazy bootstrap - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); - auto wallet (node1->wallets.create (nano::uint256_union ())); - ASSERT_NE (nullptr, wallet); - wallet->insert_adhoc (key2.prv); - node1->bootstrap_wallet (); - // Check processed blocks - system.deadline_set (10s); - while (!node1->ledger.block_exists (receive2->hash ())) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - -TEST (bootstrap_processor, wallet_lazy_pending) -{ - nano::system system (24000, 1); - nano::genesis genesis; - nano::keypair key1; - nano::keypair key2; - // Generating test chain - auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.nodes[0]->work_generate_blocking (genesis.hash ()))); - auto receive1 (std::make_shared (key1.pub, 0, key1.pub, nano::Gxrb_ratio, send1->hash (), key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (key1.pub))); - auto send2 (std::make_shared (key1.pub, receive1->hash (), key1.pub, 0, key2.pub, key1.prv, key1.pub, *system.nodes[0]->work_generate_blocking (receive1->hash ()))); - // Processing test chain - system.nodes[0]->block_processor.add (send1); - system.nodes[0]->block_processor.add (receive1); - system.nodes[0]->block_processor.add (send2); - system.nodes[0]->block_processor.flush (); - // Start wallet lazy bootstrap - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); - auto wallet (node1->wallets.create (nano::uint256_union ())); - ASSERT_NE (nullptr, wallet); - wallet->insert_adhoc (key2.prv); - node1->bootstrap_wallet (); - // Check processed blocks - system.deadline_set (10s); - while (!node1->ledger.block_exists (send2->hash ())) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - -TEST (frontier_req_response, DISABLED_destruction) -{ - { - std::shared_ptr hold; // Destructing tcp acceptor on non-existent io_context - { - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::frontier_req); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - connection->requests.push (std::unique_ptr{}); - hold = std::make_shared (connection, std::move (req)); - } - } - ASSERT_TRUE (true); -} - -TEST (frontier_req, begin) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::frontier_req); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::test_genesis_key.pub, request->current); - nano::genesis genesis; - ASSERT_EQ (genesis.hash (), request->frontier); -} - -TEST (frontier_req, end) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::frontier_req); - req->start = nano::test_genesis_key.pub.number () + 1; - req->age = std::numeric_limitsage)>::max (); - req->count = std::numeric_limitscount)>::max (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_TRUE (request->current.is_zero ()); -} - -TEST (frontier_req, count) -{ - nano::system system (24000, 1); - auto & node1 (*system.nodes[0]); - nano::genesis genesis; - // Public key FB93... after genesis in accounts table - nano::keypair key1 ("ED5AE0A6505B14B67435C29FD9FEEBC26F597D147BC92F6D795FFAD7AFD3D967"); - nano::state_block send1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0); - node1.work_generate_blocking (send1); - ASSERT_EQ (nano::process_result::progress, node1.process (send1).code); - nano::state_block receive1 (key1.pub, 0, nano::test_genesis_key.pub, nano::Gxrb_ratio, send1.hash (), key1.prv, key1.pub, 0); - node1.work_generate_blocking (receive1); - ASSERT_EQ (nano::process_result::progress, node1.process (receive1).code); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::frontier_req); - req->start.clear (); - req->age = std::numeric_limitsage)>::max (); - req->count = 1; - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::test_genesis_key.pub, request->current); - ASSERT_EQ (send1.hash (), request->frontier); -} - -TEST (frontier_req, time_bound) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::frontier_req); - req->start.clear (); - req->age = 1; - req->count = std::numeric_limitscount)>::max (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::test_genesis_key.pub, request->current); - // Wait 2 seconds until age of account will be > 1 seconds - std::this_thread::sleep_for (std::chrono::milliseconds (2100)); - std::unique_ptr req2 (new nano::frontier_req); - req2->start.clear (); - req2->age = 1; - req2->count = std::numeric_limitscount)>::max (); - auto connection2 (std::make_shared (nullptr, system.nodes[0])); - connection2->requests.push (std::unique_ptr{}); - auto request2 (std::make_shared (connection, std::move (req2))); - ASSERT_TRUE (request2->current.is_zero ()); -} - -TEST (frontier_req, time_cutoff) -{ - nano::system system (24000, 1); - auto connection (std::make_shared (nullptr, system.nodes[0])); - std::unique_ptr req (new nano::frontier_req); - req->start.clear (); - req->age = 3; - req->count = std::numeric_limitscount)>::max (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_EQ (nano::test_genesis_key.pub, request->current); - nano::genesis genesis; - ASSERT_EQ (genesis.hash (), request->frontier); - // Wait 4 seconds until age of account will be > 3 seconds - std::this_thread::sleep_for (std::chrono::milliseconds (4100)); - std::unique_ptr req2 (new nano::frontier_req); - req2->start.clear (); - req2->age = 3; - req2->count = std::numeric_limitscount)>::max (); - auto connection2 (std::make_shared (nullptr, system.nodes[0])); - connection2->requests.push (std::unique_ptr{}); - auto request2 (std::make_shared (connection, std::move (req2))); - ASSERT_TRUE (request2->frontier.is_zero ()); -} - -TEST (bulk, genesis) -{ - nano::system system (24000, 1); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - nano::block_hash latest1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - nano::block_hash latest2 (node1->latest (nano::test_genesis_key.pub)); - ASSERT_EQ (latest1, latest2); - nano::keypair key2; - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, 100)); - nano::block_hash latest3 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - ASSERT_NE (latest1, latest3); - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - system.deadline_set (10s); - while (node1->latest (nano::test_genesis_key.pub) != system.nodes[0]->latest (nano::test_genesis_key.pub)) - { - ASSERT_NO_ERROR (system.poll ()); - } - ASSERT_EQ (node1->latest (nano::test_genesis_key.pub), system.nodes[0]->latest (nano::test_genesis_key.pub)); - node1->stop (); -} - -TEST (bulk, offline_send) -{ - nano::system system (24000, 1); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - ASSERT_FALSE (node1->init_error ()); - node1->start (); - system.nodes.push_back (node1); - nano::keypair key2; - auto wallet (node1->wallets.create (nano::uint256_union ())); - wallet->insert_adhoc (key2.prv); - ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ())); - ASSERT_NE (std::numeric_limits::max (), system.nodes[0]->balance (nano::test_genesis_key.pub)); - // Wait to finish election background tasks - system.deadline_set (10s); - while (!system.nodes[0]->active.empty ()) - { - ASSERT_NO_ERROR (system.poll ()); - } - // Initiate bootstrap - node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); - // Nodes should find each other - do - { - ASSERT_NO_ERROR (system.poll ()); - } while (system.nodes[0]->network.empty () || node1->network.empty ()); - // Send block arrival via bootstrap - while (node1->balance (nano::test_genesis_key.pub) == std::numeric_limits::max ()) - { - ASSERT_NO_ERROR (system.poll ()); - } - // Receiving send block - system.deadline_set (20s); - while (node1->balance (key2.pub) != system.nodes[0]->config.receive_minimum.number ()) - { - ASSERT_NO_ERROR (system.poll ()); - } - node1->stop (); -} - TEST (network, ipv6) { boost::asio::ip::address_v6 address (boost::asio::ip::address_v6::from_string ("::ffff:127.0.0.1")); @@ -1380,60 +724,7 @@ TEST (message_buffer_manager, stats) ASSERT_EQ (1, stats.count (nano::stat::type::udp, nano::stat::detail::overflow)); } -TEST (bulk_pull_account, basics) -{ - nano::system system (24000, 1); - system.nodes[0]->config.receive_minimum = nano::uint128_union (20); - nano::keypair key1; - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - system.wallet (0)->insert_adhoc (key1.prv); - auto send1 (system.wallet (0)->send_action (nano::genesis_account, key1.pub, 25)); - auto send2 (system.wallet (0)->send_action (nano::genesis_account, key1.pub, 10)); - auto send3 (system.wallet (0)->send_action (nano::genesis_account, key1.pub, 2)); - system.deadline_set (5s); - while (system.nodes[0]->balance (key1.pub) != 25) - { - ASSERT_NO_ERROR (system.poll ()); - } - auto connection (std::make_shared (nullptr, system.nodes[0])); - - { - std::unique_ptr req (new nano::bulk_pull_account{}); - req->account = key1.pub; - req->minimum_amount = 5; - req->flags = nano::bulk_pull_account_flags (); - connection->requests.push (std::unique_ptr{}); - auto request (std::make_shared (connection, std::move (req))); - ASSERT_FALSE (request->invalid_request); - ASSERT_FALSE (request->pending_include_address); - ASSERT_FALSE (request->pending_address_only); - ASSERT_EQ (request->current_key.account, key1.pub); - ASSERT_EQ (request->current_key.hash, 0); - auto block_data (request->get_next ()); - ASSERT_EQ (send2->hash (), block_data.first.get ()->hash); - ASSERT_EQ (nano::uint128_union (10), block_data.second.get ()->amount); - ASSERT_EQ (nano::genesis_account, block_data.second.get ()->source); - ASSERT_EQ (nullptr, request->get_next ().first.get ()); - } - - { - std::unique_ptr req (new nano::bulk_pull_account{}); - req->account = key1.pub; - req->minimum_amount = 0; - req->flags = nano::bulk_pull_account_flags::pending_address_only; - auto request (std::make_shared (connection, std::move (req))); - ASSERT_TRUE (request->pending_address_only); - auto block_data (request->get_next ()); - ASSERT_NE (nullptr, block_data.first.get ()); - ASSERT_NE (nullptr, block_data.second.get ()); - ASSERT_EQ (nano::genesis_account, block_data.second.get ()->source); - block_data = request->get_next (); - ASSERT_EQ (nullptr, block_data.first.get ()); - ASSERT_EQ (nullptr, block_data.second.get ()); - } -} - -TEST (bootstrap, tcp_node_id_handshake) +TEST (tcp_listener, tcp_node_id_handshake) { nano::system system (24000, 1); auto socket (std::make_shared (system.nodes[0])); @@ -2246,7 +1537,7 @@ TEST (confirmation_height, pending_observer_callbacks) } } -TEST (bootstrap, tcp_listener_timeout_empty) +TEST (tcp_listener, tcp_listener_timeout_empty) { nano::system system (24000, 1); auto node0 (system.nodes[0]); @@ -2273,7 +1564,7 @@ TEST (bootstrap, tcp_listener_timeout_empty) } } -TEST (bootstrap, tcp_listener_timeout_node_id_handshake) +TEST (tcp_listener, tcp_listener_timeout_node_id_handshake) { nano::system system (24000, 1); auto node0 (system.nodes[0]); From ad678ab456d5ba944412674280846766e3583455 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Mon, 23 Sep 2019 14:24:15 +0300 Subject: [PATCH 02/22] Support unclear state blocks links pulls --- nano/core_test/bootstrap.cpp | 34 +++++++++++++ nano/node/bootstrap/bootstrap.cpp | 56 ++++++++++++--------- nano/node/bootstrap/bootstrap.hpp | 20 +++++--- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 10 ++-- nano/node/bootstrap/bootstrap_bulk_pull.hpp | 3 +- 5 files changed, 89 insertions(+), 34 deletions(-) diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index a011beda8b..984ac51465 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -416,6 +416,40 @@ TEST (bootstrap_processor, lazy_max_pull_count) node1->stop (); } +TEST (bootstrap_processor, lazy_unclear_state_link) +{ + nano::system system; + nano::node_flags node_flags; + node_flags.disable_legacy_bootstrap = true; + auto node1 = system.add_node (nano::node_config (24000, system.logging), node_flags); + nano::genesis genesis; + nano::keypair key; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (genesis.hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); + auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); + auto open (std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, system.work.generate (key.pub))); + ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); + auto receive (std::make_shared (key.pub, open->hash (), key.pub, 2 * nano::Gxrb_ratio, send2->hash (), key.prv, key.pub, system.work.generate (open->hash ()))); // It is not possible to define this block send/receive status based on previous block (legacy open) + ASSERT_EQ (nano::process_result::progress, node1->process (*receive).code); + // Start lazy bootstrap with last block in chain known + auto node2 = system.add_node (nano::node_config (24001, system.logging), node_flags); + node2->network.udp_channels.insert (node1->network.endpoint (), node1->network_params.protocol.protocol_version); + node2->bootstrap_initiator.bootstrap_lazy (receive->hash ()); + // Check processed blocks + system.deadline_set (10s); + while (node2->bootstrap_initiator.in_progress ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + node2->block_processor.flush (); + ASSERT_TRUE (node2->ledger.block_exists (send1->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (send2->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (open->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (receive->hash ())); +} + TEST (bootstrap_processor, wallet_lazy_frontier) { nano::system system (24000, 1); diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 4c6e56ac20..d41034240b 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -14,6 +14,7 @@ constexpr double nano::bootstrap_limits::bootstrap_connection_scale_target_blocks; constexpr double nano::bootstrap_limits::bootstrap_minimum_blocks_per_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_frontier_retry_limit; +constexpr unsigned nano::bootstrap_limits::bootstrap_lazy_retry_limit; constexpr double nano::bootstrap_limits::bootstrap_minimum_termination_time_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_max_new_connections; constexpr std::chrono::seconds nano::bootstrap_limits::lazy_flush_delay_sec; @@ -525,20 +526,20 @@ void nano::bootstrap_attempt::add_pull (nano::pull_info const & pull_a) void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) { auto pull (pull_a); - if (++pull.attempts < (nano::bootstrap_limits::bootstrap_frontier_retry_limit + (pull.processed / 10000))) + ++pull.attempts; + if (pull.attempts < (nano::bootstrap_limits::bootstrap_frontier_retry_limit + (pull.processed / 10000))) { nano::lock_guard lock (mutex); pulls.push_front (pull); condition.notify_all (); } - else if (mode == nano::bootstrap_mode::lazy) + else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= nano::bootstrap_limits::bootstrap_lazy_retry_limit)) { assert (pull.account == pull.head); if (!lazy_processed_or_exists (pull.account)) { // Retry for lazy pulls (not weak state block link assumptions) nano::lock_guard lock (mutex); - pull.attempts++; pulls.push_back (pull); condition.notify_all (); } @@ -569,17 +570,17 @@ void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) if (lazy_keys.size () < max_keys && lazy_keys.find (hash_a) == lazy_keys.end () && lazy_blocks.find (hash_a) == lazy_blocks.end ()) { lazy_keys.insert (hash_a); - lazy_pulls.push_back (hash_a); + lazy_pulls.push_back (std::make_pair (hash_a, true)); } } -void nano::bootstrap_attempt::lazy_add (nano::block_hash const & hash_a) +void nano::bootstrap_attempt::lazy_add (nano::block_hash const & hash_a, bool confirmed_head) { // Add only unknown blocks assert (!lazy_mutex.try_lock ()); if (lazy_blocks.find (hash_a) == lazy_blocks.end ()) { - lazy_pulls.push_back (hash_a); + lazy_pulls.push_back (std::make_pair (hash_a, confirmed_head)); } } @@ -605,10 +606,19 @@ void nano::bootstrap_attempt::lazy_pull_flush () for (auto & pull_start : lazy_pulls) { // Recheck if block was already processed - if (lazy_blocks.find (pull_start) == lazy_blocks.end () && !node->store.block_exists (transaction, pull_start)) + if (lazy_blocks.find (pull_start.first) == lazy_blocks.end () && !node->store.block_exists (transaction, pull_start.first)) { assert (node->network_params.bootstrap.lazy_max_pull_blocks <= std::numeric_limits::max ()); - pulls.push_back (nano::pull_info (pull_start, pull_start, nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks))); + if (pull_start.second) + { + // Confirmed head block + pulls.push_back (nano::pull_info (pull_start.first, pull_start.first, nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second)); + } + else + { + // Head is not confirmed. It can be account or hash or non-existing + pulls.push_back (nano::pull_info (pull_start.first, nano::block_hash (0), nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second)); + } } } lazy_pulls.clear (); @@ -722,12 +732,12 @@ void nano::bootstrap_attempt::lazy_run () idle.clear (); } -bool nano::bootstrap_attempt::process_block (std::shared_ptr block_a, nano::account const & known_account_a, uint64_t pull_blocks, bool block_expected) +bool nano::bootstrap_attempt::process_block (std::shared_ptr block_a, nano::account const & known_account_a, uint64_t pull_blocks, bool block_expected, bool confirmed_head) { bool stop_pull (false); if (mode != nano::bootstrap_mode::legacy && block_expected) { - stop_pull = process_block_lazy (block_a, known_account_a, pull_blocks); + stop_pull = process_block_lazy (block_a, known_account_a, pull_blocks, confirmed_head); } else if (mode != nano::bootstrap_mode::legacy) { @@ -742,7 +752,7 @@ bool nano::bootstrap_attempt::process_block (std::shared_ptr block_ return stop_pull; } -bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr block_a, nano::account const & known_account_a, uint64_t pull_blocks) +bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr block_a, nano::account const & known_account_a, uint64_t pull_blocks, bool confirmed_head) { bool stop_pull (false); auto hash (block_a->hash ()); @@ -755,11 +765,11 @@ bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr b // Search for new dependencies if (!block_a->source ().is_zero () && !node->ledger.block_exists (block_a->source ()) && block_a->source () != node->network_params.ledger.genesis_account) { - lazy_add (block_a->source ()); + lazy_add (block_a->source (), confirmed_head); } else if (block_a->type () == nano::block_type::state) { - lazy_block_state (block_a); + lazy_block_state (block_a, confirmed_head); } lazy_blocks.insert (hash); // Adding lazy balances for first processed block in pull @@ -782,7 +792,7 @@ bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr b return stop_pull; } -void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr block_a) +void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr block_a, bool confirmed_head) { std::shared_ptr block_l (std::static_pointer_cast (block_a)); if (block_l != nullptr) @@ -797,14 +807,14 @@ void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr blo // If state block previous is 0 then source block required if (previous.is_zero ()) { - lazy_add (link); + lazy_add (link, confirmed_head); } // In other cases previous block balance required to find out subtype of state block else if (node->store.block_exists (transaction, previous)) { if (node->ledger.balance (transaction, previous) <= balance) { - lazy_add (link); + lazy_add (link, confirmed_head); } } // Search balance of already processed previous blocks @@ -815,7 +825,7 @@ void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr blo { if (previous_balance->second <= balance) { - lazy_add (link); + lazy_add (link, confirmed_head); } lazy_balances.erase (previous_balance); } @@ -823,7 +833,7 @@ void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr blo // Insert in backlog state blocks if previous wasn't already processed else { - lazy_state_backlog.insert (std::make_pair (previous, std::make_pair (link, balance))); + lazy_state_backlog.emplace (previous, nano::lazy_state_backlog_item{ link, balance, confirmed_head }); } } } @@ -839,15 +849,15 @@ void nano::bootstrap_attempt::lazy_block_state_backlog_check (std::shared_ptrtype () == nano::block_type::state || block_a->type () == nano::block_type::send) { - if (block_a->balance ().number () <= next_block.second) // balance + if (block_a->balance ().number () <= next_block.balance) // balance { - lazy_add (next_block.first); // link + lazy_add (next_block.link, next_block.confirmed); // link } } // Assumption for other legacy block types else { - // Disabled + lazy_add (next_block.link, false); // Head is not confirmed. It can be account or hash or non-existing } lazy_state_backlog.erase (find_state); } @@ -862,9 +872,9 @@ void nano::bootstrap_attempt::lazy_backlog_cleanup () if (node->store.block_exists (transaction, it->first)) { auto next_block (it->second); - if (node->ledger.balance (transaction, it->first) <= next_block.second) // balance + if (node->ledger.balance (transaction, it->first) <= next_block.balance) // balance { - lazy_add (next_block.first); // link + lazy_add (next_block.link, next_block.confirmed); // link } it = lazy_state_backlog.erase (it); } diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 192981db54..a40dc5f203 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -43,6 +43,13 @@ enum class bootstrap_mode lazy, wallet_lazy }; +class lazy_state_backlog_item +{ +public: + nano::block_hash link{ 0 }; + nano::uint128_t balance{ 0 }; + bool confirmed{ false }; +}; class frontier_req_client; class bulk_push_client; class bootstrap_attempt final : public std::enable_shared_from_this @@ -67,17 +74,17 @@ class bootstrap_attempt final : public std::enable_shared_from_this, nano::account const &, uint64_t, bool); + bool process_block (std::shared_ptr, nano::account const &, uint64_t, bool, bool); /** Lazy bootstrap */ void lazy_run (); void lazy_start (nano::block_hash const &); - void lazy_add (nano::block_hash const &); + void lazy_add (nano::block_hash const &, bool = true); void lazy_requeue (nano::block_hash const &); bool lazy_finished (); void lazy_pull_flush (); void lazy_clear (); - bool process_block_lazy (std::shared_ptr, nano::account const &, uint64_t); - void lazy_block_state (std::shared_ptr); + bool process_block_lazy (std::shared_ptr, nano::account const &, uint64_t, bool); + void lazy_block_state (std::shared_ptr, bool); void lazy_block_state_backlog_check (std::shared_ptr, nano::block_hash const &); void lazy_backlog_cleanup (); bool lazy_processed_or_exists (nano::block_hash const &); @@ -110,10 +117,10 @@ class bootstrap_attempt final : public std::enable_shared_from_this lazy_blocks; - std::unordered_map> lazy_state_backlog; + std::unordered_map lazy_state_backlog; std::unordered_map lazy_balances; std::unordered_set lazy_keys; - std::deque lazy_pulls; + std::deque> lazy_pulls; std::chrono::steady_clock::time_point last_lazy_flush{ std::chrono::steady_clock::now () }; std::mutex lazy_mutex; // Wallet lazy bootstrap @@ -203,6 +210,7 @@ class bootstrap_limits final static constexpr double bootstrap_minimum_elapsed_seconds_blockrate = 0.02; static constexpr double bootstrap_minimum_frontier_blocks_per_sec = 1000.0; static constexpr unsigned bootstrap_frontier_retry_limit = 16; + static constexpr unsigned bootstrap_lazy_retry_limit = 50; static constexpr double bootstrap_minimum_termination_time_sec = 30.0; static constexpr unsigned bootstrap_max_new_connections = 10; static constexpr unsigned bulk_push_cost_limit = 200; diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index a72ff9861c..24f433e8a3 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -3,12 +3,13 @@ #include #include -nano::pull_info::pull_info (nano::account const & account_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a) : +nano::pull_info::pull_info (nano::account const & account_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a, bool confirmed_head_a) : account (account_a), head (head_a), head_original (head_a), end (end_a), -count (count_a) +count (count_a), +confirmed_head (confirmed_head_a) { } @@ -52,6 +53,7 @@ nano::bulk_pull_client::~bulk_pull_client () void nano::bulk_pull_client::request () { + assert (!pull.head.is_zero () || !pull.confirmed_head); expected = pull.head; nano::bulk_pull req; req.start = (pull.head == pull.head_original) ? pull.account : pull.head; // Account for new pulls, head for cached pulls @@ -203,7 +205,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e } // Is block expected? bool block_expected (false); - if (hash == expected) + if (hash == expected || (expected.is_zero () && !pull.confirmed_head)) { expected = block->previous (); block_expected = true; @@ -221,7 +223,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e connection->start_time = std::chrono::steady_clock::now (); } connection->attempt->total_blocks++; - bool stop_pull (connection->attempt->process_block (block, known_account, pull_blocks, block_expected)); + bool stop_pull (connection->attempt->process_block (block, known_account, pull_blocks, block_expected, pull.confirmed_head)); pull_blocks++; if (!stop_pull && !connection->hard_stop.load ()) { diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.hpp b/nano/node/bootstrap/bootstrap_bulk_pull.hpp index d0d8084400..61926bb2c4 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.hpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.hpp @@ -12,7 +12,7 @@ class pull_info public: using count_t = nano::bulk_pull::count_t; pull_info () = default; - pull_info (nano::account const &, nano::block_hash const &, nano::block_hash const &, count_t = 0); + pull_info (nano::account const &, nano::block_hash const &, nano::block_hash const &, count_t = 0, bool = false); nano::account account{ 0 }; nano::block_hash head{ 0 }; nano::block_hash head_original{ 0 }; @@ -20,6 +20,7 @@ class pull_info count_t count{ 0 }; unsigned attempts{ 0 }; uint64_t processed{ 0 }; + bool confirmed_head{ false }; }; class bootstrap_client; class bulk_pull_client final : public std::enable_shared_from_this From a9c905281d6a2e23ca7625c3766df5509b7f18cc Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Tue, 24 Sep 2019 01:24:03 +0300 Subject: [PATCH 03/22] Lazy destinations request --- nano/core_test/bootstrap.cpp | 34 +++++++++++++++++ nano/node/bootstrap/bootstrap.cpp | 62 +++++++++++++++++++++++++++++++ nano/node/bootstrap/bootstrap.hpp | 27 +++++++++++++- nano/node/json_handler.cpp | 1 + 4 files changed, 122 insertions(+), 2 deletions(-) diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index 984ac51465..19789cb251 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -450,6 +450,40 @@ TEST (bootstrap_processor, lazy_unclear_state_link) ASSERT_TRUE (node2->ledger.block_exists (receive->hash ())); } +TEST (bootstrap_processor, lazy_destinations) +{ + nano::system system; + nano::node_flags node_flags; + node_flags.disable_legacy_bootstrap = true; + auto node1 = system.add_node (nano::node_config (24000, system.logging), node_flags); + nano::genesis genesis; + nano::keypair key1, key2; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (genesis.hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); + auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key2.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); + auto open (std::make_shared (send1->hash (), key1.pub, key1.pub, key1.prv, key1.pub, system.work.generate (key1.pub))); + ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); + auto state_open (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, system.work.generate (key2.pub))); + ASSERT_EQ (nano::process_result::progress, node1->process (*state_open).code); + // Start lazy bootstrap with last block in sender chain + auto node2 = system.add_node (nano::node_config (24001, system.logging), node_flags); + node2->network.udp_channels.insert (node1->network.endpoint (), node1->network_params.protocol.protocol_version); + node2->bootstrap_initiator.bootstrap_lazy (send2->hash ()); + // Check processed blocks + system.deadline_set (10s); + while (node2->bootstrap_initiator.in_progress ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + node2->block_processor.flush (); + ASSERT_TRUE (node2->ledger.block_exists (send1->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (send2->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (open->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (state_open->hash ())); +} + TEST (bootstrap_processor, wallet_lazy_frontier) { nano::system system (24000, 1); diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index d41034240b..3a54dcfc36 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -18,6 +18,8 @@ constexpr unsigned nano::bootstrap_limits::bootstrap_lazy_retry_limit; constexpr double nano::bootstrap_limits::bootstrap_minimum_termination_time_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_max_new_connections; constexpr std::chrono::seconds nano::bootstrap_limits::lazy_flush_delay_sec; +constexpr unsigned nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit; +constexpr std::chrono::seconds nano::bootstrap_limits::lazy_destinations_flush_delay_min; nano::bootstrap_client::bootstrap_client (std::shared_ptr node_a, std::shared_ptr attempt_a, std::shared_ptr channel_a) : node (node_a), @@ -697,6 +699,7 @@ void nano::bootstrap_attempt::lazy_run () if (pulls.empty ()) { lazy_backlog_cleanup (); + lazy_destinations_flush (); } } if (!stopped) @@ -771,6 +774,14 @@ bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr b { lazy_block_state (block_a, confirmed_head); } + else if (block_a->type () == nano::block_type::send) + { + std::shared_ptr block_l (std::static_pointer_cast (block_a)); + if (block_l != nullptr && !block_l->hashables.destination.is_zero ()) + { + lazy_destinations_increment (block_l->hashables.destination); + } + } lazy_blocks.insert (hash); // Adding lazy balances for first processed block in pull if (pull_blocks == 0 && (block_a->type () == nano::block_type::state || block_a->type () == nano::block_type::send)) @@ -816,6 +827,10 @@ void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr blo { lazy_add (link, confirmed_head); } + else + { + lazy_destinations_increment (link); + } } // Search balance of already processed previous blocks else if (lazy_blocks.find (previous) != lazy_blocks.end ()) @@ -827,6 +842,10 @@ void nano::bootstrap_attempt::lazy_block_state (std::shared_ptr blo { lazy_add (link, confirmed_head); } + else + { + lazy_destinations_increment (link); + } lazy_balances.erase (previous_balance); } } @@ -853,6 +872,10 @@ void nano::bootstrap_attempt::lazy_block_state_backlog_check (std::shared_ptr ().find (destination_a)); + if (existing != lazy_destinations.get ().end ()) + { + lazy_destinations.get ().modify (existing, [](nano::lazy_destinations_item & item_a) { + ++item_a.count; + }); + } + else + { + lazy_destinations.insert (nano::lazy_destinations_item{ destination_a, 1 }); + } +} + +void nano::bootstrap_attempt::lazy_destinations_flush () +{ + size_t count (0); + nano::unique_lock lazy_lock (lazy_mutex); + if (last_lazy_destinations_flush + nano::bootstrap_limits::lazy_destinations_flush_delay_min < std::chrono::steady_clock::now ()) + { + for (auto it (lazy_destinations.get ().begin ()), end (lazy_destinations.get ().end ()); it != end && count < nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit && !stopped;) + { + lazy_add (it->account, false); + it = lazy_destinations.get ().erase (it); + ++count; + } + if (count > nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit / 4) + { + last_lazy_destinations_flush = std::chrono::steady_clock::now (); + } + } +} + bool nano::bootstrap_attempt::lazy_processed_or_exists (nano::block_hash const & hash_a) { bool result (false); diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index a40dc5f203..ae2b7d11dd 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -43,13 +43,19 @@ enum class bootstrap_mode lazy, wallet_lazy }; -class lazy_state_backlog_item +class lazy_state_backlog_item final { public: nano::block_hash link{ 0 }; nano::uint128_t balance{ 0 }; bool confirmed{ false }; }; +class lazy_destinations_item final +{ +public: + nano::account account{ 0 }; + uint64_t count{ 0 }; +}; class frontier_req_client; class bulk_push_client; class bootstrap_attempt final : public std::enable_shared_from_this @@ -87,6 +93,8 @@ class bootstrap_attempt final : public std::enable_shared_from_this, bool); void lazy_block_state_backlog_check (std::shared_ptr, nano::block_hash const &); void lazy_backlog_cleanup (); + void lazy_destinations_increment (nano::account const &); + void lazy_destinations_flush (); bool lazy_processed_or_exists (nano::block_hash const &); /** Lazy bootstrap */ /** Wallet bootstrap */ @@ -122,6 +130,19 @@ class bootstrap_attempt final : public std::enable_shared_from_this lazy_keys; std::deque> lazy_pulls; std::chrono::steady_clock::time_point last_lazy_flush{ std::chrono::steady_clock::now () }; + class account_tag + { + }; + class count_tag + { + }; + boost::multi_index_container< + lazy_destinations_item, + boost::multi_index::indexed_by< + boost::multi_index::ordered_non_unique, boost::multi_index::member, std::greater>, + boost::multi_index::hashed_unique, boost::multi_index::member>>> + lazy_destinations; + std::chrono::steady_clock::time_point last_lazy_destinations_flush{ std::chrono::steady_clock::time_point{} }; std::mutex lazy_mutex; // Wallet lazy bootstrap std::deque wallet_accounts; @@ -210,10 +231,12 @@ class bootstrap_limits final static constexpr double bootstrap_minimum_elapsed_seconds_blockrate = 0.02; static constexpr double bootstrap_minimum_frontier_blocks_per_sec = 1000.0; static constexpr unsigned bootstrap_frontier_retry_limit = 16; - static constexpr unsigned bootstrap_lazy_retry_limit = 50; + static constexpr unsigned bootstrap_lazy_retry_limit = bootstrap_frontier_retry_limit * 10; static constexpr double bootstrap_minimum_termination_time_sec = 30.0; static constexpr unsigned bootstrap_max_new_connections = 10; static constexpr unsigned bulk_push_cost_limit = 200; static constexpr std::chrono::seconds lazy_flush_delay_sec = std::chrono::seconds (5); + static constexpr unsigned bootstrap_lazy_destinations_request_limit = 200; + static constexpr std::chrono::seconds lazy_destinations_flush_delay_min = std::chrono::minutes (2); }; } diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 433a8cbd2b..4989decb4e 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1666,6 +1666,7 @@ void nano::json_handler::bootstrap_status () response_l.put ("lazy_blocks", std::to_string (attempt->lazy_blocks.size ())); response_l.put ("lazy_state_backlog", std::to_string (attempt->lazy_state_backlog.size ())); response_l.put ("lazy_balances", std::to_string (attempt->lazy_balances.size ())); + response_l.put ("lazy_destinations", std::to_string (attempt->lazy_destinations.size ())); response_l.put ("lazy_pulls", std::to_string (attempt->lazy_pulls.size ())); response_l.put ("lazy_keys", std::to_string (attempt->lazy_keys.size ())); if (!attempt->lazy_keys.empty ()) From 1385baf7a31fb17a86eaaf3e21ce0ee761fec75f Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Tue, 24 Sep 2019 13:15:14 +0300 Subject: [PATCH 04/22] Update expected block status with unconfirmed head only if account is expected --- nano/node/bootstrap/bootstrap.cpp | 13 ++----------- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 3 ++- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 3a54dcfc36..9d6f31d672 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -540,7 +540,7 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) assert (pull.account == pull.head); if (!lazy_processed_or_exists (pull.account)) { - // Retry for lazy pulls (not weak state block link assumptions) + // Retry for lazy pulls nano::lock_guard lock (mutex); pulls.push_back (pull); condition.notify_all (); @@ -611,16 +611,7 @@ void nano::bootstrap_attempt::lazy_pull_flush () if (lazy_blocks.find (pull_start.first) == lazy_blocks.end () && !node->store.block_exists (transaction, pull_start.first)) { assert (node->network_params.bootstrap.lazy_max_pull_blocks <= std::numeric_limits::max ()); - if (pull_start.second) - { - // Confirmed head block - pulls.push_back (nano::pull_info (pull_start.first, pull_start.first, nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second)); - } - else - { - // Head is not confirmed. It can be account or hash or non-existing - pulls.push_back (nano::pull_info (pull_start.first, nano::block_hash (0), nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second)); - } + pulls.push_back (nano::pull_info (pull_start.first, pull_start.first, nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second)); } } lazy_pulls.clear (); diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 24f433e8a3..9f009c2f7b 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -205,7 +205,8 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e } // Is block expected? bool block_expected (false); - if (hash == expected || (expected.is_zero () && !pull.confirmed_head)) + bool unconfirmed_account_head (pull_blocks == 0 && !pull.confirmed_head && expected == pull.account && block->account () == pull.account); + if (hash == expected || unconfirmed_account_head) { expected = block->previous (); block_expected = true; From 5f8a273ebdb13f910582942a1c5ef133870701ec Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Tue, 24 Sep 2019 19:25:47 +0300 Subject: [PATCH 05/22] Remove expected rewrite for stopped pulls --- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 9f009c2f7b..ae28264f61 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -238,7 +238,6 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e } else if (stop_pull && block_expected) { - expected = pull.end; connection->attempt->pool_connection (connection); } } From e2061e32fa57a87d87ada2244aecb28ae4eaf19f Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Tue, 24 Sep 2019 20:55:34 +0300 Subject: [PATCH 06/22] Add bootstrap_hash_or_account class --- nano/core_test/bootstrap.cpp | 10 +++--- nano/lib/numbers.cpp | 36 +++++++++++++++++++++ nano/lib/numbers.hpp | 36 +++++++++++++++++++++ nano/node/bootstrap/bootstrap.cpp | 18 +++++------ nano/node/bootstrap/bootstrap.hpp | 6 ++-- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 14 ++++---- nano/node/bootstrap/bootstrap_bulk_pull.hpp | 8 ++--- nano/node/common.hpp | 2 +- 8 files changed, 101 insertions(+), 29 deletions(-) diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index 19789cb251..b792e9f1a0 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -335,7 +335,7 @@ TEST (bootstrap_processor, push_one) nano::system system (24000, 1); nano::keypair key1; auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); - auto wallet (node1->wallets.create (nano::uint256_union ())); + auto wallet (node1->wallets.create (nano::random_wallet_id ())); ASSERT_NE (nullptr, wallet); wallet->insert_adhoc (nano::test_genesis_key.prv); nano::uint128_t balance1 (node1->balance (nano::test_genesis_key.pub)); @@ -504,7 +504,7 @@ TEST (bootstrap_processor, wallet_lazy_frontier) // Start wallet lazy bootstrap auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); - auto wallet (node1->wallets.create (nano::uint256_union ())); + auto wallet (node1->wallets.create (nano::random_wallet_id ())); ASSERT_NE (nullptr, wallet); wallet->insert_adhoc (key2.prv); node1->bootstrap_wallet (); @@ -535,7 +535,7 @@ TEST (bootstrap_processor, wallet_lazy_pending) // Start wallet lazy bootstrap auto node1 (std::make_shared (system.io_ctx, 24001, nano::unique_path (), system.alarm, system.logging, system.work)); node1->network.udp_channels.insert (system.nodes[0]->network.endpoint (), node1->network_params.protocol.protocol_version); - auto wallet (node1->wallets.create (nano::uint256_union ())); + auto wallet (node1->wallets.create (nano::random_wallet_id ())); ASSERT_NE (nullptr, wallet); wallet->insert_adhoc (key2.prv); node1->bootstrap_wallet (); @@ -698,7 +698,7 @@ TEST (bulk, offline_send) node1->start (); system.nodes.push_back (node1); nano::keypair key2; - auto wallet (node1->wallets.create (nano::uint256_union ())); + auto wallet (node1->wallets.create (nano::random_wallet_id ())); wallet->insert_adhoc (key2.prv); ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::test_genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ())); ASSERT_NE (std::numeric_limits::max (), system.nodes[0]->balance (nano::test_genesis_key.pub)); @@ -732,7 +732,7 @@ TEST (bulk, offline_send) TEST (bulk_pull_account, basics) { nano::system system (24000, 1); - system.nodes[0]->config.receive_minimum = nano::uint128_union (20); + system.nodes[0]->config.receive_minimum = 20; nano::keypair key1; system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); system.wallet (0)->insert_adhoc (key1.prv); diff --git a/nano/lib/numbers.cpp b/nano/lib/numbers.cpp index af5d5ee726..9872ee44a0 100644 --- a/nano/lib/numbers.cpp +++ b/nano/lib/numbers.cpp @@ -837,6 +837,11 @@ hash_or_account (value_a) { } +nano::bootstrap_hash_or_account::bootstrap_hash_or_account (uint64_t value_a) : +hash_or_account (value_a) +{ +} + nano::block_hash const & nano::root::previous () const { return hash; @@ -862,6 +867,16 @@ bool nano::link::operator!= (nano::link const & link_a) const return !(*this == link_a); } +bool nano::bootstrap_hash_or_account::operator== (nano::bootstrap_hash_or_account const & bootstrap_hash_or_account_a) const +{ + return bytes == bootstrap_hash_or_account_a.bytes; +} + +bool nano::bootstrap_hash_or_account::operator!= (nano::bootstrap_hash_or_account const & bootstrap_hash_or_account_a) const +{ + return !(*this == bootstrap_hash_or_account_a); +} + std::string nano::to_string_hex (uint64_t const value_a) { std::stringstream stream; @@ -939,6 +954,11 @@ nano::public_key::operator nano::root const & () const return nano::to_root (*this); } +nano::public_key::operator nano::bootstrap_hash_or_account const & () const +{ + return nano::to_bootstrap (*this); +} + nano::block_hash::operator nano::link const & () const { return nano::to_link (*this); @@ -948,3 +968,19 @@ nano::block_hash::operator nano::root const & () const { return nano::to_root (*this); } + +nano::block_hash::operator nano::bootstrap_hash_or_account const & () const +{ + return nano::to_bootstrap (*this); +} + +nano::link::operator nano::bootstrap_hash_or_account const & () const +{ + return nano::to_bootstrap (*this); +} + + +nano::root::operator nano::bootstrap_hash_or_account const & () const +{ + return nano::to_bootstrap (*this); +} diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index 4d892df4f6..ad61011433 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -100,6 +100,7 @@ static_assert (std::is_nothrow_move_constructible::value, "uint25 class link; class root; +class bootstrap_hash_or_account; // All keys and hashes are 256 bit. class block_hash final : public uint256_union @@ -108,6 +109,7 @@ class block_hash final : public uint256_union using uint256_union::uint256_union; operator nano::link const & () const; operator nano::root const & () const; + operator nano::bootstrap_hash_or_account const & () const; }; class public_key final : public uint256_union @@ -122,6 +124,7 @@ class public_key final : public uint256_union operator nano::link const & () const; operator nano::root const & () const; + operator nano::bootstrap_hash_or_account const & () const; }; class wallet_id : public uint256_union @@ -169,6 +172,7 @@ class link final : public hash_or_account bool operator== (nano::link const &) const; bool operator!= (nano::link const &) const; + operator nano::bootstrap_hash_or_account const & () const; }; // A root can either be an open block hash or a previous hash @@ -180,10 +184,22 @@ class root final : public hash_or_account bool operator== (nano::root const &) const; bool operator!= (nano::root const &) const; + operator nano::bootstrap_hash_or_account const & () const; nano::block_hash const & previous () const; }; +// Bootstrap hash or account +class bootstrap_hash_or_account final : public hash_or_account +{ +public: + bootstrap_hash_or_account () = default; + bootstrap_hash_or_account (uint64_t value_a); + + bool operator== (nano::bootstrap_hash_or_account const &) const; + bool operator!= (nano::bootstrap_hash_or_account const &) const; +}; + class private_key : public uint256_union { public: @@ -247,6 +263,26 @@ inline nano::root const & to_root (nano::account const & account_a) return reinterpret_cast (account_a); } +inline nano::bootstrap_hash_or_account const & to_bootstrap (nano::block_hash const & hash_a) +{ + return reinterpret_cast (hash_a); +} + +inline nano::bootstrap_hash_or_account const & to_bootstrap (nano::account const & account_a) +{ + return reinterpret_cast (account_a); +} + +inline nano::bootstrap_hash_or_account const & to_bootstrap (nano::link const & link_a) +{ + return reinterpret_cast (link_a); +} + +inline nano::bootstrap_hash_or_account const & to_bootstrap (nano::root const & root_a) +{ + return reinterpret_cast (root_a); +} + inline nano::account const & root_as_account (nano::root const & root_a) { static_assert (sizeof (nano::root) == sizeof (nano::account), "Sizes do not match"); diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 4fd42c6f4e..fd7c60510c 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -537,8 +537,8 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) } else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= nano::bootstrap_limits::bootstrap_lazy_retry_limit)) { - assert (pull.root == pull.head); - if (!lazy_processed_or_exists (pull.root)) + assert (pull.account_or_head == pull.head); + if (!lazy_processed_or_exists (pull.account_or_head)) { // Retry for lazy pulls nano::lock_guard lock (mutex); @@ -550,7 +550,7 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) { if (node->config.logging.bulk_pull_logging ()) { - node->logger.try_log (boost::str (boost::format ("Failed to pull account %1% down to %2% after %3% attempts and %4% blocks processed") % pull.root.to_account () % pull.end.to_string () % pull.attempts % pull.processed)); + node->logger.try_log (boost::str (boost::format ("Failed to pull account %1% down to %2% after %3% attempts and %4% blocks processed") % pull.account_or_head.to_account () % pull.end.to_string () % pull.attempts % pull.processed)); } node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in); @@ -576,13 +576,13 @@ void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) } } -void nano::bootstrap_attempt::lazy_add (nano::block_hash const & hash_a, bool confirmed_head) +void nano::bootstrap_attempt::lazy_add (nano::bootstrap_hash_or_account const & hash_or_account_a, bool confirmed_head) { // Add only unknown blocks assert (!lazy_mutex.try_lock ()); - if (lazy_blocks.find (hash_a) == lazy_blocks.end ()) + if (lazy_blocks.find (hash_or_account_a) == lazy_blocks.end ()) { - lazy_pulls.push_back (std::make_pair (hash_a, confirmed_head)); + lazy_pulls.push_back (std::make_pair (hash_or_account_a, confirmed_head)); } } @@ -1233,7 +1233,7 @@ void nano::pulls_cache::add (nano::pull_info const & pull_a) cache.erase (cache.begin ()); } assert (cache.size () <= cache_size_max); - nano::uint512_union head_512 (pull_a.root, pull_a.head_original); + nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original); auto existing (cache.get ().find (head_512)); if (existing == cache.get ().end ()) { @@ -1256,7 +1256,7 @@ void nano::pulls_cache::add (nano::pull_info const & pull_a) void nano::pulls_cache::update_pull (nano::pull_info & pull_a) { nano::lock_guard guard (pulls_cache_mutex); - nano::uint512_union head_512 (pull_a.root, pull_a.head_original); + nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original); auto existing (cache.get ().find (head_512)); if (existing != cache.get ().end ()) { @@ -1267,7 +1267,7 @@ void nano::pulls_cache::update_pull (nano::pull_info & pull_a) void nano::pulls_cache::remove (nano::pull_info const & pull_a) { nano::lock_guard guard (pulls_cache_mutex); - nano::uint512_union head_512 (pull_a.root, pull_a.head_original); + nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original); cache.get ().erase (head_512); } diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index ae2b7d11dd..5f5c3b45da 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -46,7 +46,7 @@ enum class bootstrap_mode class lazy_state_backlog_item final { public: - nano::block_hash link{ 0 }; + nano::link link{ 0 }; nano::uint128_t balance{ 0 }; bool confirmed{ false }; }; @@ -84,7 +84,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this lazy_state_backlog; std::unordered_map lazy_balances; std::unordered_set lazy_keys; - std::deque> lazy_pulls; + std::deque> lazy_pulls; std::chrono::steady_clock::time_point last_lazy_flush{ std::chrono::steady_clock::now () }; class account_tag { diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index b37014d836..5bc8f08559 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -3,8 +3,8 @@ #include #include -nano::pull_info::pull_info (nano::root const & root_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a, bool confirmed_head_a) : -root (root_a), +nano::pull_info::pull_info (nano::bootstrap_hash_or_account const & account_or_head_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a, bool confirmed_head_a) : +account_or_head (account_or_head_a), head (head_a), head_original (head_a), end (end_a), @@ -31,13 +31,13 @@ nano::bulk_pull_client::~bulk_pull_client () pull.head = expected; if (connection->attempt->mode != nano::bootstrap_mode::legacy) { - pull.root = expected; + pull.account_or_head = expected; } pull.processed += pull_blocks - unexpected_count; connection->attempt->requeue_pull (pull); if (connection->node->config.logging.bulk_pull_logging ()) { - connection->node->logger.try_log (boost::str (boost::format ("Bulk pull end block is not expected %1% for account %2%") % pull.end.to_string () % pull.root.to_account ())); + connection->node->logger.try_log (boost::str (boost::format ("Bulk pull end block is not expected %1% for account %2%") % pull.end.to_string () % pull.account_or_head.to_account ())); } } else @@ -59,7 +59,7 @@ void nano::bulk_pull_client::request () if (pull.head == pull.head_original) { // Account for new pulls - req.start = pull.root; + req.start = pull.account_or_head; } else { @@ -73,7 +73,7 @@ void nano::bulk_pull_client::request () if (connection->node->config.logging.bulk_pull_logging ()) { nano::unique_lock lock (connection->attempt->mutex); - connection->node->logger.try_log (boost::str (boost::format ("Requesting account %1% from %2%. %3% accounts in queue") % pull.root.to_account () % connection->channel->to_string () % connection->attempt->pulls.size ())); + connection->node->logger.try_log (boost::str (boost::format ("Requesting account %1% from %2%. %3% accounts in queue") % pull.account_or_head.to_account () % connection->channel->to_string () % connection->attempt->pulls.size ())); } else if (connection->node->config.logging.network_logging () && connection->attempt->should_log ()) { @@ -214,7 +214,7 @@ void nano::bulk_pull_client::received_block (boost::system::error_code const & e } // Is block expected? bool block_expected (false); - bool unconfirmed_account_head (pull_blocks == 0 && !pull.confirmed_head && expected == pull.root && block->account () == pull.root); + bool unconfirmed_account_head (pull_blocks == 0 && !pull.confirmed_head && expected == pull.account_or_head && block->account () == pull.account_or_head); if (hash == expected || unconfirmed_account_head) { expected = block->previous (); diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.hpp b/nano/node/bootstrap/bootstrap_bulk_pull.hpp index ec9b1cf532..19f443276a 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.hpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.hpp @@ -12,8 +12,8 @@ class pull_info public: using count_t = nano::bulk_pull::count_t; pull_info () = default; - pull_info (nano::root const &, nano::block_hash const &, nano::block_hash const &, count_t = 0, bool = false); - nano::root root{ 0 }; + pull_info (nano::bootstrap_hash_or_account const &, nano::block_hash const &, nano::block_hash const &, count_t = 0, bool = false); + nano::bootstrap_hash_or_account account_or_head{ 0 }; nano::block_hash head{ 0 }; nano::block_hash head_original{ 0 }; nano::block_hash end{ 0 }; @@ -35,7 +35,7 @@ class bulk_pull_client final : public std::enable_shared_from_this connection; - nano::root expected; + nano::block_hash expected; nano::account known_account; nano::pull_info pull; uint64_t pull_blocks; @@ -66,7 +66,7 @@ class bulk_pull_server final : public std::enable_shared_from_this connection; std::unique_ptr request; - nano::root current; + nano::block_hash current; bool include_start; nano::bulk_pull::count_t max_count; nano::bulk_pull::count_t sent_count; diff --git a/nano/node/common.hpp b/nano/node/common.hpp index 6f01b3f5a3..59cc01b409 100644 --- a/nano/node/common.hpp +++ b/nano/node/common.hpp @@ -355,7 +355,7 @@ class bulk_pull final : public message void serialize (nano::stream &) const override; bool deserialize (nano::stream &); void visit (nano::message_visitor &) const override; - nano::root start; + nano::bootstrap_hash_or_account start; nano::block_hash end; count_t count; bool is_count_present () const; From b6bf777c6b2c76941667b5c064b1e2ea582d3d0b Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Wed, 25 Sep 2019 00:23:59 +0300 Subject: [PATCH 07/22] Formatting --- nano/lib/numbers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/nano/lib/numbers.cpp b/nano/lib/numbers.cpp index 9872ee44a0..59f043aea1 100644 --- a/nano/lib/numbers.cpp +++ b/nano/lib/numbers.cpp @@ -979,7 +979,6 @@ nano::link::operator nano::bootstrap_hash_or_account const & () const return nano::to_bootstrap (*this); } - nano::root::operator nano::bootstrap_hash_or_account const & () const { return nano::to_bootstrap (*this); From 72ab97f8aa624ad13f95eb0a47e765bff942bcdf Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Wed, 25 Sep 2019 01:23:36 +0300 Subject: [PATCH 08/22] More frequent destinations check & backlog cleanup --- nano/node/bootstrap/bootstrap.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index fd7c60510c..dcb4ea154c 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -651,6 +651,7 @@ void nano::bootstrap_attempt::lazy_clear () lazy_pulls.clear (); lazy_state_backlog.clear (); lazy_balances.clear (); + lazy_destinations.clear (); } void nano::bootstrap_attempt::lazy_run () @@ -683,6 +684,12 @@ void nano::bootstrap_attempt::lazy_run () { lazy_pull_flush (); } + // Start destinations check & backlog cleanup + if (iterations % 200 == 0 && pulls.empty ()) + { + lazy_backlog_cleanup (); + lazy_destinations_flush (); + } } // Flushing lazy pulls lazy_pull_flush (); From 59dd4b5a380b0b710ab8843f2dcc7b03fe75dc87 Mon Sep 17 00:00:00 2001 From: wezrule Date: Wed, 25 Sep 2019 10:56:41 +0100 Subject: [PATCH 09/22] Allow hash_or_account to be concrete --- nano/lib/numbers.cpp | 46 +++++++----------- nano/lib/numbers.hpp | 53 +++++---------------- nano/node/bootstrap/bootstrap.cpp | 12 ++--- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 12 ++--- nano/node/bootstrap/bootstrap_bulk_pull.hpp | 8 ++-- nano/node/common.hpp | 2 +- 6 files changed, 46 insertions(+), 87 deletions(-) diff --git a/nano/lib/numbers.cpp b/nano/lib/numbers.cpp index af5d5ee726..1254b0b77b 100644 --- a/nano/lib/numbers.cpp +++ b/nano/lib/numbers.cpp @@ -827,39 +827,19 @@ nano::hash_or_account::operator nano::uint256_union const & () const return raw; } -nano::link::link (uint64_t value_a) : -hash_or_account (value_a) -{ -} - -nano::root::root (uint64_t value_a) : -hash_or_account (value_a) -{ -} - nano::block_hash const & nano::root::previous () const { return hash; } -bool nano::root::operator== (nano::root const & root_a) const -{ - return bytes == root_a.bytes; -} - -bool nano::root::operator!= (nano::root const & root_a) const +bool nano::hash_or_account::operator== (nano::hash_or_account const & hash_or_account_a) const { - return !(*this == root_a); + return bytes == hash_or_account_a.bytes; } -bool nano::link::operator== (nano::link const & link_a) const +bool nano::hash_or_account::operator!= (nano::hash_or_account const & hash_or_account_a) const { - return bytes == link_a.bytes; -} - -bool nano::link::operator!= (nano::link const & link_a) const -{ - return !(*this == link_a); + return !(*this == hash_or_account_a); } std::string nano::to_string_hex (uint64_t const value_a) @@ -931,20 +911,30 @@ double nano::difficulty::to_multiplier (uint64_t const difficulty_a, uint64_t co nano::public_key::operator nano::link const & () const { - return nano::to_link (*this); + return reinterpret_cast (*this); } nano::public_key::operator nano::root const & () const { - return nano::to_root (*this); + return reinterpret_cast (*this); +} + +nano::public_key::operator nano::hash_or_account const & () const +{ + return reinterpret_cast (*this); } nano::block_hash::operator nano::link const & () const { - return nano::to_link (*this); + return reinterpret_cast (*this); } nano::block_hash::operator nano::root const & () const { - return nano::to_root (*this); + return reinterpret_cast (*this); +} + +nano::block_hash::operator nano::hash_or_account const & () const +{ + return reinterpret_cast (*this); } diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index 4d892df4f6..4735af1bba 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -100,6 +100,7 @@ static_assert (std::is_nothrow_move_constructible::value, "uint25 class link; class root; +class hash_or_account; // All keys and hashes are 256 bit. class block_hash final : public uint256_union @@ -108,6 +109,7 @@ class block_hash final : public uint256_union using uint256_union::uint256_union; operator nano::link const & () const; operator nano::root const & () const; + operator nano::hash_or_account const & () const; }; class public_key final : public uint256_union @@ -122,6 +124,7 @@ class public_key final : public uint256_union operator nano::link const & () const; operator nano::root const & () const; + operator nano::hash_or_account const & () const; }; class wallet_id : public uint256_union @@ -135,6 +138,9 @@ using account = public_key; class hash_or_account { public: + hash_or_account () = default; + hash_or_account (uint64_t value_a); + bool is_zero () const; void clear (); std::string to_string () const; @@ -146,6 +152,9 @@ class hash_or_account operator nano::account const & () const; operator nano::uint256_union const & () const; + bool operator== (nano::hash_or_account const &) const; + bool operator!= (nano::hash_or_account const &) const; + union { std::array bytes; @@ -153,33 +162,20 @@ class hash_or_account nano::account account; nano::block_hash hash; }; - -protected: - // Cannot instantiate a concrete version of this class - hash_or_account () = default; - hash_or_account (uint64_t value_a); }; // A link can either be a destination account or source hash class link final : public hash_or_account { public: - link () = default; - link (uint64_t value_a); - - bool operator== (nano::link const &) const; - bool operator!= (nano::link const &) const; + using hash_or_account::hash_or_account; }; // A root can either be an open block hash or a previous hash class root final : public hash_or_account { public: - root () = default; - root (uint64_t value_a); - - bool operator== (nano::root const &) const; - bool operator!= (nano::root const &) const; + using hash_or_account::hash_or_account; nano::block_hash const & previous () const; }; @@ -227,33 +223,6 @@ class uint512_union }; static_assert (std::is_nothrow_move_constructible::value, "uint512_union should be noexcept MoveConstructible"); -inline nano::link const & to_link (nano::block_hash const & hash_a) -{ - return reinterpret_cast (hash_a); -} - -inline nano::link const & to_link (nano::account const & account_a) -{ - return reinterpret_cast (account_a); -} - -inline nano::root const & to_root (nano::block_hash const & hash_a) -{ - return reinterpret_cast (hash_a); -} - -inline nano::root const & to_root (nano::account const & account_a) -{ - return reinterpret_cast (account_a); -} - -inline nano::account const & root_as_account (nano::root const & root_a) -{ - static_assert (sizeof (nano::root) == sizeof (nano::account), "Sizes do not match"); - static_assert (std::is_standard_layout::value && std::is_standard_layout::value, "Both types must have standard layout"); - return reinterpret_cast (root_a); -} - class signature : public uint512_union { public: diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index f14a5c539c..b1c5dea398 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -533,8 +533,8 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) } else if (mode == nano::bootstrap_mode::lazy) { - assert (pull.root == pull.head); - if (!lazy_processed_or_exists (pull.root)) + assert (pull.account_or_head == pull.head); + if (!lazy_processed_or_exists (pull.account_or_head)) { // Retry for lazy pulls (not weak state block link assumptions) nano::lock_guard lock (mutex); @@ -547,7 +547,7 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) { if (node->config.logging.bulk_pull_logging ()) { - node->logger.try_log (boost::str (boost::format ("Failed to pull account %1% down to %2% after %3% attempts and %4% blocks processed") % pull.root.to_account () % pull.end.to_string () % pull.attempts % pull.processed)); + node->logger.try_log (boost::str (boost::format ("Failed to pull account %1% down to %2% after %3% attempts and %4% blocks processed") % pull.account_or_head.to_account () % pull.end.to_string () % pull.attempts % pull.processed)); } node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in); @@ -1170,7 +1170,7 @@ void nano::pulls_cache::add (nano::pull_info const & pull_a) cache.erase (cache.begin ()); } assert (cache.size () <= cache_size_max); - nano::uint512_union head_512 (pull_a.root, pull_a.head_original); + nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original); auto existing (cache.get ().find (head_512)); if (existing == cache.get ().end ()) { @@ -1193,7 +1193,7 @@ void nano::pulls_cache::add (nano::pull_info const & pull_a) void nano::pulls_cache::update_pull (nano::pull_info & pull_a) { nano::lock_guard guard (pulls_cache_mutex); - nano::uint512_union head_512 (pull_a.root, pull_a.head_original); + nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original); auto existing (cache.get ().find (head_512)); if (existing != cache.get ().end ()) { @@ -1204,7 +1204,7 @@ void nano::pulls_cache::update_pull (nano::pull_info & pull_a) void nano::pulls_cache::remove (nano::pull_info const & pull_a) { nano::lock_guard guard (pulls_cache_mutex); - nano::uint512_union head_512 (pull_a.root, pull_a.head_original); + nano::uint512_union head_512 (pull_a.account_or_head, pull_a.head_original); cache.get ().erase (head_512); } diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 2f5c39e2ee..1bd430bc2b 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -3,8 +3,8 @@ #include #include -nano::pull_info::pull_info (nano::root const & root_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a) : -root (root_a), +nano::pull_info::pull_info (nano::hash_or_account const & account_or_head_a, nano::block_hash const & head_a, nano::block_hash const & end_a, count_t count_a) : +account_or_head (account_or_head_a), head (head_a), head_original (head_a), end (end_a), @@ -30,13 +30,13 @@ nano::bulk_pull_client::~bulk_pull_client () pull.head = expected; if (connection->attempt->mode != nano::bootstrap_mode::legacy) { - pull.root = expected; + pull.account_or_head = expected; } pull.processed += pull_blocks - unexpected_count; connection->attempt->requeue_pull (pull); if (connection->node->config.logging.bulk_pull_logging ()) { - connection->node->logger.try_log (boost::str (boost::format ("Bulk pull end block is not expected %1% for account %2%") % pull.end.to_string () % pull.root.to_account ())); + connection->node->logger.try_log (boost::str (boost::format ("Bulk pull end block is not expected %1% for account %2%") % pull.end.to_string () % pull.account_or_head.to_account ())); } } else @@ -57,7 +57,7 @@ void nano::bulk_pull_client::request () if (pull.head == pull.head_original) { // Account for new pulls - req.start = pull.root; + req.start = pull.account_or_head; } else { @@ -71,7 +71,7 @@ void nano::bulk_pull_client::request () if (connection->node->config.logging.bulk_pull_logging ()) { nano::unique_lock lock (connection->attempt->mutex); - connection->node->logger.try_log (boost::str (boost::format ("Requesting account %1% from %2%. %3% accounts in queue") % pull.root.to_account () % connection->channel->to_string () % connection->attempt->pulls.size ())); + connection->node->logger.try_log (boost::str (boost::format ("Requesting account %1% from %2%. %3% accounts in queue") % pull.account_or_head.to_account () % connection->channel->to_string () % connection->attempt->pulls.size ())); } else if (connection->node->config.logging.network_logging () && connection->attempt->should_log ()) { diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.hpp b/nano/node/bootstrap/bootstrap_bulk_pull.hpp index 329e167564..2f1dc6af39 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.hpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.hpp @@ -12,8 +12,8 @@ class pull_info public: using count_t = nano::bulk_pull::count_t; pull_info () = default; - pull_info (nano::root const &, nano::block_hash const &, nano::block_hash const &, count_t = 0); - nano::root root{ 0 }; + pull_info (nano::hash_or_account const &, nano::block_hash const &, nano::block_hash const &, count_t = 0); + nano::hash_or_account account_or_head{ 0 }; nano::block_hash head{ 0 }; nano::block_hash head_original{ 0 }; nano::block_hash end{ 0 }; @@ -34,7 +34,7 @@ class bulk_pull_client final : public std::enable_shared_from_this connection; - nano::root expected; + nano::block_hash expected; nano::account known_account; nano::pull_info pull; uint64_t pull_blocks; @@ -65,7 +65,7 @@ class bulk_pull_server final : public std::enable_shared_from_this connection; std::unique_ptr request; - nano::root current; + nano::block_hash current; bool include_start; nano::bulk_pull::count_t max_count; nano::bulk_pull::count_t sent_count; diff --git a/nano/node/common.hpp b/nano/node/common.hpp index 6f01b3f5a3..3a9ed3f6be 100644 --- a/nano/node/common.hpp +++ b/nano/node/common.hpp @@ -355,7 +355,7 @@ class bulk_pull final : public message void serialize (nano::stream &) const override; bool deserialize (nano::stream &); void visit (nano::message_visitor &) const override; - nano::root start; + nano::hash_or_account start; nano::block_hash end; count_t count; bool is_count_present () const; From e9f5c80714bc791f8e98d383c8a6ecffb46ae496 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Wed, 25 Sep 2019 15:44:36 +0300 Subject: [PATCH 10/22] Fix merge issues --- nano/lib/numbers.cpp | 30 ------------------------------ nano/lib/numbers.hpp | 11 ----------- nano/node/bootstrap/bootstrap.cpp | 2 +- nano/node/bootstrap/bootstrap.hpp | 4 ++-- 4 files changed, 3 insertions(+), 44 deletions(-) diff --git a/nano/lib/numbers.cpp b/nano/lib/numbers.cpp index c45470fda8..1254b0b77b 100644 --- a/nano/lib/numbers.cpp +++ b/nano/lib/numbers.cpp @@ -842,16 +842,6 @@ bool nano::hash_or_account::operator!= (nano::hash_or_account const & hash_or_ac return !(*this == hash_or_account_a); } -bool nano::bootstrap_hash_or_account::operator== (nano::bootstrap_hash_or_account const & bootstrap_hash_or_account_a) const -{ - return bytes == bootstrap_hash_or_account_a.bytes; -} - -bool nano::bootstrap_hash_or_account::operator!= (nano::bootstrap_hash_or_account const & bootstrap_hash_or_account_a) const -{ - return !(*this == bootstrap_hash_or_account_a); -} - std::string nano::to_string_hex (uint64_t const value_a) { std::stringstream stream; @@ -934,11 +924,6 @@ nano::public_key::operator nano::hash_or_account const & () const return reinterpret_cast (*this); } -nano::public_key::operator nano::bootstrap_hash_or_account const & () const -{ - return nano::to_bootstrap (*this); -} - nano::block_hash::operator nano::link const & () const { return reinterpret_cast (*this); @@ -953,18 +938,3 @@ nano::block_hash::operator nano::hash_or_account const & () const { return reinterpret_cast (*this); } - -nano::block_hash::operator nano::bootstrap_hash_or_account const & () const -{ - return nano::to_bootstrap (*this); -} - -nano::link::operator nano::bootstrap_hash_or_account const & () const -{ - return nano::to_bootstrap (*this); -} - -nano::root::operator nano::bootstrap_hash_or_account const & () const -{ - return nano::to_bootstrap (*this); -} diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index c3cd1828db..4735af1bba 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -180,17 +180,6 @@ class root final : public hash_or_account nano::block_hash const & previous () const; }; -// Bootstrap hash or account -class bootstrap_hash_or_account final : public hash_or_account -{ -public: - bootstrap_hash_or_account () = default; - bootstrap_hash_or_account (uint64_t value_a); - - bool operator== (nano::bootstrap_hash_or_account const &) const; - bool operator!= (nano::bootstrap_hash_or_account const &) const; -}; - class private_key : public uint256_union { public: diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index dcb4ea154c..f363781e43 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -576,7 +576,7 @@ void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) } } -void nano::bootstrap_attempt::lazy_add (nano::bootstrap_hash_or_account const & hash_or_account_a, bool confirmed_head) +void nano::bootstrap_attempt::lazy_add (nano::hash_or_account const & hash_or_account_a, bool confirmed_head) { // Add only unknown blocks assert (!lazy_mutex.try_lock ()); diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 5f5c3b45da..3ab6a20484 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -84,7 +84,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this lazy_state_backlog; std::unordered_map lazy_balances; std::unordered_set lazy_keys; - std::deque> lazy_pulls; + std::deque> lazy_pulls; std::chrono::steady_clock::time_point last_lazy_flush{ std::chrono::steady_clock::now () }; class account_tag { From 2e17ef1fb0b4de4a2bc9e97b0ae00d54fb173212 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Wed, 25 Sep 2019 16:11:15 +0300 Subject: [PATCH 11/22] Higher retry limit for lazy pulls with a lot of processed blocks --- nano/node/bootstrap/bootstrap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index f363781e43..1bff3c7915 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -535,7 +535,7 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) pulls.push_front (pull); condition.notify_all (); } - else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= nano::bootstrap_limits::bootstrap_lazy_retry_limit)) + else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= nano::bootstrap_limits::bootstrap_lazy_retry_limit + (pull.processed / node->network_params.bootstrap.lazy_max_pull_blocks))) { assert (pull.account_or_head == pull.head); if (!lazy_processed_or_exists (pull.account_or_head)) From c27031c1c9c80d0cc060f89dc66709f7ba2b6062 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Wed, 25 Sep 2019 23:49:13 +0300 Subject: [PATCH 12/22] Improve requeue invalid signature function - add confirmed status to unchecked_info - requeue lazy pull with correct confirmation status --- nano/node/blockprocessor.cpp | 9 +++++---- nano/node/blockprocessor.hpp | 2 +- nano/node/bootstrap/bootstrap.cpp | 6 +++--- nano/node/bootstrap/bootstrap.hpp | 2 +- nano/secure/common.cpp | 5 +++-- nano/secure/common.hpp | 3 ++- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index fbfbdc2660..529ca26d6a 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -226,7 +226,7 @@ void nano::block_processor::verify_state_blocks (nano::unique_lock & else { blocks_filter.erase (filter_item (hashes[i], blocks_signatures[i])); - requeue_invalid (hashes[i]); + requeue_invalid (hashes[i], item); } items.pop_front (); } @@ -478,7 +478,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction { node.logger.try_log (boost::str (boost::format ("Bad signature for: %1%") % hash.to_string ())); } - requeue_invalid (hash); + requeue_invalid (hash, info_a); break; } case nano::process_result::negative_spend: @@ -574,11 +574,12 @@ nano::block_hash nano::block_processor::filter_item (nano::block_hash const & ha return result; } -void nano::block_processor::requeue_invalid (nano::block_hash const & hash_a) +void nano::block_processor::requeue_invalid (nano::block_hash const & hash_a, nano::unchecked_info const & info_a) { + assert (hash_a == info_a.block->hash ()); auto attempt (node.bootstrap_initiator.current_attempt ()); if (attempt != nullptr && attempt->mode == nano::bootstrap_mode::lazy) { - attempt->lazy_requeue (hash_a); + attempt->lazy_requeue (hash_a, info_a.block->previous (), info_a.confirmed); } } diff --git a/nano/node/blockprocessor.hpp b/nano/node/blockprocessor.hpp index 479fde2c25..49fc64dd98 100644 --- a/nano/node/blockprocessor.hpp +++ b/nano/node/blockprocessor.hpp @@ -59,7 +59,7 @@ class block_processor final void verify_state_blocks (nano::unique_lock &, size_t = std::numeric_limits::max ()); void process_batch (nano::unique_lock &); void process_live (nano::block_hash const &, std::shared_ptr, const bool = false); - void requeue_invalid (nano::block_hash const &); + void requeue_invalid (nano::block_hash const &, nano::unchecked_info const &); bool stopped; bool active; bool awaiting_write{ false }; diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 1bff3c7915..e08f4e89ab 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -586,7 +586,7 @@ void nano::bootstrap_attempt::lazy_add (nano::hash_or_account const & hash_or_ac } } -void nano::bootstrap_attempt::lazy_requeue (nano::block_hash const & hash_a) +void nano::bootstrap_attempt::lazy_requeue (nano::block_hash const & hash_a, nano::block_hash const & previous_a, bool confirmed_a) { nano::unique_lock lock (lazy_mutex); // Add only known blocks @@ -595,7 +595,7 @@ void nano::bootstrap_attempt::lazy_requeue (nano::block_hash const & hash_a) { lazy_blocks.erase (existing); lazy_mutex.unlock (); - requeue_pull (nano::pull_info (hash_a, hash_a, nano::block_hash (0), static_cast (1))); + requeue_pull (nano::pull_info (hash_a, hash_a, previous_a, static_cast (1), confirmed_a)); } } @@ -761,7 +761,7 @@ bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr b // Processing new blocks if (lazy_blocks.find (hash) == lazy_blocks.end ()) { - nano::unchecked_info info (block_a, known_account_a, 0, nano::signature_verification::unknown); + nano::unchecked_info info (block_a, known_account_a, 0, nano::signature_verification::unknown, confirmed_head); node->block_processor.add (info); // Search for new dependencies if (!block_a->source ().is_zero () && !node->ledger.block_exists (block_a->source ()) && block_a->source () != node->network_params.ledger.genesis_account) diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 3ab6a20484..929277854c 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -85,7 +85,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this block_a, nano::account const & account_a, uint64_t modified_a, nano::signature_verification verified_a) : +nano::unchecked_info::unchecked_info (std::shared_ptr block_a, nano::account const & account_a, uint64_t modified_a, nano::signature_verification verified_a, bool confirmed_a) : block (block_a), account (account_a), modified (modified_a), -verified (verified_a) +verified (verified_a), +confirmed (confirmed_a) { } diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 567991fda8..a5ca99835b 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -187,7 +187,7 @@ class unchecked_info final { public: unchecked_info () = default; - unchecked_info (std::shared_ptr, nano::account const &, uint64_t, nano::signature_verification = nano::signature_verification::unknown); + unchecked_info (std::shared_ptr, nano::account const &, uint64_t, nano::signature_verification = nano::signature_verification::unknown, bool = false); void serialize (nano::stream &) const; bool deserialize (nano::stream &); std::shared_ptr block; @@ -195,6 +195,7 @@ class unchecked_info final /** Seconds since posix epoch */ uint64_t modified{ 0 }; nano::signature_verification verified{ nano::signature_verification::unknown }; + bool confirmed{ false }; }; class block_info final From ce8e046d9d08f152eda89dbee41fc501ac439e5f Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 26 Sep 2019 00:17:16 +0300 Subject: [PATCH 13/22] Use emplace & emplace_back in bootstrap.cpp --- nano/node/bootstrap/bootstrap.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index e08f4e89ab..5f510faa81 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -561,7 +561,7 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) void nano::bootstrap_attempt::add_bulk_push_target (nano::block_hash const & head, nano::block_hash const & end) { nano::lock_guard lock (mutex); - bulk_push_targets.push_back (std::make_pair (head, end)); + bulk_push_targets.emplace_back (head, end); } void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) @@ -572,7 +572,7 @@ void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) if (lazy_keys.size () < max_keys && lazy_keys.find (hash_a) == lazy_keys.end () && lazy_blocks.find (hash_a) == lazy_blocks.end ()) { lazy_keys.insert (hash_a); - lazy_pulls.push_back (std::make_pair (hash_a, true)); + lazy_pulls.emplace_back (hash_a, true); } } @@ -582,7 +582,7 @@ void nano::bootstrap_attempt::lazy_add (nano::hash_or_account const & hash_or_ac assert (!lazy_mutex.try_lock ()); if (lazy_blocks.find (hash_or_account_a) == lazy_blocks.end ()) { - lazy_pulls.push_back (std::make_pair (hash_or_account_a, confirmed_head)); + lazy_pulls.emplace_back (hash_or_account_a, confirmed_head); } } @@ -611,7 +611,7 @@ void nano::bootstrap_attempt::lazy_pull_flush () if (lazy_blocks.find (pull_start.first) == lazy_blocks.end () && !node->store.block_exists (transaction, pull_start.first)) { assert (node->network_params.bootstrap.lazy_max_pull_blocks <= std::numeric_limits::max ()); - pulls.push_back (nano::pull_info (pull_start.first, pull_start.first, nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second)); + pulls.emplace_back (pull_start.first, pull_start.first, nano::block_hash (0), static_cast (node->network_params.bootstrap.lazy_max_pull_blocks), pull_start.second); } } lazy_pulls.clear (); @@ -784,7 +784,7 @@ bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr b // Adding lazy balances for first processed block in pull if (pull_blocks == 0 && (block_a->type () == nano::block_type::state || block_a->type () == nano::block_type::send)) { - lazy_balances.insert (std::make_pair (hash, block_a->balance ().number ())); + lazy_balances.emplace (hash, block_a->balance ().number ()); } // Clearing lazy balances for previous block if (!block_a->previous ().is_zero () && lazy_balances.find (block_a->previous ()) != lazy_balances.end ()) From 8ca0757aa54ea1a9fab8e3f3add3aac77e33a19e Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 26 Sep 2019 00:23:29 +0300 Subject: [PATCH 14/22] Use lock_guard for regular lazy functions instead of unique_lock --- nano/node/bootstrap/bootstrap.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 5f510faa81..e8622ee610 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -145,7 +145,7 @@ void nano::bootstrap_attempt::request_pull (nano::unique_lock & lock if (mode != nano::bootstrap_mode::legacy) { // Check if pull is obsolete (head was processed) - nano::unique_lock lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); auto transaction (node->store.tx_begin_read ()); while (!pulls.empty () && !pull.head.is_zero () && (lazy_blocks.find (pull.head) != lazy_blocks.end () || node->store.block_exists (transaction, pull.head))) { @@ -566,7 +566,7 @@ void nano::bootstrap_attempt::add_bulk_push_target (nano::block_hash const & hea void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) { - nano::unique_lock lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); // Add start blocks, limit 1024 (32k with disabled legacy bootstrap) size_t max_keys (node->flags.disable_legacy_bootstrap ? 32 * 1024 : 1024); if (lazy_keys.size () < max_keys && lazy_keys.find (hash_a) == lazy_keys.end () && lazy_blocks.find (hash_a) == lazy_blocks.end ()) @@ -588,7 +588,7 @@ void nano::bootstrap_attempt::lazy_add (nano::hash_or_account const & hash_or_ac void nano::bootstrap_attempt::lazy_requeue (nano::block_hash const & hash_a, nano::block_hash const & previous_a, bool confirmed_a) { - nano::unique_lock lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); // Add only known blocks auto existing (lazy_blocks.find (hash_a)); if (existing != lazy_blocks.end ()) @@ -603,7 +603,7 @@ void nano::bootstrap_attempt::lazy_pull_flush () { assert (!mutex.try_lock ()); last_lazy_flush = std::chrono::steady_clock::now (); - nano::unique_lock lazy_lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); auto transaction (node->store.tx_begin_read ()); for (auto & pull_start : lazy_pulls) { @@ -621,7 +621,7 @@ bool nano::bootstrap_attempt::lazy_finished () { bool result (true); auto transaction (node->store.tx_begin_read ()); - nano::unique_lock lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); for (auto it (lazy_keys.begin ()), end (lazy_keys.end ()); it != end && !stopped;) { if (node->store.block_exists (transaction, *it)) @@ -757,7 +757,7 @@ bool nano::bootstrap_attempt::process_block_lazy (std::shared_ptr b { bool stop_pull (false); auto hash (block_a->hash ()); - nano::unique_lock lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); // Processing new blocks if (lazy_blocks.find (hash) == lazy_blocks.end ()) { @@ -887,7 +887,7 @@ void nano::bootstrap_attempt::lazy_block_state_backlog_check (std::shared_ptrstore.tx_begin_read ()); - nano::lock_guard lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); for (auto it (lazy_state_backlog.begin ()), end (lazy_state_backlog.end ()); it != end && !stopped;) { if (node->store.block_exists (transaction, it->first)) @@ -929,7 +929,7 @@ void nano::bootstrap_attempt::lazy_destinations_increment (nano::account const & void nano::bootstrap_attempt::lazy_destinations_flush () { size_t count (0); - nano::unique_lock lazy_lock (lazy_mutex); + nano::lock_guard lazy_lock (lazy_mutex); if (last_lazy_destinations_flush + nano::bootstrap_limits::lazy_destinations_flush_delay_min < std::chrono::steady_clock::now ()) { for (auto it (lazy_destinations.get ().begin ()), end (lazy_destinations.get ().end ()); it != end && count < nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit && !stopped;) From 2003b5a747cdbb6971fc9d6ba7e193b9ed16db90 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 26 Sep 2019 00:24:49 +0300 Subject: [PATCH 15/22] Clarify lazy_destinations_flush_delay_minutes --- nano/node/bootstrap/bootstrap.cpp | 4 ++-- nano/node/bootstrap/bootstrap.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index e8622ee610..2a911fe1fc 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -19,7 +19,7 @@ constexpr double nano::bootstrap_limits::bootstrap_minimum_termination_time_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_max_new_connections; constexpr std::chrono::seconds nano::bootstrap_limits::lazy_flush_delay_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit; -constexpr std::chrono::seconds nano::bootstrap_limits::lazy_destinations_flush_delay_min; +constexpr std::chrono::seconds nano::bootstrap_limits::lazy_destinations_flush_delay_minutes; nano::bootstrap_client::bootstrap_client (std::shared_ptr node_a, std::shared_ptr attempt_a, std::shared_ptr channel_a) : node (node_a), @@ -930,7 +930,7 @@ void nano::bootstrap_attempt::lazy_destinations_flush () { size_t count (0); nano::lock_guard lazy_lock (lazy_mutex); - if (last_lazy_destinations_flush + nano::bootstrap_limits::lazy_destinations_flush_delay_min < std::chrono::steady_clock::now ()) + if (last_lazy_destinations_flush + nano::bootstrap_limits::lazy_destinations_flush_delay_minutes < std::chrono::steady_clock::now ()) { for (auto it (lazy_destinations.get ().begin ()), end (lazy_destinations.get ().end ()); it != end && count < nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit && !stopped;) { diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 929277854c..105e577318 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -237,6 +237,6 @@ class bootstrap_limits final static constexpr unsigned bulk_push_cost_limit = 200; static constexpr std::chrono::seconds lazy_flush_delay_sec = std::chrono::seconds (5); static constexpr unsigned bootstrap_lazy_destinations_request_limit = 200; - static constexpr std::chrono::seconds lazy_destinations_flush_delay_min = std::chrono::minutes (2); + static constexpr std::chrono::seconds lazy_destinations_flush_delay_minutes = std::chrono::minutes (2); }; } From 937a7ad2c199b6314aff73a72fd1a0b317892029 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 26 Sep 2019 13:14:07 +0300 Subject: [PATCH 16/22] lazy_destinations_flush_delay in seconds --- nano/node/bootstrap/bootstrap.cpp | 4 ++-- nano/node/bootstrap/bootstrap.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 2a911fe1fc..20f1296185 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -19,7 +19,7 @@ constexpr double nano::bootstrap_limits::bootstrap_minimum_termination_time_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_max_new_connections; constexpr std::chrono::seconds nano::bootstrap_limits::lazy_flush_delay_sec; constexpr unsigned nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit; -constexpr std::chrono::seconds nano::bootstrap_limits::lazy_destinations_flush_delay_minutes; +constexpr std::chrono::seconds nano::bootstrap_limits::lazy_destinations_flush_delay_sec; nano::bootstrap_client::bootstrap_client (std::shared_ptr node_a, std::shared_ptr attempt_a, std::shared_ptr channel_a) : node (node_a), @@ -930,7 +930,7 @@ void nano::bootstrap_attempt::lazy_destinations_flush () { size_t count (0); nano::lock_guard lazy_lock (lazy_mutex); - if (last_lazy_destinations_flush + nano::bootstrap_limits::lazy_destinations_flush_delay_minutes < std::chrono::steady_clock::now ()) + if (last_lazy_destinations_flush + nano::bootstrap_limits::lazy_destinations_flush_delay_sec < std::chrono::steady_clock::now ()) { for (auto it (lazy_destinations.get ().begin ()), end (lazy_destinations.get ().end ()); it != end && count < nano::bootstrap_limits::bootstrap_lazy_destinations_request_limit && !stopped;) { diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 105e577318..d622dce4a9 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -237,6 +237,6 @@ class bootstrap_limits final static constexpr unsigned bulk_push_cost_limit = 200; static constexpr std::chrono::seconds lazy_flush_delay_sec = std::chrono::seconds (5); static constexpr unsigned bootstrap_lazy_destinations_request_limit = 200; - static constexpr std::chrono::seconds lazy_destinations_flush_delay_minutes = std::chrono::minutes (2); + static constexpr std::chrono::seconds lazy_destinations_flush_delay_sec = std::chrono::minutes (2); }; } From a8474abc336dff4ea27eafc24307cafae1e370c0 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Mon, 7 Oct 2019 00:14:20 +0300 Subject: [PATCH 17/22] Fix updated tests --- nano/core_test/bootstrap.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index b792e9f1a0..f7c3158725 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -141,9 +141,9 @@ TEST (bulk_pull, count_limit) nano::system system (24000, 1); nano::genesis genesis; - auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), nano::test_genesis_key.pub, 1, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); + auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), nano::test_genesis_key.pub, 1, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send1).code); - auto receive1 (std::make_shared (send1->hash (), send1->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + auto receive1 (std::make_shared (send1->hash (), send1->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send1->hash ()))); ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*receive1).code); auto connection (std::make_shared (nullptr, system.nodes[0])); @@ -283,13 +283,13 @@ TEST (bootstrap_processor, pull_diamond) { nano::system system (24000, 1); nano::keypair key; - auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); + auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send1).code); - auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); + auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*open).code); - auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, system.work.generate (open->hash ()))); + auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, *system.work.generate (open->hash ()))); ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*send2).code); - auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send1->hash ()))); ASSERT_EQ (nano::process_result::progress, system.nodes[0]->process (*receive).code); auto node1 (std::make_shared (system.io_ctx, 24002, nano::unique_path (), system.alarm, system.logging, system.work)); ASSERT_FALSE (node1->init_error ()); @@ -312,13 +312,13 @@ TEST (bootstrap_processor, push_diamond) auto wallet1 (node1->wallets.create (100)); wallet1->insert_adhoc (nano::test_genesis_key.prv); wallet1->insert_adhoc (key.prv); - auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); + auto send1 (std::make_shared (system.nodes[0]->latest (nano::test_genesis_key.pub), key.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (system.nodes[0]->latest (nano::test_genesis_key.pub)))); ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); - auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, system.work.generate (key.pub))); + auto open (std::make_shared (send1->hash (), 1, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); - auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, system.work.generate (open->hash ()))); + auto send2 (std::make_shared (open->hash (), nano::test_genesis_key.pub, std::numeric_limits::max () - 100, key.prv, key.pub, *system.work.generate (open->hash ()))); ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); - auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + auto receive (std::make_shared (send1->hash (), send2->hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send1->hash ()))); ASSERT_EQ (nano::process_result::progress, node1->process (*receive).code); node1->bootstrap_initiator.bootstrap (system.nodes[0]->network.endpoint ()); system.deadline_set (10s); @@ -425,13 +425,13 @@ TEST (bootstrap_processor, lazy_unclear_state_link) nano::genesis genesis; nano::keypair key; // Generating test chain - auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (genesis.hash ()))); + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.hash ()))); ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); - auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send1->hash ()))); ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); - auto open (std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, system.work.generate (key.pub))); + auto open (std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); - auto receive (std::make_shared (key.pub, open->hash (), key.pub, 2 * nano::Gxrb_ratio, send2->hash (), key.prv, key.pub, system.work.generate (open->hash ()))); // It is not possible to define this block send/receive status based on previous block (legacy open) + auto receive (std::make_shared (key.pub, open->hash (), key.pub, 2 * nano::Gxrb_ratio, send2->hash (), key.prv, key.pub, *system.work.generate (open->hash ()))); // It is not possible to define this block send/receive status based on previous block (legacy open) ASSERT_EQ (nano::process_result::progress, node1->process (*receive).code); // Start lazy bootstrap with last block in chain known auto node2 = system.add_node (nano::node_config (24001, system.logging), node_flags); @@ -459,13 +459,13 @@ TEST (bootstrap_processor, lazy_destinations) nano::genesis genesis; nano::keypair key1, key2; // Generating test chain - auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (genesis.hash ()))); + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.hash ()))); ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); - auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key2.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (send1->hash ()))); + auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key2.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send1->hash ()))); ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); - auto open (std::make_shared (send1->hash (), key1.pub, key1.pub, key1.prv, key1.pub, system.work.generate (key1.pub))); + auto open (std::make_shared (send1->hash (), key1.pub, key1.pub, key1.prv, key1.pub, *system.work.generate (key1.pub))); ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); - auto state_open (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, system.work.generate (key2.pub))); + auto state_open (std::make_shared (key2.pub, 0, key2.pub, nano::Gxrb_ratio, send2->hash (), key2.prv, key2.pub, *system.work.generate (key2.pub))); ASSERT_EQ (nano::process_result::progress, node1->process (*state_open).code); // Start lazy bootstrap with last block in sender chain auto node2 = system.add_node (nano::node_config (24001, system.logging), node_flags); From 48f86e5e8309e611eaf485b492ec2503e8cd5de6 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Tue, 8 Oct 2019 13:33:34 +0300 Subject: [PATCH 18/22] Test for non existing state blocks links --- nano/core_test/bootstrap.cpp | 36 +++++++++++++++++++++++++++++++ nano/node/bootstrap/bootstrap.cpp | 4 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index f7c3158725..36bb44c90d 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -448,6 +448,42 @@ TEST (bootstrap_processor, lazy_unclear_state_link) ASSERT_TRUE (node2->ledger.block_exists (send2->hash ())); ASSERT_TRUE (node2->ledger.block_exists (open->hash ())); ASSERT_TRUE (node2->ledger.block_exists (receive->hash ())); + ASSERT_EQ (0, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); +} + +TEST (bootstrap_processor, lazy_unclear_state_link_not_existing) +{ + nano::system system; + nano::node_flags node_flags; + node_flags.disable_legacy_bootstrap = true; + auto node1 = system.add_node (nano::node_config (24000, system.logging), node_flags); + nano::genesis genesis; + nano::keypair key, key2; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); + auto send2 (std::make_shared (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - 2 * nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (send1->hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send2).code); + auto open (std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); + ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); + auto send3 (std::make_shared (key.pub, open->hash (), key.pub, 0, key2.pub, key.prv, key.pub, *system.work.generate (open->hash ()))); // It is not possible to define this block send/receive status based on previous block (legacy open) + ASSERT_EQ (nano::process_result::progress, node1->process (*send3).code); + // Start lazy bootstrap with last block in chain known + auto node2 = system.add_node (nano::node_config (24001, system.logging), node_flags); + node2->network.udp_channels.insert (node1->network.endpoint (), node1->network_params.protocol.protocol_version); + node2->bootstrap_initiator.bootstrap_lazy (send3->hash ()); + // Check processed blocks + system.deadline_set (15s); + while (node2->bootstrap_initiator.in_progress ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + node2->block_processor.flush (); + ASSERT_TRUE (node2->ledger.block_exists (send1->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (send2->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (open->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (send3->hash ())); + ASSERT_EQ (1, node2->stats.count (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in)); } TEST (bootstrap_processor, lazy_destinations) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 20f1296185..c7d36c9841 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -529,13 +529,13 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) { auto pull (pull_a); ++pull.attempts; - if (pull.attempts < (nano::bootstrap_limits::bootstrap_frontier_retry_limit + (pull.processed / 10000))) + if (pull.attempts < (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_frontier_retry_limit : nano::bootstrap_limits::bootstrap_frontier_retry_limit / 8 + (pull.processed / 10000))) { nano::lock_guard lock (mutex); pulls.push_front (pull); condition.notify_all (); } - else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= nano::bootstrap_limits::bootstrap_lazy_retry_limit + (pull.processed / node->network_params.bootstrap.lazy_max_pull_blocks))) + else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_lazy_retry_limit : nano::bootstrap_limits::bootstrap_frontier_retry_limit / 8 + (pull.processed / node->network_params.bootstrap.lazy_max_pull_blocks)))) { assert (pull.account_or_head == pull.head); if (!lazy_processed_or_exists (pull.account_or_head)) From a6e7cd3061642bd5430fc73ee8202207e8c5bae8 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Tue, 8 Oct 2019 14:07:07 +0300 Subject: [PATCH 19/22] List of lazy undefined links to prevent multiple requests for same account --- nano/node/bootstrap/bootstrap.cpp | 3 ++- nano/node/bootstrap/bootstrap.hpp | 1 + nano/node/json_handler.cpp | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index c7d36c9841..b641181ded 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -876,9 +876,10 @@ void nano::bootstrap_attempt::lazy_block_state_backlog_check (std::shared_ptr lazy_blocks; std::unordered_map lazy_state_backlog; + std::unordered_set lazy_undefined_links; std::unordered_map lazy_balances; std::unordered_set lazy_keys; std::deque> lazy_pulls; diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 14a628c87a..5b41254c28 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1696,6 +1696,7 @@ void nano::json_handler::bootstrap_status () response_l.put ("lazy_state_backlog", std::to_string (attempt->lazy_state_backlog.size ())); response_l.put ("lazy_balances", std::to_string (attempt->lazy_balances.size ())); response_l.put ("lazy_destinations", std::to_string (attempt->lazy_destinations.size ())); + response_l.put ("lazy_undefined_links", std::to_string (attempt->lazy_undefined_links.size ())); response_l.put ("lazy_pulls", std::to_string (attempt->lazy_pulls.size ())); response_l.put ("lazy_keys", std::to_string (attempt->lazy_keys.size ())); if (!attempt->lazy_keys.empty ()) From 7f28c294d0961fcc45bde508dd0599e9ee956782 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Wed, 9 Oct 2019 00:10:59 +0300 Subject: [PATCH 20/22] Start lazy bootstrap for not found RPC accounts --- nano/node/bootstrap/bootstrap.cpp | 16 ++-- nano/node/bootstrap/bootstrap.hpp | 4 +- nano/node/bootstrap/bootstrap_server.cpp | 38 +++++----- nano/node/bootstrap/bootstrap_server.hpp | 3 +- nano/node/json_handler.cpp | 95 ++++++++++-------------- nano/node/json_handler.hpp | 1 + nano/rpc_test/rpc.cpp | 48 ++++++++++++ 7 files changed, 117 insertions(+), 88 deletions(-) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index b641181ded..b46d98d73e 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -529,13 +529,13 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) { auto pull (pull_a); ++pull.attempts; - if (pull.attempts < (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_frontier_retry_limit : nano::bootstrap_limits::bootstrap_frontier_retry_limit / 8 + (pull.processed / 10000))) + if (pull.attempts < (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_frontier_retry_limit : (nano::bootstrap_limits::bootstrap_frontier_retry_limit / 8) + (pull.processed / 10000))) { nano::lock_guard lock (mutex); pulls.push_front (pull); condition.notify_all (); } - else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_lazy_retry_limit : nano::bootstrap_limits::bootstrap_frontier_retry_limit / 8 + (pull.processed / node->network_params.bootstrap.lazy_max_pull_blocks)))) + else if (mode == nano::bootstrap_mode::lazy && (pull.confirmed_head || pull.attempts <= (!node->network_params.network.is_test_network () ? nano::bootstrap_limits::bootstrap_lazy_retry_limit : (nano::bootstrap_limits::bootstrap_frontier_retry_limit / 8) + (pull.processed / node->network_params.bootstrap.lazy_max_pull_blocks)))) { assert (pull.account_or_head == pull.head); if (!lazy_processed_or_exists (pull.account_or_head)) @@ -564,15 +564,15 @@ void nano::bootstrap_attempt::add_bulk_push_target (nano::block_hash const & hea bulk_push_targets.emplace_back (head, end); } -void nano::bootstrap_attempt::lazy_start (nano::block_hash const & hash_a) +void nano::bootstrap_attempt::lazy_start (nano::hash_or_account const & hash_or_account_a, bool confirmed) { nano::lock_guard lazy_lock (lazy_mutex); // Add start blocks, limit 1024 (32k with disabled legacy bootstrap) size_t max_keys (node->flags.disable_legacy_bootstrap ? 32 * 1024 : 1024); - if (lazy_keys.size () < max_keys && lazy_keys.find (hash_a) == lazy_keys.end () && lazy_blocks.find (hash_a) == lazy_blocks.end ()) + if (lazy_keys.size () < max_keys && lazy_keys.find (hash_or_account_a) == lazy_keys.end () && lazy_blocks.find (hash_or_account_a) == lazy_blocks.end ()) { - lazy_keys.insert (hash_a); - lazy_pulls.emplace_back (hash_a, true); + lazy_keys.insert (hash_or_account_a); + lazy_pulls.emplace_back (hash_or_account_a, confirmed); } } @@ -1091,7 +1091,7 @@ void nano::bootstrap_initiator::bootstrap (nano::endpoint const & endpoint_a, bo } } -void nano::bootstrap_initiator::bootstrap_lazy (nano::block_hash const & hash_a, bool force) +void nano::bootstrap_initiator::bootstrap_lazy (nano::hash_or_account const & hash_or_account_a, bool force, bool confirmed) { { nano::unique_lock lock (mutex); @@ -1110,7 +1110,7 @@ void nano::bootstrap_initiator::bootstrap_lazy (nano::block_hash const & hash_a, { attempt = std::make_shared (node.shared (), nano::bootstrap_mode::lazy); } - attempt->lazy_start (hash_a); + attempt->lazy_start (hash_or_account_a, confirmed); } condition.notify_all (); } diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 8e3f01882f..900d2f17e7 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -83,7 +83,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this, nano::account const &, uint64_t, bool, bool); /** Lazy bootstrap */ void lazy_run (); - void lazy_start (nano::block_hash const &); + void lazy_start (nano::hash_or_account const &, bool confirmed = true); void lazy_add (nano::hash_or_account const &, bool = true); void lazy_requeue (nano::block_hash const &, nano::block_hash const &, bool); bool lazy_finished (); @@ -199,7 +199,7 @@ class bootstrap_initiator final ~bootstrap_initiator (); void bootstrap (nano::endpoint const &, bool add_to_peers = true); void bootstrap (); - void bootstrap_lazy (nano::block_hash const &, bool = false); + void bootstrap_lazy (nano::hash_or_account const &, bool force = false, bool confirmed = true); void bootstrap_wallet (std::deque &); void run_bootstrap (); void notify_listeners (bool); diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index 8003ea163c..957de68fa8 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -80,6 +80,14 @@ std::unique_ptr collect_seq_con_info (bootstrap_listener } } +nano::bootstrap_server::bootstrap_server (std::shared_ptr socket_a, std::shared_ptr node_a) : +receive_buffer (std::make_shared> ()), +socket (socket_a), +node (node_a) +{ + receive_buffer->resize (1024); +} + nano::bootstrap_server::~bootstrap_server () { if (node->config.logging.bulk_pull_logging ()) @@ -117,14 +125,6 @@ void nano::bootstrap_server::stop () } } -nano::bootstrap_server::bootstrap_server (std::shared_ptr socket_a, std::shared_ptr node_a) : -receive_buffer (std::make_shared> ()), -socket (socket_a), -node (node_a) -{ - receive_buffer->resize (512); -} - void nano::bootstrap_server::receive () { // Increase timeout to receive TCP header (idle server socket) @@ -153,12 +153,12 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co nano::message_header header (error, type_stream); if (!error) { + auto this_l (shared_from_this ()); switch (header.type) { case nano::message_type::bulk_pull: { node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull, nano::stat::dir::in); - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_bulk_pull_action (ec, size_a, header); }); @@ -167,7 +167,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co case nano::message_type::bulk_pull_account: { node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_account, nano::stat::dir::in); - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_bulk_pull_account_action (ec, size_a, header); }); @@ -176,7 +175,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co case nano::message_type::frontier_req: { node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::frontier_req, nano::stat::dir::in); - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_frontier_req_action (ec, size_a, header); }); @@ -193,7 +191,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co } case nano::message_type::keepalive: { - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_keepalive_action (ec, size_a, header); }); @@ -201,7 +198,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co } case nano::message_type::publish: { - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_publish_action (ec, size_a, header); }); @@ -209,7 +205,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co } case nano::message_type::confirm_ack: { - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_confirm_ack_action (ec, size_a, header); }); @@ -217,7 +212,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co } case nano::message_type::confirm_req: { - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_confirm_req_action (ec, size_a, header); }); @@ -225,7 +219,6 @@ void nano::bootstrap_server::receive_header_action (boost::system::error_code co } case nano::message_type::node_id_handshake: { - auto this_l (shared_from_this ()); socket->async_read (receive_buffer, header.payload_length_bytes (), [this_l, header](boost::system::error_code const & ec, size_t size_a) { this_l->receive_node_id_handshake_action (ec, size_a, header); }); @@ -334,7 +327,7 @@ void nano::bootstrap_server::receive_keepalive_action (boost::system::error_code std::unique_ptr request (new nano::keepalive (error, stream, header_a)); if (!error) { - if (type == nano::bootstrap_server_type::realtime || type == nano::bootstrap_server_type::realtime_response_server) + if (is_realtime_connection ()) { add_request (std::unique_ptr (request.release ())); } @@ -359,7 +352,7 @@ void nano::bootstrap_server::receive_publish_action (boost::system::error_code c std::unique_ptr request (new nano::publish (error, stream, header_a)); if (!error) { - if (type == nano::bootstrap_server_type::realtime || type == nano::bootstrap_server_type::realtime_response_server) + if (is_realtime_connection ()) { add_request (std::unique_ptr (request.release ())); } @@ -384,7 +377,7 @@ void nano::bootstrap_server::receive_confirm_req_action (boost::system::error_co std::unique_ptr request (new nano::confirm_req (error, stream, header_a)); if (!error) { - if (type == nano::bootstrap_server_type::realtime || type == nano::bootstrap_server_type::realtime_response_server) + if (is_realtime_connection ()) { add_request (std::unique_ptr (request.release ())); } @@ -406,7 +399,7 @@ void nano::bootstrap_server::receive_confirm_ack_action (boost::system::error_co std::unique_ptr request (new nano::confirm_ack (error, stream, header_a)); if (!error) { - if (type == nano::bootstrap_server_type::realtime || type == nano::bootstrap_server_type::realtime_response_server) + if (is_realtime_connection ()) { add_request (std::unique_ptr (request.release ())); } @@ -650,3 +643,8 @@ bool nano::bootstrap_server::is_bootstrap_connection () } return type == nano::bootstrap_server_type::bootstrap; } + +bool nano::bootstrap_server::is_realtime_connection () +{ + return type == nano::bootstrap_server_type::realtime || type == nano::bootstrap_server_type::realtime_response_server; +} diff --git a/nano/node/bootstrap/bootstrap_server.hpp b/nano/node/bootstrap/bootstrap_server.hpp index 4ce39c1817..03a50dd195 100644 --- a/nano/node/bootstrap/bootstrap_server.hpp +++ b/nano/node/bootstrap/bootstrap_server.hpp @@ -60,9 +60,10 @@ class bootstrap_server final : public std::enable_shared_from_this); void finish_request (); void finish_request_async (); - void run_next (); void timeout (); + void run_next (); bool is_bootstrap_connection (); + bool is_realtime_connection (); std::shared_ptr> receive_buffer; std::shared_ptr socket; std::shared_ptr node; diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 5b41254c28..442fb00639 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -228,6 +228,20 @@ nano::account nano::json_handler::account_impl (std::string account_text, std::e return result; } +nano::account_info nano::json_handler::account_info_impl (nano::transaction const & transaction_a, nano::account const & account_a) +{ + nano::account_info result; + if (!ec) + { + if (node.store.account_get (transaction_a, account_a, result)) + { + ec = nano::error_common::account_not_found; + node.bootstrap_initiator.bootstrap_lazy (account_a, false, false); + } + } + return result; +} + nano::amount nano::json_handler::amount_impl () { nano::amount result (0); @@ -442,15 +456,11 @@ void nano::json_handler::account_block_count () if (!ec) { auto transaction (node.store.tx_begin_read ()); - nano::account_info info; - if (!node.store.account_get (transaction, account, info)) + auto info (account_info_impl (transaction, account)); + if (!ec) { response_l.put ("block_count", std::to_string (info.block_count)); } - else - { - ec = nano::error_common::account_not_found; - } } response_errors (); } @@ -522,10 +532,13 @@ void nano::json_handler::account_info () const bool weight = request.get ("weight", false); const bool pending = request.get ("pending", false); auto transaction (node.store.tx_begin_read ()); - nano::account_info info; + auto info (account_info_impl (transaction, account)); uint64_t confirmation_height; - auto error = node.store.account_get (transaction, account, info) | node.store.confirmation_height_get (transaction, account, confirmation_height); - if (!error) + if (node.store.confirmation_height_get (transaction, account, confirmation_height)) + { + ec = nano::error_common::account_not_found; + } + if (!ec) { response_l.put ("frontier", info.head.to_string ()); response_l.put ("open_block", info.open_block.to_string ()); @@ -552,10 +565,6 @@ void nano::json_handler::account_info () response_l.put ("pending", account_pending.convert_to ()); } } - else - { - ec = nano::error_common::account_not_found; - } } response_errors (); } @@ -655,15 +664,11 @@ void nano::json_handler::account_representative () if (!ec) { auto transaction (node.store.tx_begin_read ()); - nano::account_info info; - if (!node.store.account_get (transaction, account, info)) + auto info (account_info_impl (transaction, account)); + if (!ec) { response_l.put ("representative", info.representative.to_account ()); } - else - { - ec = nano::error_common::account_not_found; - } } response_errors (); } @@ -688,19 +693,15 @@ void nano::json_handler::account_representative_set () rpc_l->wallet_account_impl (transaction, wallet, account); if (!rpc_l->ec) { - nano::account_info info; auto block_transaction (rpc_l->node.store.tx_begin_read ()); - if (!rpc_l->node.store.account_get (block_transaction, account, info)) + auto info (rpc_l->account_info_impl (block_transaction, account)); + if (!rpc_l->ec) { if (nano::work_validate (info.head, work)) { rpc_l->ec = nano::error_common::invalid_work; } } - else - { - rpc_l->ec = nano::error_common::account_not_found; - } } } else if (!rpc_l->ec) // work == 0 @@ -2288,15 +2289,11 @@ void nano::json_handler::account_history () { if (reverse) { - nano::account_info info; - if (!node.store.account_get (transaction, account, info)) + auto info (account_info_impl (transaction, account)); + if (!ec) { hash = info.open_block; } - else - { - ec = nano::error_common::account_not_found; - } } else { @@ -3448,35 +3445,19 @@ void nano::json_handler::send () { auto transaction (node.wallets.tx_begin_read ()); auto block_transaction (node.store.tx_begin_read ()); - if (wallet->store.valid_password (transaction)) + wallet_locked_impl (transaction, wallet); + wallet_account_impl (transaction, wallet, source); + auto info (account_info_impl (block_transaction, source)); + if (!ec) { - if (wallet->store.find (transaction, source) != wallet->store.end ()) - { - nano::account_info info; - if (!node.store.account_get (block_transaction, source, info)) - { - balance = (info.balance).number (); - } - else - { - ec = nano::error_common::account_not_found; - } - if (!ec && work) - { - if (nano::work_validate (info.head, work)) - { - ec = nano::error_common::invalid_work; - } - } - } - else - { - ec = nano::error_common::account_not_found_wallet; - } + balance = (info.balance).number (); } - else + if (!ec && work) { - ec = nano::error_common::wallet_locked; + if (nano::work_validate (info.head, work)) + { + ec = nano::error_common::invalid_work; + } } } if (!ec) diff --git a/nano/node/json_handler.hpp b/nano/node/json_handler.hpp index 11a6c40dec..4412beed42 100644 --- a/nano/node/json_handler.hpp +++ b/nano/node/json_handler.hpp @@ -144,6 +144,7 @@ class json_handler : public std::enable_shared_from_this bool wallet_locked_impl (nano::transaction const &, std::shared_ptr); bool wallet_account_impl (nano::transaction const &, std::shared_ptr, nano::account const &); nano::account account_impl (std::string = "", std::error_code = nano::error_common::bad_account_number); + nano::account_info account_info_impl (nano::transaction const &, nano::account const &); nano::amount amount_impl (); std::shared_ptr block_impl (bool = true); std::shared_ptr block_json_impl (bool = true); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 6a70802ea8..74f3161558 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -7147,3 +7147,51 @@ TEST (rpc, deprecated_account_format) boost::optional deprecated_account_format2 (response2.json.get_optional ("deprecated_account_format")); ASSERT_TRUE (deprecated_account_format2.is_initialized ()); } + +TEST (rpc, account_lazy_start) +{ + nano::system system; + nano::node_flags node_flags; + node_flags.disable_legacy_bootstrap = true; + auto node1 = system.add_node (nano::node_config (24000, system.logging), node_flags); + nano::genesis genesis; + nano::keypair key; + // Generating test chain + auto send1 (std::make_shared (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.hash ()))); + ASSERT_EQ (nano::process_result::progress, node1->process (*send1).code); + auto open (std::make_shared (send1->hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub))); + ASSERT_EQ (nano::process_result::progress, node1->process (*open).code); + + // Start lazy bootstrap with account + auto node2 = system.add_node (nano::node_config (24001, system.logging), node_flags); + node2->network.udp_channels.insert (node1->network.endpoint (), node1->network_params.protocol.protocol_version); + enable_ipc_transport_tcp (node2->config.ipc_config.transport_tcp); + nano::node_rpc_config node_rpc_config; + nano::ipc::ipc_server ipc_server (*node2, node_rpc_config); + nano::rpc_config rpc_config (true); + nano::ipc_rpc_processor ipc_rpc_processor (system.io_ctx, rpc_config); + nano::rpc rpc (system.io_ctx, rpc_config, ipc_rpc_processor); + rpc.start (); + boost::property_tree::ptree request; + request.put ("action", "account_info"); + request.put ("account", key.pub.to_account ()); + test_response response (request, rpc.config.port, system.io_ctx); + system.deadline_set (5s); + while (response.status == 0) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (200, response.status); + boost::optional account_error (response.json.get_optional ("error")); + ASSERT_TRUE (account_error.is_initialized ()); + + // Check processed blocks + system.deadline_set (10s); + while (node2->bootstrap_initiator.in_progress ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + node2->block_processor.flush (); + ASSERT_TRUE (node2->ledger.block_exists (send1->hash ())); + ASSERT_TRUE (node2->ledger.block_exists (open->hash ())); +} From 1e5af7a85697abe67c9dd4b5b0e52693f49713a4 Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 10 Oct 2019 11:19:16 +0300 Subject: [PATCH 21/22] Improve lazy_backlog_cleanup (), requeue lazy pulls if expected blocks were processed --- nano/node/bootstrap/bootstrap.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index b46d98d73e..e1594720fb 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -555,6 +555,12 @@ void nano::bootstrap_attempt::requeue_pull (nano::pull_info const & pull_a) node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_failed_account, nano::stat::dir::in); node->bootstrap_initiator.cache.add (pull); + if (mode == nano::bootstrap_mode::lazy && pull.processed > 0) + { + assert (pull.account_or_head == pull.head); + nano::lock_guard lazy_lock (lazy_mutex); + lazy_add (pull.account_or_head, pull.confirmed_head); + } } } @@ -906,6 +912,7 @@ void nano::bootstrap_attempt::lazy_backlog_cleanup () } else { + lazy_add (it->first, it->second.confirmed); ++it; } } From 99a63d27e3d1aa4dd687102beafbf39f0027413d Mon Sep 17 00:00:00 2001 From: Sergey Kroshnin Date: Thu, 10 Oct 2019 13:33:59 +0300 Subject: [PATCH 22/22] Add bootstrap attempt duration --- nano/node/bootstrap/bootstrap.hpp | 1 + nano/node/json_handler.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 900d2f17e7..e6f4883535 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -120,6 +120,7 @@ class bootstrap_attempt final : public std::enable_shared_from_this runs_count; std::vector> bulk_push_targets; std::atomic stopped; + std::chrono::steady_clock::time_point attempt_start{ std::chrono::steady_clock::now () }; nano::bootstrap_mode mode; std::mutex mutex; nano::condition_variable condition; diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 1faaf66a1d..e56b592edf 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1704,6 +1704,7 @@ void nano::json_handler::bootstrap_status () { response_l.put ("lazy_key_1", (*(attempt->lazy_keys.begin ())).to_string ()); } + response_l.put ("duration", std::chrono::duration_cast (std::chrono::steady_clock::now () - attempt->attempt_start).count ()); } else {