From 49fcda81a2b947d6baf9134cac377708d5385df7 Mon Sep 17 00:00:00 2001 From: wezrule Date: Tue, 14 Jan 2020 14:29:26 +0000 Subject: [PATCH 1/6] Upgrade confirmation height table to include cemented frontier --- nano/core_test/block_store.cpp | 219 +++++++++------ nano/core_test/confirmation_height.cpp | 285 ++++++++++---------- nano/core_test/ledger.cpp | 24 +- nano/core_test/versioning.cpp | 18 +- nano/core_test/wallets.cpp | 1 + nano/nano_node/entry.cpp | 11 +- nano/node/active_transactions.cpp | 24 +- nano/node/blockprocessor.cpp | 2 +- nano/node/cli.cpp | 10 +- nano/node/confirmation_height_processor.cpp | 12 +- nano/node/confirmation_solicitor.cpp | 1 - nano/node/confirmation_solicitor.hpp | 10 +- nano/node/json_handler.cpp | 9 +- nano/node/lmdb/lmdb.cpp | 67 ++++- nano/node/lmdb/lmdb.hpp | 4 +- nano/node/node.cpp | 38 +-- nano/rpc_test/rpc.cpp | 20 +- nano/secure/blockstore.hpp | 32 ++- nano/secure/blockstore_partial.hpp | 37 +-- nano/secure/common.cpp | 27 ++ nano/secure/common.hpp | 12 + nano/secure/ledger.cpp | 16 +- nano/secure/ledger.hpp | 2 +- nano/slow_test/node.cpp | 17 +- 24 files changed, 553 insertions(+), 345 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 9456501894..bcc1add1e0 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -29,6 +29,7 @@ namespace void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, nano::block_hash const & rep_block); void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height, nano::block_hash const & rep_block); void modify_genesis_account_info_to_v5 (nano::mdb_store & store, nano::transaction const & transaction_a); +void modify_confirmation_height_to_v15 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height); void write_sideband_v12 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block & block_a, nano::block_hash const & successor_a, MDB_dbi db_a); void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a); } @@ -288,9 +289,10 @@ TEST (block_store, genesis) ASSERT_LE (info.modified, nano::seconds_since_epoch ()); ASSERT_EQ (info.block_count, 1); // Genesis block should be confirmed by default - uint64_t confirmation_height; - ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (confirmation_height, 1); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 1); + ASSERT_EQ (confirmation_height_info.frontier, hash); auto test_pub_text (nano::test_genesis_key.pub.to_string ()); auto test_pub_account (nano::test_genesis_key.pub.to_account ()); auto test_prv_text (nano::test_genesis_key.prv.data.to_string ()); @@ -472,7 +474,7 @@ TEST (block_store, frontier_retrieval) nano::account account1 (0); nano::account_info info1 (0, 0, 0, 0, 0, 0, nano::epoch::epoch_0); auto transaction (store->tx_begin_write ()); - store->confirmation_height_put (transaction, account1, 0); + store->confirmation_height_put (transaction, account1, { 0, nano::block_hash (0) }); store->account_put (transaction, account1, info1); nano::account_info info2; store->account_get (transaction, account1, info2); @@ -487,7 +489,7 @@ TEST (block_store, one_account) nano::account account (0); nano::block_hash hash (0); auto transaction (store->tx_begin_write ()); - store->confirmation_height_put (transaction, account, 20); + store->confirmation_height_put (transaction, account, { 20, nano::block_hash (15) }); store->account_put (transaction, account, { hash, account, hash, 42, 100, 200, nano::epoch::epoch_0 }); auto begin (store->latest_begin (transaction)); auto end (store->latest_end ()); @@ -498,9 +500,10 @@ TEST (block_store, one_account) ASSERT_EQ (42, info.balance.number ()); ASSERT_EQ (100, info.modified); ASSERT_EQ (200, info.block_count); - uint64_t confirmation_height; - ASSERT_FALSE (store->confirmation_height_get (transaction, account, confirmation_height)); - ASSERT_EQ (20, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store->confirmation_height_get (transaction, account, confirmation_height_info)); + ASSERT_EQ (20, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (15), confirmation_height_info.frontier); ++begin; ASSERT_EQ (end, begin); } @@ -538,9 +541,9 @@ TEST (block_store, two_account) nano::account account2 (3); nano::block_hash hash2 (4); auto transaction (store->tx_begin_write ()); - store->confirmation_height_put (transaction, account1, 20); + store->confirmation_height_put (transaction, account1, { 20, nano::block_hash (10) }); store->account_put (transaction, account1, { hash1, account1, hash1, 42, 100, 300, nano::epoch::epoch_0 }); - store->confirmation_height_put (transaction, account2, 30); + store->confirmation_height_put (transaction, account2, { 30, nano::block_hash (20) }); store->account_put (transaction, account2, { hash2, account2, hash2, 84, 200, 400, nano::epoch::epoch_0 }); auto begin (store->latest_begin (transaction)); auto end (store->latest_end ()); @@ -551,9 +554,10 @@ TEST (block_store, two_account) ASSERT_EQ (42, info1.balance.number ()); ASSERT_EQ (100, info1.modified); ASSERT_EQ (300, info1.block_count); - uint64_t confirmation_height; - ASSERT_FALSE (store->confirmation_height_get (transaction, account1, confirmation_height)); - ASSERT_EQ (20, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store->confirmation_height_get (transaction, account1, confirmation_height_info)); + ASSERT_EQ (20, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (10), confirmation_height_info.frontier); ++begin; ASSERT_NE (end, begin); ASSERT_EQ (account2, nano::account (begin->first)); @@ -562,8 +566,9 @@ TEST (block_store, two_account) ASSERT_EQ (84, info2.balance.number ()); ASSERT_EQ (200, info2.modified); ASSERT_EQ (400, info2.block_count); - ASSERT_FALSE (store->confirmation_height_get (transaction, account2, confirmation_height)); - ASSERT_EQ (30, confirmation_height); + ASSERT_FALSE (store->confirmation_height_get (transaction, account2, confirmation_height_info)); + ASSERT_EQ (30, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (20), confirmation_height_info.frontier); ++begin; ASSERT_EQ (end, begin); } @@ -578,9 +583,9 @@ TEST (block_store, latest_find) nano::account account2 (3); nano::block_hash hash2 (4); auto transaction (store->tx_begin_write ()); - store->confirmation_height_put (transaction, account1, 0); + store->confirmation_height_put (transaction, account1, { 0, nano::block_hash (0) }); store->account_put (transaction, account1, { hash1, account1, hash1, 100, 0, 300, nano::epoch::epoch_0 }); - store->confirmation_height_put (transaction, account2, 0); + store->confirmation_height_put (transaction, account2, { 0, nano::block_hash (0) }); store->account_put (transaction, account2, { hash2, account2, hash2, 200, 0, 400, nano::epoch::epoch_0 }); auto first (store->latest_begin (transaction)); auto second (store->latest_begin (transaction)); @@ -649,7 +654,7 @@ TEST (block_store, latest_exists) nano::account two (2); nano::account_info info; auto transaction (store->tx_begin_write ()); - store->confirmation_height_put (transaction, two, 0); + store->confirmation_height_put (transaction, two, { 0, nano::block_hash (0) }); store->account_put (transaction, two, info); nano::account one (1); ASSERT_FALSE (store->account_exists (transaction, one)); @@ -667,7 +672,7 @@ TEST (block_store, large_iteration) nano::account account; nano::random_pool::generate_block (account.bytes.data (), account.bytes.size ()); accounts1.insert (account); - store->confirmation_height_put (transaction, account, 0); + store->confirmation_height_put (transaction, account, { 0, nano::block_hash (0) }); store->account_put (transaction, account, nano::account_info ()); } std::unordered_set accounts2; @@ -741,7 +746,7 @@ TEST (block_store, account_count) auto transaction (store->tx_begin_write ()); ASSERT_EQ (0, store->account_count (transaction)); nano::account account (200); - store->confirmation_height_put (transaction, account, 0); + store->confirmation_height_put (transaction, account, { 0, nano::block_hash (0) }); store->account_put (transaction, account, nano::account_info ()); } auto transaction (store->tx_begin_read ()); @@ -824,6 +829,7 @@ TEST (mdb_block_store, upgrade_v2_v3) auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (sizeof (info_old), &info_old), 0)); (void)status; assert (status == 0); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -901,6 +907,7 @@ TEST (mdb_block_store, upgrade_v4_v5) // The pending send needs to be the correct version auto status (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (key0.pub, block0.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); ASSERT_EQ (status, MDB_SUCCESS); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -939,6 +946,7 @@ TEST (mdb_block_store, upgrade_v5_v6) store.initialize (transaction, genesis, ledger_cache); store.version_put (transaction, 5); modify_genesis_account_info_to_v5 (store, transaction); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -966,6 +974,7 @@ TEST (mdb_block_store, upgrade_v6_v7) store.unchecked_put (transaction, send1->hash (), send1); store.flush (transaction); ASSERT_NE (store.unchecked_end (), store.unchecked_begin (transaction)); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1165,6 +1174,7 @@ TEST (mdb_block_store, upgrade_sideband_genesis) auto genesis_block2 (store.block_get_v14 (transaction, genesis.hash (), &sideband1)); ASSERT_NE (nullptr, genesis_block); ASSERT_EQ (0, sideband1.height); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1203,6 +1213,7 @@ TEST (mdb_block_store, upgrade_sideband_two_blocks) modify_account_info_to_v13 (store, transaction, nano::genesis_account, hash2); auto status (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, block.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); ASSERT_EQ (status, MDB_SUCCESS); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1250,6 +1261,8 @@ TEST (mdb_block_store, upgrade_sideband_two_accounts) write_sideband_v12 (store, transaction, block2, 0, store.state_blocks_v0); modify_account_info_to_v13 (store, transaction, nano::genesis_account, hash2); modify_account_info_to_v13 (store, transaction, block2.account (), hash3); + store.confirmation_height_del (transaction, nano::genesis_account); + store.confirmation_height_del (transaction, key.pub); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1338,6 +1351,7 @@ TEST (mdb_block_store, upgrade_sideband_epoch) ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); modify_account_info_to_v13 (store, transaction, nano::genesis_account, hash2); store.account_del (transaction, nano::genesis_account); + store.confirmation_height_del (transaction, nano::genesis_account); } nano::logger_mt logger; nano::mdb_store store (logger, path); @@ -1558,18 +1572,15 @@ TEST (block_store, online_weight) TEST (mdb_block_store, upgrade_v13_v14) { auto path (nano::unique_path ()); + nano::genesis genesis; { nano::logger_mt logger; - nano::genesis genesis; nano::mdb_store store (logger, path); auto transaction (store.tx_begin_write ()); nano::ledger_cache ledger_cache; store.initialize (transaction, genesis, ledger_cache); nano::account_info account_info; ASSERT_FALSE (store.account_get (transaction, nano::genesis_account, account_info)); - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (confirmation_height, 1); store.version_put (transaction, 13); modify_account_info_to_v13 (store, transaction, nano::genesis_account, genesis.open->hash ()); @@ -1578,6 +1589,7 @@ TEST (mdb_block_store, upgrade_v13_v14) ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (nano::genesis_account), value)); nano::account_info_v14 info; ASSERT_NE (value.size (), info.db_size ()); + store.confirmation_height_del (transaction, nano::genesis_account); } // Now do the upgrade @@ -1594,9 +1606,10 @@ TEST (mdb_block_store, upgrade_v13_v14) ASSERT_EQ (value.size (), info.db_size ()); // Confirmation height should exist and be correct - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (confirmation_height, 1); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 1); + ASSERT_EQ (confirmation_height_info.frontier, genesis.hash ()); // Test deleting node ID nano::uint256_union node_id_mdb_key (3); @@ -1625,9 +1638,10 @@ TEST (mdb_block_store, upgrade_v14_v15) store.initialize (transaction, genesis, ledger.cache); nano::account_info account_info; ASSERT_FALSE (store.account_get (transaction, nano::genesis_account, account_info)); - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (confirmation_height, 1); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 1); + ASSERT_EQ (confirmation_height_info.frontier, genesis.hash ()); // These databases get remove after an upgrade, so readd them ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1)); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); @@ -1638,7 +1652,7 @@ TEST (mdb_block_store, upgrade_v14_v15) // Lower the database to the previous version store.version_put (transaction, 14); store.confirmation_height_del (transaction, nano::genesis_account); - modify_account_info_to_v14 (store, transaction, nano::genesis_account, confirmation_height, state_send.hash ()); + modify_account_info_to_v14 (store, transaction, nano::genesis_account, confirmation_height_info.height, state_send.hash ()); store.pending_del (transaction, nano::pending_key (nano::genesis_account, state_send.hash ())); @@ -1661,7 +1675,7 @@ TEST (mdb_block_store, upgrade_v14_v15) store.account_del (transaction, nano::genesis_account); // Confirmation height for the account should be deleted - ASSERT_TRUE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); + ASSERT_TRUE (mdb_get (store.env.tx (transaction), store.confirmation_height, nano::mdb_val (nano::genesis_account), value)); } // Now do the upgrade @@ -1678,9 +1692,10 @@ TEST (mdb_block_store, upgrade_v14_v15) ASSERT_EQ (value.size (), info.db_size ()); // Confirmation height should exist - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (confirmation_height, 1); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 1); + ASSERT_EQ (confirmation_height_info.frontier, genesis.hash ()); // accounts_v1, state_blocks_v1 & pending_v1 tables should be deleted auto error_get_accounts_v1 (mdb_get (store.env.tx (transaction), store.accounts_v1, nano::mdb_val (nano::genesis_account), value)); @@ -1712,38 +1727,60 @@ TEST (mdb_block_store, upgrade_v14_v15) TEST (mdb_block_store, upgrade_v15_v16) { - auto path (nano::unique_path ()); - nano::mdb_val value; - { - nano::genesis genesis; + nano::genesis genesis; + nano::work_pool pool (std::numeric_limits::max ()); + nano::state_block block1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); + nano::state_block block2 (nano::test_genesis_key.pub, block1.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio - 1, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (block1.hash ())); + + auto code = [block1, block2](auto confirmation_height, nano::block_hash const & expected_cemented_frontier) { + auto path (nano::unique_path ()); + nano::mdb_val value; + { + nano::genesis genesis; + nano::logger_mt logger; + nano::mdb_store store (logger, path); + nano::stat stats; + nano::ledger ledger (store, stats); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, genesis, ledger.cache); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); + // The representation table should get removed after, so readd it so that we can later confirm this actually happens + auto txn = store.env.tx (transaction); + ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); + auto weight = ledger.cache.rep_weights.representation_get (nano::genesis_account); + ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.representation, nano::mdb_val (nano::genesis_account), nano::mdb_val (nano::uint128_union (weight)), 0)); + modify_confirmation_height_to_v15 (store, transaction, nano::genesis_account, confirmation_height); + + // Lower the database to the previous version + store.version_put (transaction, 15); + // Confirm the rep weight exists in the database + ASSERT_EQ (MDB_SUCCESS, mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); + } + + // Now do the upgrade nano::logger_mt logger; + auto error (false); nano::mdb_store store (logger, path); - nano::stat stats; - nano::ledger ledger (store, stats); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, genesis, ledger.cache); - // The representation table should get removed after, so readd it so that we can later confirm this actually happens - auto txn = store.env.tx (transaction); - ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); - auto weight = ledger.cache.rep_weights.representation_get (nano::genesis_account); - ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.representation, nano::mdb_val (nano::genesis_account), nano::mdb_val (nano::uint128_union (weight)), 0)); - // Lower the database to the previous version - store.version_put (transaction, 15); - // Confirm the rep weight exists in the database - ASSERT_EQ (MDB_SUCCESS, mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); - } + ASSERT_FALSE (error); + auto transaction (store.tx_begin_read ()); - // Now do the upgrade - nano::logger_mt logger; - auto error (false); - nano::mdb_store store (logger, path); - ASSERT_FALSE (error); - auto transaction (store.tx_begin_read ()); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, confirmation_height); + + // Check confirmation height frontier is correct + ASSERT_EQ (confirmation_height_info.frontier, expected_cemented_frontier); - // The representation table should now be deleted - auto error_get_representation (mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); - ASSERT_NE (MDB_SUCCESS, error_get_representation); - ASSERT_EQ (store.representation, 0); + // The representation table should now be deleted + auto error_get_representation (mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); + ASSERT_NE (MDB_SUCCESS, error_get_representation); + ASSERT_EQ (store.representation, 0); + }; + + code (1, genesis.hash ()); + code (2, block1.hash ()); + code (3, block2.hash ()); } TEST (mdb_block_store, upgrade_backup) @@ -1794,32 +1831,41 @@ TEST (block_store, confirmation_height) nano::account account1 (0); nano::account account2 (1); nano::account account3 (2); + nano::block_hash cemented_frontier1 (3); + nano::block_hash cemented_frontier2 (4); + nano::block_hash cemented_frontier3 (5); { auto transaction (store.tx_begin_write ()); - store.confirmation_height_put (transaction, account1, 500); - store.confirmation_height_put (transaction, account2, std::numeric_limits::max ()); - store.confirmation_height_put (transaction, account3, 10); - - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, account1, confirmation_height)); - ASSERT_EQ (confirmation_height, 500); - ASSERT_FALSE (store.confirmation_height_get (transaction, account2, confirmation_height)); - ASSERT_EQ (confirmation_height, std::numeric_limits::max ()); - ASSERT_FALSE (store.confirmation_height_get (transaction, account3, confirmation_height)); - ASSERT_EQ (confirmation_height, 10); + store.confirmation_height_put (transaction, account1, { 500, cemented_frontier1 }); + store.confirmation_height_put (transaction, account2, { std::numeric_limits::max (), cemented_frontier2 }); + store.confirmation_height_put (transaction, account3, { 10, cemented_frontier3 }); + + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, account1, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 500); + ASSERT_EQ (confirmation_height_info.frontier, cemented_frontier1); + ASSERT_FALSE (store.confirmation_height_get (transaction, account2, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, std::numeric_limits::max ()); + ASSERT_EQ (confirmation_height_info.frontier, cemented_frontier2); + ASSERT_FALSE (store.confirmation_height_get (transaction, account3, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 10); + ASSERT_EQ (confirmation_height_info.frontier, cemented_frontier3); // Check cleaning of confirmation heights store.confirmation_height_clear (transaction); } auto transaction (store.tx_begin_read ()); ASSERT_EQ (store.confirmation_height_count (transaction), 3); - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, account1, confirmation_height)); - ASSERT_EQ (confirmation_height, 0); - ASSERT_FALSE (store.confirmation_height_get (transaction, account2, confirmation_height)); - ASSERT_EQ (confirmation_height, 0); - ASSERT_FALSE (store.confirmation_height_get (transaction, account3, confirmation_height)); - ASSERT_EQ (confirmation_height, 0); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, account1, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 0); + ASSERT_EQ (confirmation_height_info.frontier, nano::block_hash (0)); + ASSERT_FALSE (store.confirmation_height_get (transaction, account2, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 0); + ASSERT_EQ (confirmation_height_info.frontier, nano::block_hash (0)); + ASSERT_FALSE (store.confirmation_height_get (transaction, account3, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 0); + ASSERT_EQ (confirmation_height_info.frontier, nano::block_hash (0)); } // Upgrade many accounts and check they all have a confirmation height of 0 (except genesis which should have 1) @@ -1851,6 +1897,7 @@ TEST (mdb_block_store, upgrade_confirmation_height_many) auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0)); ASSERT_EQ (status, 0); } + store.confirmation_height_del (transaction, nano::genesis_account); ASSERT_EQ (store.count (transaction, store.accounts_v0), total_num_accounts); } @@ -1864,7 +1911,8 @@ TEST (mdb_block_store, upgrade_confirmation_height_many) for (auto i (store.confirmation_height_begin (transaction)), n (store.confirmation_height_end ()); i != n; ++i) { - ASSERT_EQ (i->second, (i->first == nano::genesis_account) ? 1 : 0); + ASSERT_EQ (i->second.height, (i->first == nano::genesis_account) ? 1 : 0); + ASSERT_EQ (i->second.frontier, (i->first == nano::genesis_account) ? genesis.hash () : nano::block_hash (0)); } } @@ -2011,6 +2059,13 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons assert (status == 0); } +void modify_confirmation_height_to_v15 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height) +{ + auto status (mdb_put (store.env.tx (transaction), store.confirmation_height, nano::mdb_val (account), nano::mdb_val (confirmation_height), 0)); + (void)status; + assert (status == 0); +} + void modify_genesis_account_info_to_v5 (nano::mdb_store & store, nano::transaction const & transaction) { nano::account_info info; diff --git a/nano/core_test/confirmation_height.cpp b/nano/core_test/confirmation_height.cpp index 7af8142b68..89a05c9941 100644 --- a/nano/core_test/confirmation_height.cpp +++ b/nano/core_test/confirmation_height.cpp @@ -21,43 +21,42 @@ void add_callback_stats (nano::node & node) TEST (confirmation_height, single) { auto amount (std::numeric_limits::max ()); - nano::system system (2); + nano::system system; + auto node = system.add_node (); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - nano::block_hash latest1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - system.wallet (1)->insert_adhoc (key1.prv); - auto send1 (std::make_shared (latest1, key1.pub, amount - system.nodes[0]->config.receive_minimum.number (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (latest1))); + nano::block_hash latest1 (node->latest (nano::test_genesis_key.pub)); + auto send1 (std::make_shared (nano::test_genesis_key.pub, latest1, nano::test_genesis_key.pub, amount - 100, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (latest1))); // Check confirmation heights before, should be uninitialized (1 for genesis). - uint64_t confirmation_height; - for (auto & node : system.nodes) - { - add_callback_stats (*node); - auto transaction = node->store.tx_begin_read (); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (1, confirmation_height); - } + nano::confirmation_height_info confirmation_height_info; + add_callback_stats (*node); + auto transaction = node->store.tx_begin_read (); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (nano::genesis ().hash (), confirmation_height_info.frontier); - for (auto & node : system.nodes) - { - node->process_active (send1); - node->block_processor.flush (); + node->process_active (send1); + node->block_processor.flush (); - system.deadline_set (10s); - while (true) + system.deadline_set (10s); + while (true) + { + auto transaction = node->store.tx_begin_read (); + if (node->ledger.block_confirmed (transaction, send1->hash ())) { - auto transaction = node->store.tx_begin_read (); - if (node->ledger.block_confirmed (transaction, send1->hash ())) - { - break; - } - - ASSERT_NO_ERROR (system.poll ()); + break; } + ASSERT_NO_ERROR (system.poll ()); + } + + { auto transaction = node->store.tx_begin_write (); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (2, confirmation_height); + ASSERT_TRUE (node->ledger.block_confirmed (transaction, send1->hash ())); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (2, confirmation_height_info.height); + ASSERT_EQ (send1->hash (), confirmation_height_info.frontier); // Rollbacks should fail as these blocks have been cemented ASSERT_TRUE (node->ledger.rollback (transaction, latest1)); @@ -72,17 +71,15 @@ TEST (confirmation_height, multiple_accounts) nano::system system; nano::node_config node_config (nano::get_available_port (), system.logging); node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled; - system.add_node (node_config); - node_config.peering_port = nano::get_available_port (); - system.add_node (node_config); + auto node = system.add_node (node_config); nano::keypair key1; nano::keypair key2; nano::keypair key3; system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); nano::block_hash latest1 (system.nodes[0]->latest (nano::test_genesis_key.pub)); - system.wallet (1)->insert_adhoc (key1.prv); + system.wallet (0)->insert_adhoc (key1.prv); system.wallet (0)->insert_adhoc (key2.prv); - system.wallet (1)->insert_adhoc (key3.prv); + system.wallet (0)->insert_adhoc (key3.prv); // Send to all accounts nano::send_block send1 (latest1, key1.pub, system.nodes.front ()->config.online_weight_minimum.number () + 300, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (latest1)); @@ -94,7 +91,7 @@ TEST (confirmation_height, multiple_accounts) nano::open_block open2 (send2.hash (), nano::genesis_account, key2.pub, key2.prv, key2.pub, *system.work.generate (key2.pub)); nano::open_block open3 (send3.hash (), nano::genesis_account, key3.pub, key3.prv, key3.pub, *system.work.generate (key3.pub)); - // Send and recieve various blocks to these accounts + // Send and receive various blocks to these accounts nano::send_block send4 (open1.hash (), key2.pub, 50, key1.prv, key1.pub, *system.work.generate (open1.hash ())); nano::send_block send5 (send4.hash (), key2.pub, 10, key1.prv, key1.pub, *system.work.generate (send4.hash ())); @@ -102,10 +99,9 @@ TEST (confirmation_height, multiple_accounts) nano::send_block send6 (receive1.hash (), key3.pub, 10, key2.prv, key2.pub, *system.work.generate (receive1.hash ())); nano::receive_block receive2 (send6.hash (), send5.hash (), key2.prv, key2.pub, *system.work.generate (send6.hash ())); - for (auto & node : system.nodes) - { - add_callback_stats (*node); + add_callback_stats (*node); + { auto transaction = node->store.tx_begin_write (); ASSERT_EQ (nano::process_result::progress, node->ledger.process (transaction, send1).code); ASSERT_EQ (nano::process_result::progress, node->ledger.process (transaction, send2).code); @@ -124,82 +120,87 @@ TEST (confirmation_height, multiple_accounts) // Check confirmation heights of all the accounts are uninitialized (0), // as we have any just added them to the ledger and not processed any live transactions yet. - uint64_t confirmation_height; - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (1, confirmation_height); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); - ASSERT_EQ (0, confirmation_height); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height)); - ASSERT_EQ (0, confirmation_height); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key3.pub, confirmation_height)); - ASSERT_EQ (0, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (nano::genesis ().hash (), confirmation_height_info.frontier); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (0), confirmation_height_info.frontier); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (0), confirmation_height_info.frontier); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key3.pub, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (0), confirmation_height_info.frontier); } // The nodes process a live receive which propagates across to all accounts auto receive3 = std::make_shared (open3.hash (), send6.hash (), key3.prv, key3.pub, *system.work.generate (open3.hash ())); - for (auto & node : system.nodes) - { - node->process_active (receive3); - node->block_processor.flush (); - - system.deadline_set (10s); - while (true) - { - auto transaction = node->store.tx_begin_read (); - if (node->ledger.block_confirmed (transaction, receive3->hash ())) - { - break; - } - - ASSERT_NO_ERROR (system.poll ()); - } + node->process_active (receive3); + node->block_processor.flush (); - nano::account_info account_info; - uint64_t confirmation_height; - auto & store = node->store; + system.deadline_set (10s); + while (true) + { auto transaction = node->store.tx_begin_read (); - ASSERT_FALSE (store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (4, confirmation_height); - ASSERT_EQ (4, account_info.block_count); - ASSERT_FALSE (store.account_get (transaction, key1.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); - ASSERT_EQ (2, confirmation_height); - ASSERT_EQ (3, account_info.block_count); - ASSERT_FALSE (store.account_get (transaction, key2.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height)); - ASSERT_EQ (3, confirmation_height); - ASSERT_EQ (4, account_info.block_count); - ASSERT_FALSE (store.account_get (transaction, key3.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key3.pub, confirmation_height)); - ASSERT_EQ (2, confirmation_height); - ASSERT_EQ (2, account_info.block_count); - - // The accounts for key1 and key2 have 1 more block in the chain than is confirmed. - // So this can be rolled back, but the one before that cannot. Check that this is the case + if (node->ledger.block_confirmed (transaction, receive3->hash ())) { - auto transaction = node->store.tx_begin_write (); - ASSERT_FALSE (node->ledger.rollback (transaction, node->latest (key2.pub))); - ASSERT_FALSE (node->ledger.rollback (transaction, node->latest (key1.pub))); + break; } - { - // These rollbacks should fail - auto transaction = node->store.tx_begin_write (); - ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (key1.pub))); - ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (key2.pub))); - // Confirm the other latest can't be rolled back either - ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (key3.pub))); - ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (nano::test_genesis_key.pub))); + ASSERT_NO_ERROR (system.poll ()); + } - // Attempt some others which have been cemented - ASSERT_TRUE (node->ledger.rollback (transaction, open1.hash ())); - ASSERT_TRUE (node->ledger.rollback (transaction, send2.hash ())); - } - ASSERT_EQ (10, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); - ASSERT_EQ (10, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); + nano::account_info account_info; + nano::confirmation_height_info confirmation_height_info; + auto & store = node->store; + auto transaction = node->store.tx_begin_read (); + ASSERT_FALSE (store.account_get (transaction, nano::test_genesis_key.pub, account_info)); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (4, confirmation_height_info.height); + ASSERT_EQ (send3.hash (), confirmation_height_info.frontier); + ASSERT_EQ (4, account_info.block_count); + ASSERT_FALSE (store.account_get (transaction, key1.pub, account_info)); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height_info)); + ASSERT_EQ (2, confirmation_height_info.height); + ASSERT_EQ (send4.hash (), confirmation_height_info.frontier); + ASSERT_EQ (3, account_info.block_count); + ASSERT_FALSE (store.account_get (transaction, key2.pub, account_info)); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height_info)); + ASSERT_EQ (3, confirmation_height_info.height); + ASSERT_EQ (send6.hash (), confirmation_height_info.frontier); + ASSERT_EQ (4, account_info.block_count); + ASSERT_FALSE (store.account_get (transaction, key3.pub, account_info)); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key3.pub, confirmation_height_info)); + ASSERT_EQ (2, confirmation_height_info.height); + ASSERT_EQ (receive3->hash (), confirmation_height_info.frontier); + ASSERT_EQ (2, account_info.block_count); + + // The accounts for key1 and key2 have 1 more block in the chain than is confirmed. + // So this can be rolled back, but the one before that cannot. Check that this is the case + { + auto transaction = node->store.tx_begin_write (); + ASSERT_FALSE (node->ledger.rollback (transaction, node->latest (key2.pub))); + ASSERT_FALSE (node->ledger.rollback (transaction, node->latest (key1.pub))); } + { + // These rollbacks should fail + auto transaction = node->store.tx_begin_write (); + ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (key1.pub))); + ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (key2.pub))); + + // Confirm the other latest can't be rolled back either + ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (key3.pub))); + ASSERT_TRUE (node->ledger.rollback (transaction, node->latest (nano::test_genesis_key.pub))); + + // Attempt some others which have been cemented + ASSERT_TRUE (node->ledger.rollback (transaction, open1.hash ())); + ASSERT_TRUE (node->ledger.rollback (transaction, send2.hash ())); + } + ASSERT_EQ (10, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); + ASSERT_EQ (10, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); } TEST (confirmation_height, gap_bootstrap) @@ -241,9 +242,10 @@ TEST (confirmation_height, gap_bootstrap) auto unchecked_count (node1.store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 2); - uint64_t confirmation_height; - ASSERT_FALSE (node1.store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (1, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node1.store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (genesis.hash (), confirmation_height_info.frontier); } // Now complete the chain where the block comes in on the bootstrap network. @@ -256,11 +258,13 @@ TEST (confirmation_height, gap_bootstrap) auto unchecked_count (node1.store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 0); - uint64_t confirmation_height; - ASSERT_FALSE (node1.store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (1, confirmation_height); - ASSERT_FALSE (node1.store.confirmation_height_get (transaction, destination.pub, confirmation_height)); - ASSERT_EQ (0, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node1.store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (genesis.hash (), confirmation_height_info.frontier); + ASSERT_FALSE (node1.store.confirmation_height_get (transaction, destination.pub, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (0), confirmation_height_info.frontier); } ASSERT_EQ (0, node1.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (0, node1.stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); @@ -310,9 +314,10 @@ TEST (confirmation_height, gap_live) // Confirmation heights should not be updated { auto transaction = node->store.tx_begin_read (); - uint64_t confirmation_height; - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (1, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (nano::genesis ().hash (), confirmation_height_info.frontier); } // Now complete the chain where the block comes in on the live network @@ -336,11 +341,14 @@ TEST (confirmation_height, gap_live) auto unchecked_count (node->store.unchecked_count (transaction)); ASSERT_EQ (unchecked_count, 0); - uint64_t confirmation_height; - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (4, confirmation_height); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, destination.pub, confirmation_height)); - ASSERT_EQ (3, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_TRUE (node->ledger.block_confirmed (transaction, receive2->hash ())); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (4, confirmation_height_info.height); + ASSERT_EQ (send3->hash (), confirmation_height_info.frontier); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, destination.pub, confirmation_height_info)); + ASSERT_EQ (3, confirmation_height_info.height); + ASSERT_EQ (receive2->hash (), confirmation_height_info.frontier); ASSERT_EQ (6, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (6, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); @@ -359,8 +367,8 @@ TEST (confirmation_height, send_receive_between_2_accounts) system.wallet (0)->insert_adhoc (key1.prv); nano::send_block send1 (latest, key1.pub, node->config.online_weight_minimum.number () + 2, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (latest)); - nano::open_block open1 (send1.hash (), nano::genesis_account, key1.pub, key1.prv, key1.pub, *system.work.generate (key1.pub)); + nano::open_block open1 (send1.hash (), nano::genesis_account, key1.pub, key1.prv, key1.pub, *system.work.generate (key1.pub)); nano::send_block send2 (open1.hash (), nano::genesis_account, 1000, key1.prv, key1.pub, *system.work.generate (open1.hash ())); nano::send_block send3 (send2.hash (), nano::genesis_account, 900, key1.prv, key1.pub, *system.work.generate (send2.hash ())); nano::send_block send4 (send3.hash (), nano::genesis_account, 500, key1.prv, key1.pub, *system.work.generate (send3.hash ())); @@ -410,17 +418,18 @@ TEST (confirmation_height, send_receive_between_2_accounts) } auto transaction (node->store.tx_begin_read ()); - nano::account_info account_info; - uint64_t confirmation_height; + nano::confirmation_height_info confirmation_height_info; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (6, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (6, confirmation_height_info.height); + ASSERT_EQ (send5.hash (), confirmation_height_info.frontier); ASSERT_EQ (7, account_info.block_count); ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); - ASSERT_EQ (5, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height_info)); + ASSERT_EQ (5, confirmation_height_info.height); + ASSERT_EQ (receive4->hash (), confirmation_height_info.frontier); ASSERT_EQ (5, account_info.block_count); ASSERT_EQ (10, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); @@ -479,13 +488,14 @@ TEST (confirmation_height, send_receive_self) auto transaction (node->store.tx_begin_read ()); nano::account_info account_info; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - uint64_t confirmation_height; - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (7, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (7, confirmation_height_info.height); + ASSERT_EQ (receive3->hash (), confirmation_height_info.frontier); ASSERT_EQ (8, account_info.block_count); ASSERT_EQ (6, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); ASSERT_EQ (6, node->stats.count (nano::stat::type::http_callback, nano::stat::detail::http_callback, nano::stat::dir::out)); - ASSERT_EQ (confirmation_height, node->ledger.cache.cemented_count); + ASSERT_EQ (confirmation_height_info.height, node->ledger.cache.cemented_count); } TEST (confirmation_height, all_block_types) @@ -572,20 +582,23 @@ TEST (confirmation_height, all_block_types) auto transaction (node->store.tx_begin_read ()); nano::account_info account_info; - uint64_t confirmation_height; + nano::confirmation_height_info confirmation_height_info; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (3, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (3, confirmation_height_info.height); + ASSERT_EQ (send1.hash (), confirmation_height_info.frontier); ASSERT_LE (4, account_info.block_count); ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); - ASSERT_EQ (6, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height_info)); + ASSERT_EQ (state_send1.hash (), confirmation_height_info.frontier); + ASSERT_EQ (6, confirmation_height_info.height); ASSERT_LE (7, account_info.block_count); ASSERT_FALSE (node->store.account_get (transaction, key2.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height)); - ASSERT_EQ (7, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key2.pub, confirmation_height_info)); + ASSERT_EQ (7, confirmation_height_info.height); + ASSERT_EQ (state_send2->hash (), confirmation_height_info.frontier); ASSERT_LE (8, account_info.block_count); ASSERT_EQ (15, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); @@ -634,12 +647,12 @@ TEST (confirmation_height, conflict_rollback_cemented) { auto transaction (node1->store.tx_begin_write ()); ASSERT_TRUE (node1->store.block_exists (transaction, publish1.block->hash ())); - node1->store.confirmation_height_put (transaction, nano::genesis_account, 2); + node1->store.confirmation_height_put (transaction, nano::genesis_account, nano::confirmation_height_info{ 2, send2->hash () }); } { auto transaction (node2->store.tx_begin_write ()); ASSERT_TRUE (node2->store.block_exists (transaction, publish2.block->hash ())); - node2->store.confirmation_height_put (transaction, nano::genesis_account, 2); + node2->store.confirmation_height_put (transaction, nano::genesis_account, nano::confirmation_height_info{ 2, send2->hash () }); } auto rollback_log_entry = boost::str (boost::format ("Failed to roll back %1%") % send2->hash ().to_string ()); diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 44df878d6e..326342b640 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -53,9 +53,10 @@ TEST (ledger, genesis_balance) ASSERT_GE (nano::seconds_since_epoch (), info.modified); ASSERT_LT (nano::seconds_since_epoch () - info.modified, 10); // Genesis block should be confirmed by default - uint64_t confirmation_height; - ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (confirmation_height, 1); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (confirmation_height_info.height, 1); + ASSERT_EQ (confirmation_height_info.frontier, genesis.hash ()); } // All nodes in the system should agree on the genesis balance @@ -2836,16 +2837,19 @@ TEST (ledger, confirmation_height_not_updated) ASSERT_FALSE (store->account_get (transaction, nano::test_genesis_key.pub, account_info)); nano::keypair key; nano::send_block send1 (account_info.head, key.pub, 50, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (account_info.head)); - uint64_t confirmation_height; - ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (1, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (genesis.hash (), confirmation_height_info.frontier); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code); - ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height)); - ASSERT_EQ (1, confirmation_height); + ASSERT_FALSE (store->confirmation_height_get (transaction, nano::genesis_account, confirmation_height_info)); + ASSERT_EQ (1, confirmation_height_info.height); + ASSERT_EQ (genesis.hash (), confirmation_height_info.frontier); nano::open_block open1 (send1.hash (), nano::genesis_account, key.pub, key.prv, key.pub, *pool.generate (key.pub)); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, open1).code); - ASSERT_FALSE (store->confirmation_height_get (transaction, key.pub, confirmation_height)); - ASSERT_EQ (0, confirmation_height); + ASSERT_FALSE (store->confirmation_height_get (transaction, key.pub, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); + ASSERT_EQ (nano::block_hash (0), confirmation_height_info.frontier); } TEST (ledger, zero_rep) diff --git a/nano/core_test/versioning.cpp b/nano/core_test/versioning.cpp index 0f1bddd06f..c7cd7dcba6 100644 --- a/nano/core_test/versioning.cpp +++ b/nano/core_test/versioning.cpp @@ -36,9 +36,9 @@ TEST (versioning, account_info_v1) ASSERT_EQ (v1.modified, v_latest.modified); ASSERT_EQ (v1.rep_block, open.hash ()); ASSERT_EQ (1, v_latest.block_count); - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height)); - ASSERT_EQ (0, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); ASSERT_EQ (nano::epoch::epoch_0, v_latest.epoch ()); } @@ -72,9 +72,9 @@ TEST (versioning, account_info_v5) ASSERT_EQ (v5.modified, v_latest.modified); ASSERT_EQ (v5.rep_block, open.hash ()); ASSERT_EQ (1, v_latest.block_count); - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height)); - ASSERT_EQ (0, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); ASSERT_EQ (nano::epoch::epoch_0, v_latest.epoch ()); } @@ -108,8 +108,8 @@ TEST (versioning, account_info_v13) ASSERT_EQ (v13.modified, v_latest.modified); ASSERT_EQ (v13.rep_block, open.hash ()); ASSERT_EQ (v13.block_count, v_latest.block_count); - uint64_t confirmation_height; - ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height)); - ASSERT_EQ (0, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (store.confirmation_height_get (transaction, account, confirmation_height_info)); + ASSERT_EQ (0, confirmation_height_info.height); ASSERT_EQ (v13.epoch, v_latest.epoch ()); } diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index b55868e2b6..a3a851c1e1 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -112,6 +112,7 @@ TEST (wallets, upgrade) auto status (mdb_put (mdb_store.env.tx (transaction_destination), info.epoch () == nano::epoch::epoch_0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0)); (void)status; assert (status == 0); + mdb_store.confirmation_height_del (transaction_destination, nano::genesis_account); } auto node1 (std::make_shared (system.io_ctx, path, system.alarm, node_config1, system.work)); ASSERT_EQ (1, node1->wallets.items.size ()); diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index d2aa2d40e7..bbac0bb972 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -617,7 +617,8 @@ int main (int argc, char * const * argv) run_addr2line (false); { std::ofstream ofs (crash_report_filename, std::ios_base::out | std::ios_base::app); - ofs << std::endl << "Using relative addresses:" << std::endl; // Add an empty line to separate the absolute & relative output + ofs << std::endl + << "Using relative addresses:" << std::endl; // Add an empty line to separate the absolute & relative output } // Now run using relative addresses. This will give actual results for other dlls, the results from the nano_node executable. @@ -975,12 +976,12 @@ int main (int argc, char * const * argv) } nano::account_info const & info (i->second); nano::account const & account (i->first); - uint64_t confirmation_height; - node.node->store.confirmation_height_get (transaction, account, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + node.node->store.confirmation_height_get (transaction, account, confirmation_height_info); - if (confirmation_height > info.block_count) + if (confirmation_height_info.height > info.block_count) { - std::cerr << "Confirmation height " << confirmation_height << " greater than block count " << info.block_count << " for account: " << account.to_account () << std::endl; + std::cerr << "Confirmation height " << confirmation_height_info.height << " greater than block count " << info.block_count << " for account: " << account.to_account () << std::endl; } auto hash (info.open_block); diff --git a/nano/node/active_transactions.cpp b/nano/node/active_transactions.cpp index db04bae1f3..407058eed1 100644 --- a/nano/node/active_transactions.cpp +++ b/nano/node/active_transactions.cpp @@ -81,11 +81,11 @@ void nano::active_transactions::search_frontiers (nano::transaction const & tran auto error = node.store.account_get (transaction_a, cementable_account.account, info); if (!error) { - uint64_t confirmation_height; - error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + error = node.store.confirmation_height_get (transaction_a, cementable_account.account, confirmation_height_info); release_assert (!error); - if (info.block_count > confirmation_height && !this->node.pending_confirmation_height.is_processing_block (info.head)) + if (info.block_count > confirmation_height_info.height && !this->node.pending_confirmation_height.is_processing_block (info.head)) { auto block (this->node.store.block_get (transaction_a, info.head)); if (!this->start (block, true)) @@ -408,11 +408,11 @@ void nano::active_transactions::prioritize_frontiers_for_confirmation (nano::tra auto i (wallet->store.begin (wallet_transaction, next_wallet_frontier_account)); auto n (wallet->store.end ()); - uint64_t confirmation_height = 0; + nano::confirmation_height_info confirmation_height_info; for (; i != n; ++i) { auto const & account (i->first); - if (!node.store.account_get (transaction_a, account, info) && !node.store.confirmation_height_get (transaction_a, account, confirmation_height)) + if (!node.store.account_get (transaction_a, account, info) && !node.store.confirmation_height_get (transaction_a, account, confirmation_height_info)) { // If it exists in normal priority collection delete from there. auto it = priority_cementable_frontiers.find (account); @@ -423,7 +423,7 @@ void nano::active_transactions::prioritize_frontiers_for_confirmation (nano::tra priority_cementable_frontiers_size = priority_cementable_frontiers.size (); } - prioritize_account_for_confirmation (priority_wallet_cementable_frontiers, priority_wallet_cementable_frontiers_size, account, info, confirmation_height); + prioritize_account_for_confirmation (priority_wallet_cementable_frontiers, priority_wallet_cementable_frontiers_size, account, info, confirmation_height_info.height); if (wallet_account_timer.since_start () >= wallet_account_time_a) { @@ -454,16 +454,16 @@ void nano::active_transactions::prioritize_frontiers_for_confirmation (nano::tra auto i (node.store.latest_begin (transaction_a, next_frontier_account)); auto n (node.store.latest_end ()); - uint64_t confirmation_height = 0; + nano::confirmation_height_info confirmation_height_info; for (; i != n && !stopped; ++i) { auto const & account (i->first); auto const & info (i->second); if (priority_wallet_cementable_frontiers.find (account) == priority_wallet_cementable_frontiers.end ()) { - if (!node.store.confirmation_height_get (transaction_a, account, confirmation_height)) + if (!node.store.confirmation_height_get (transaction_a, account, confirmation_height_info)) { - prioritize_account_for_confirmation (priority_cementable_frontiers, priority_cementable_frontiers_size, account, info, confirmation_height); + prioritize_account_for_confirmation (priority_cementable_frontiers, priority_cementable_frontiers_size, account, info, confirmation_height_info.height); } } next_frontier_account = account.number () + 1; @@ -636,9 +636,9 @@ void nano::active_transactions::update_difficulty (std::shared_ptr auto hash (block_a->hash ()); auto existing_block (node.store.block_get (*opt_transaction_a, hash, &existing_sideband)); release_assert (existing_block != nullptr); - uint64_t confirmation_height; - release_assert (!node.store.confirmation_height_get (*opt_transaction_a, node.store.block_account (*opt_transaction_a, hash), confirmation_height)); - bool confirmed = (confirmation_height >= existing_sideband.height); + nano::confirmation_height_info confirmation_height_info; + release_assert (!node.store.confirmation_height_get (*opt_transaction_a, node.store.block_account (*opt_transaction_a, hash), confirmation_height_info)); + bool confirmed = (confirmation_height_info.height >= existing_sideband.height); if (!confirmed && existing_block->block_work () != block_a->block_work ()) { uint64_t existing_difficulty; diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index a08ff8def7..bbb6b57ca3 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -427,7 +427,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction info_a.modified = nano::seconds_since_epoch (); } node.store.unchecked_put (transaction_a, nano::unchecked_key (node.ledger.block_source (transaction_a, *(info_a.block)), hash), info_a); - ++node.ledger.cache.unchecked_count; + ++node.ledger.cache.unchecked_count; node.gap_cache.add (hash); break; } diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 1035c32d06..784def899d 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -483,20 +483,20 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map nano::account account; if (!account.decode_account (account_str)) { - uint64_t confirmation_height; + nano::confirmation_height_info confirmation_height_info; auto transaction (node.node->store.tx_begin_read ()); - if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height)) + if (!node.node->store.confirmation_height_get (transaction, account, confirmation_height_info)) { auto transaction (node.node->store.tx_begin_write ()); auto conf_height_reset_num = 0; if (account == node.node->network_params.ledger.genesis_account) { conf_height_reset_num = 1; - node.node->store.confirmation_height_put (transaction, account, confirmation_height); + node.node->store.confirmation_height_put (transaction, account, { confirmation_height_info.height, node.node->network_params.ledger.genesis_block }); } else { - node.node->store.confirmation_height_clear (transaction, account, confirmation_height); + node.node->store.confirmation_height_clear (transaction, account, confirmation_height_info.height); } std::cout << "Confirmation height of account " << account_str << " is set to " << conf_height_reset_num << std::endl; @@ -1170,7 +1170,7 @@ void reset_confirmation_heights (nano::block_store & store) // Then make sure the confirmation height of the genesis account open block is 1 nano::network_params network_params; - store.confirmation_height_put (transaction, network_params.ledger.genesis_account, 1); + store.confirmation_height_put (transaction, network_params.ledger.genesis_account, { 1, network_params.ledger.genesis_block }); } bool is_using_rocksdb (boost::filesystem::path const & data_path, std::error_code & ec) diff --git a/nano/node/confirmation_height_processor.cpp b/nano/node/confirmation_height_processor.cpp index ba5f4a9058..24a7eaaee3 100644 --- a/nano/node/confirmation_height_processor.cpp +++ b/nano/node/confirmation_height_processor.cpp @@ -142,8 +142,9 @@ void nano::confirmation_height_processor::add_confirmation_height (nano::block_h auto block_height (ledger.store.block_account_height (read_transaction, current)); nano::account account (ledger.store.block_account (read_transaction, current)); - uint64_t confirmation_height; - release_assert (!ledger.store.confirmation_height_get (read_transaction, account, confirmation_height)); + nano::confirmation_height_info confirmation_height_info; + release_assert (!ledger.store.confirmation_height_get (read_transaction, account, confirmation_height_info)); + auto confirmation_height = confirmation_height_info.height; auto iterated_height = confirmation_height; auto account_it = confirmed_iterated_pairs.find (account); if (account_it != confirmed_iterated_pairs.cend ()) @@ -304,9 +305,10 @@ bool nano::confirmation_height_processor::write_pending (std::deque confirmation_height) { #ifndef NDEBUG @@ -339,7 +341,7 @@ bool nano::confirmation_height_processor::write_pending (std::deque - #include #include diff --git a/nano/node/confirmation_solicitor.hpp b/nano/node/confirmation_solicitor.hpp index 213a69ffab..6bf606ade2 100644 --- a/nano/node/confirmation_solicitor.hpp +++ b/nano/node/confirmation_solicitor.hpp @@ -24,11 +24,12 @@ class confirmation_solicitor final class request_hash { public: - size_t operator () (nano::confirmation_solicitor::request const & item_a) const + size_t operator() (nano::confirmation_solicitor::request const & item_a) const { - return std::hash> ()(item_a.election) ^ std::hash ()(*item_a.channel); + return std::hash> () (item_a.election) ^ std::hash () (*item_a.channel); } }; + public: confirmation_solicitor (nano::node &); /** Prepare object for batching election confirmation requests*/ @@ -37,15 +38,16 @@ class confirmation_solicitor final void add (std::shared_ptr); /** Bundle hashes together for identical channels in to a single confirm_req by hash packet */ void flush (); + private: static size_t constexpr max_confirm_req_batches = 20; static size_t constexpr max_confirm_req = 5; static size_t constexpr max_block_broadcasts = 30; - int rebroadcasted { 0 }; + int rebroadcasted{ 0 }; nano::node & node; std::vector representatives; /** Unique channel/hash to be requested */ std::unordered_set requests; - bool prepared { false }; + bool prepared{ false }; }; } diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 2e7d39b6bc..a25599b4e2 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -533,8 +533,8 @@ void nano::json_handler::account_info () const bool pending = request.get ("pending", false); auto transaction (node.store.tx_begin_read ()); auto info (account_info_impl (transaction, account)); - uint64_t confirmation_height; - if (node.store.confirmation_height_get (transaction, account, confirmation_height)) + nano::confirmation_height_info confirmation_height_info; + if (node.store.confirmation_height_get (transaction, account, confirmation_height_info)) { ec = nano::error_common::account_not_found; } @@ -549,7 +549,8 @@ void nano::json_handler::account_info () response_l.put ("modified_timestamp", std::to_string (info.modified)); response_l.put ("block_count", std::to_string (info.block_count)); response_l.put ("account_version", epoch_as_string (info.epoch ())); - response_l.put ("confirmation_height", std::to_string (confirmation_height)); + response_l.put ("confirmation_height", std::to_string (confirmation_height_info.height)); + response_l.put ("confirmation_height_frontier", confirmation_height_info.frontier.to_string ()); if (representative) { response_l.put ("representative", info.representative.to_account ()); @@ -1774,7 +1775,7 @@ void nano::json_handler::confirmation_height_currently_processing () auto hash = node.pending_confirmation_height.current (); if (!hash.is_zero ()) { - response_l.put ("hash", node.pending_confirmation_height.current ().to_string ()); + response_l.put ("hash", hash.to_string ()); } else { diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 051dc0828e..fff2d68ab9 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -578,7 +578,7 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_ release_assert (rep_block != nullptr); account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i_account.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); // Move confirmation height from account_info database to its own table - confirmation_height_put (transaction_a, account, account_info_v14.confirmation_height); + mdb_put (env.tx (transaction_a), confirmation_height, nano::mdb_val (account), nano::mdb_val (account_info_v14.confirmation_height), MDB_APPEND); i_account.from_first_database ? ++account_counters.after_v0 : ++account_counters.after_v1; } @@ -687,6 +687,71 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ representation = 0; } + auto account_info_i = latest_begin (transaction_a); + auto account_info_n = latest_end (); + + // Set the confirmed frontier for each account in the confirmation height table + std::vector> confirmation_height_infos; + for (nano::mdb_iterator i (transaction_a, confirmation_height), n (nano::mdb_iterator{}); i != n; ++i, ++account_info_i) + { + nano::account account (i->first); + uint64_t confirmation_height (i->second); + + // Check account hashes matches both the accounts table and confirmation height table + assert (account == account_info_i->first); + + auto const & account_info = account_info_i->second; + + if (confirmation_height == 0) + { + confirmation_height_infos.emplace_back (account, confirmation_height_info{ 0, nano::block_hash (0) }); + } + else + { + if (account_info_i->second.block_count / 2 > confirmation_height) + { + // The confirmation height of the account to closer to the bottom of the chain, so start there and work up + nano::block_sideband sideband; + auto block = block_get (transaction_a, account_info.open_block, &sideband); + assert (block); + auto height = 1; + + while (height != confirmation_height) + { + block = block_get (transaction_a, sideband.successor, &sideband); + assert (block); + ++height; + } + + assert (sideband.height == confirmation_height); + confirmation_height_infos.emplace_back (account, confirmation_height_info{ confirmation_height, block->hash () }); + } + else + { + // The confirmation height of the account to closer to the top of the chain so start there and work down + nano::block_sideband sideband; + auto block = block_get (transaction_a, account_info.head, &sideband); + auto height = sideband.height; + while (height != confirmation_height) + { + block = block_get (transaction_a, block->previous ()); + assert (block); + --height; + } + confirmation_height_infos.emplace_back (account, confirmation_height_info{ confirmation_height, block->hash () }); + } + } + } + + // Clear it then append + auto status (mdb_drop (env.tx (transaction_a), confirmation_height, 0)); + release_assert (status == MDB_SUCCESS); + + for (auto const & confirmation_height_info_pair : confirmation_height_infos) + { + mdb_put (env.tx (transaction_a), confirmation_height, nano::mdb_val (confirmation_height_info_pair.first), nano::mdb_val (confirmation_height_info_pair.second), MDB_APPEND); + } + version_put (transaction_a, 16); } diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 9837c2723d..77f1df266a 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -184,8 +184,8 @@ class mdb_store : public block_store_partial MDB_dbi peers{ 0 }; /* - * Confirmation height of an account - * nano::account -> uint64_t + * Confirmation height of an account, and the hash for the block at that height + * nano::account -> uint64_t, nano::block_hash */ MDB_dbi confirmation_height{ 0 }; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index d6982bccff..2d28886365 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -556,24 +556,24 @@ void nano::node::process_fork (nano::transaction const & transaction_a, std::sha { std::weak_ptr this_w (shared_from_this ()); if (!active.start (ledger_block, false, [this_w, root](std::shared_ptr) { - if (auto this_l = this_w.lock ()) - { - auto attempt (this_l->bootstrap_initiator.current_attempt ()); - if (attempt && attempt->mode == nano::bootstrap_mode::legacy) - { - auto transaction (this_l->store.tx_begin_read ()); - auto account (this_l->ledger.store.frontier_get (transaction, root)); - if (!account.is_zero ()) - { - attempt->requeue_pull (nano::pull_info (account, root, root)); - } - else if (this_l->ledger.store.account_exists (transaction, root)) - { - attempt->requeue_pull (nano::pull_info (root, nano::block_hash (0), nano::block_hash (0))); - } - } - } - })) + if (auto this_l = this_w.lock ()) + { + auto attempt (this_l->bootstrap_initiator.current_attempt ()); + if (attempt && attempt->mode == nano::bootstrap_mode::legacy) + { + auto transaction (this_l->store.tx_begin_read ()); + auto account (this_l->ledger.store.frontier_get (transaction, root)); + if (!account.is_zero ()) + { + attempt->requeue_pull (nano::pull_info (account, root, root)); + } + else if (this_l->ledger.store.account_exists (transaction, root)) + { + attempt->requeue_pull (nano::pull_info (root, nano::block_hash (0), nano::block_hash (0))); + } + } + } + })) { logger.always_log (boost::str (boost::format ("Resolving fork between our block: %1% and block %2% both with root %3%") % ledger_block->hash ().to_string () % block_a->hash ().to_string () % block_a->root ().to_string ())); network.broadcast_confirm_req (ledger_block); @@ -592,7 +592,7 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (collect_container_info (node.active, "active")); composite->add_component (collect_container_info (node.bootstrap_initiator, "bootstrap_initiator")); composite->add_component (collect_container_info (node.bootstrap, "bootstrap")); - composite->add_component (collect_container_info (node.network, "network")); + composite->add_component (collect_container_info (node.network, "network")); composite->add_component (collect_container_info (node.observers, "observers")); composite->add_component (collect_container_info (node.wallets, "wallets")); composite->add_component (collect_container_info (node.vote_processor, "vote_processor")); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index e9f8dc23ce..08e27cf506 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -110,9 +110,11 @@ std::shared_ptr add_ipc_enabled_node (nano::system & system) void reset_confirmation_height (nano::block_store & store, nano::account const & account) { auto transaction = store.tx_begin_write (); - uint64_t confirmation_height; - store.confirmation_height_get (transaction, account, confirmation_height); - store.confirmation_height_clear (transaction, account, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + if (!store.confirmation_height_get (transaction, account, confirmation_height_info)) + { + store.confirmation_height_clear (transaction, account, confirmation_height_info.height); + } } void check_block_response_count (nano::system & system, nano::rpc & rpc, boost::property_tree::ptree & request, uint64_t size_count) @@ -1309,7 +1311,7 @@ TEST (rpc, frontier) nano::block_hash hash; nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); source[key.pub] = hash; - node->store.confirmation_height_put (transaction, key.pub, 0); + node->store.confirmation_height_put (transaction, key.pub, { 0, nano::block_hash (0) }); node->store.account_put (transaction, key.pub, nano::account_info (hash, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); } } @@ -1360,7 +1362,7 @@ TEST (rpc, frontier_limited) nano::block_hash hash; nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); source[key.pub] = hash; - node->store.confirmation_height_put (transaction, key.pub, 0); + node->store.confirmation_height_put (transaction, key.pub, { 0, nano::block_hash (0) }); node->store.account_put (transaction, key.pub, nano::account_info (hash, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); } } @@ -1402,7 +1404,7 @@ TEST (rpc, frontier_startpoint) nano::block_hash hash; nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); source[key.pub] = hash; - node->store.confirmation_height_put (transaction, key.pub, 0); + node->store.confirmation_height_put (transaction, key.pub, { 0, nano::block_hash (0) }); node->store.account_put (transaction, key.pub, nano::account_info (hash, 0, 0, 0, 0, 0, nano::epoch::epoch_0)); } } @@ -4873,7 +4875,7 @@ TEST (rpc, account_info) auto time (nano::seconds_since_epoch ()); { auto transaction = node1.store.tx_begin_write (); - node1.store.confirmation_height_put (transaction, nano::test_genesis_key.pub, 1); + node1.store.confirmation_height_put (transaction, nano::test_genesis_key.pub, { 1, genesis.hash () }); } scoped_thread_name_io.renew (); @@ -4901,6 +4903,8 @@ TEST (rpc, account_info) ASSERT_EQ ("2", block_count); std::string confirmation_height (response.json.get ("confirmation_height")); ASSERT_EQ ("1", confirmation_height); + std::string confirmation_height_frontier (response.json.get ("confirmation_height_frontier")); + ASSERT_EQ (genesis.hash ().to_string (), confirmation_height_frontier); ASSERT_EQ (0, response.json.get ("account_version")); boost::optional weight (response.json.get_optional ("weight")); ASSERT_FALSE (weight.is_initialized ()); @@ -7076,7 +7080,7 @@ TEST (rpc, database_txn_tracker) request.put ("min_read_time", "1000"); test_response response (request, rpc.config.port, system.io_ctx); // It can take a long time to generate stack traces - system.deadline_set (30s); + system.deadline_set (60s); while (response.status == 0) { ASSERT_NO_ERROR (system.poll ()); diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index a2d09a4f12..6fc066d626 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -92,6 +92,16 @@ class db_val static_assert (std::is_standard_layout::value, "Standard layout is required"); } + db_val (nano::confirmation_height_info const & val_a) : + buffer (std::make_shared> ()) + { + { + nano::vectorstream stream (*buffer); + val_a.serialize (stream); + } + convert_buffer_to_value (); + } + db_val (nano::block_info const & val_a) : db_val (sizeof (val_a), const_cast (&val_a)) { @@ -183,6 +193,16 @@ class db_val return result; } + explicit operator nano::confirmation_height_info () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + nano::confirmation_height_info result; + bool error (result.deserialize (stream)); + (void)error; + assert (!error); + return result; + } + explicit operator nano::unchecked_info () const { nano::bufferstream stream (reinterpret_cast (data ()), size ()); @@ -659,7 +679,7 @@ class block_store virtual void account_del (nano::write_transaction const &, nano::account const &) = 0; virtual bool account_exists (nano::transaction const &, nano::account const &) = 0; virtual size_t account_count (nano::transaction const &) = 0; - virtual void confirmation_height_clear (nano::write_transaction const &, nano::account const & account, uint64_t existing_confirmation_height) = 0; + virtual void confirmation_height_clear (nano::write_transaction const &, nano::account const &, uint64_t) = 0; virtual void confirmation_height_clear (nano::write_transaction const &) = 0; virtual nano::store_iterator latest_begin (nano::transaction const &, nano::account const &) = 0; virtual nano::store_iterator latest_begin (nano::transaction const &) = 0; @@ -719,14 +739,14 @@ class block_store virtual nano::store_iterator peers_begin (nano::transaction const & transaction_a) const = 0; virtual nano::store_iterator peers_end () const = 0; - virtual void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) = 0; - virtual bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t & confirmation_height_a) = 0; + virtual void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) = 0; + virtual bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) = 0; virtual bool confirmation_height_exists (nano::transaction const & transaction_a, nano::account const & account_a) const = 0; virtual void confirmation_height_del (nano::write_transaction const & transaction_a, nano::account const & account_a) = 0; virtual uint64_t confirmation_height_count (nano::transaction const & transaction_a) = 0; - virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) = 0; - virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) = 0; - virtual nano::store_iterator confirmation_height_end () = 0; + virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) = 0; + virtual nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) = 0; + virtual nano::store_iterator confirmation_height_end () = 0; virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; virtual std::mutex & get_cache_mutex () = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index c695cfd25b..0826db59fd 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -34,7 +34,7 @@ class block_store_partial : public block_store nano::block_sideband sideband (nano::block_type::open, network_params.ledger.genesis_account, 0, network_params.ledger.genesis_amount, 1, nano::seconds_since_epoch (), nano::epoch::epoch_0); block_put (transaction_a, hash_l, *genesis_a.open, sideband); ++ledge_cache_a.block_count; - confirmation_height_put (transaction_a, network_params.ledger.genesis_account, 1); + confirmation_height_put (transaction_a, network_params.ledger.genesis_account, nano::confirmation_height_info{ 1, genesis_a.hash () }); ++ledge_cache_a.cemented_count; account_put (transaction_a, network_params.ledger.genesis_account, { hash_l, network_params.ledger.genesis_account, genesis_a.open->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); ledge_cache_a.rep_weights.representation_put (network_params.ledger.genesis_account, std::numeric_limits::max ()); @@ -55,11 +55,11 @@ class block_store_partial : public block_store return iterator != latest_end () && nano::account (iterator->first) == account_a; } - void confirmation_height_clear (nano::write_transaction const & transaction_a, nano::account const & account, uint64_t existing_confirmation_height) override + void confirmation_height_clear (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t existing_confirmation_height_a) override { - if (existing_confirmation_height > 0) + if (existing_confirmation_height_a > 0) { - confirmation_height_put (transaction_a, account, 0); + confirmation_height_put (transaction_a, account_a, { 0, nano::block_hash{ 0 } }); } } @@ -67,7 +67,7 @@ class block_store_partial : public block_store { for (auto i (confirmation_height_begin (transaction_a)), n (confirmation_height_end ()); i != n; ++i) { - confirmation_height_clear (transaction_a, i->first, i->second); + confirmation_height_clear (transaction_a, i->first, i->second.height); } } @@ -342,9 +342,9 @@ class block_store_partial : public block_store return nano::store_iterator (nullptr); } - nano::store_iterator confirmation_height_end () override + nano::store_iterator confirmation_height_end () override { - return nano::store_iterator (nullptr); + return nano::store_iterator (nullptr); } std::mutex & get_cache_mutex () override @@ -675,24 +675,25 @@ class block_store_partial : public block_store return count (transaction_a, tables::confirmation_height); } - void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height_a) override + void confirmation_height_put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) override { - nano::db_val confirmation_height (confirmation_height_a); - auto status = put (transaction_a, tables::confirmation_height, account_a, confirmation_height); + nano::db_val confirmation_height_info (confirmation_height_info_a); + auto status = put (transaction_a, tables::confirmation_height, account_a, confirmation_height_info); release_assert (success (status)); } - bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, uint64_t & confirmation_height_a) override + bool confirmation_height_get (nano::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) override { nano::db_val value; auto status = get (transaction_a, tables::confirmation_height, nano::db_val (account_a), value); release_assert (success (status) || not_found (status)); - confirmation_height_a = 0; + bool result (true); if (success (status)) { - confirmation_height_a = static_cast (value); + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = confirmation_height_info_a.deserialize (stream); } - return (!success (status)); + return result; } void confirmation_height_del (nano::write_transaction const & transaction_a, nano::account const & account_a) override @@ -751,14 +752,14 @@ class block_store_partial : public block_store return make_iterator (transaction_a, tables::peers); } - nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) override + nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a, nano::account const & account_a) override { - return make_iterator (transaction_a, tables::confirmation_height, nano::db_val (account_a)); + return make_iterator (transaction_a, tables::confirmation_height, nano::db_val (account_a)); } - nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) override + nano::store_iterator confirmation_height_begin (nano::transaction const & transaction_a) override { - return make_iterator (transaction_a, tables::confirmation_height); + return make_iterator (transaction_a, tables::confirmation_height); } size_t unchecked_count (nano::transaction const & transaction_a) override diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index 17d437f46b..25b5b4315e 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -378,6 +378,33 @@ uint16_t nano::endpoint_key::port () const return boost::endian::big_to_native (network_port); } +nano::confirmation_height_info::confirmation_height_info (uint64_t confirmation_height_a, nano::block_hash const & confirmed_frontier_a) : +height (confirmation_height_a), +frontier (confirmed_frontier_a) +{ +} + +void nano::confirmation_height_info::serialize (nano::stream & stream_a) const +{ + nano::write (stream_a, height); + nano::write (stream_a, frontier); +} + +bool nano::confirmation_height_info::deserialize (nano::stream & stream_a) +{ + auto error (false); + try + { + nano::read (stream_a, height); + nano::read (stream_a, frontier); + } + catch (std::runtime_error const &) + { + error = true; + } + return error; +} + nano::block_info::block_info (nano::account const & account_a, nano::amount const & balance_a) : account (account_a), balance (balance_a) diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index a4f9573b9b..3cf07ac11e 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -218,6 +218,18 @@ class block_counts final size_t change{ 0 }; size_t state{ 0 }; }; + +class confirmation_height_info final +{ +public: + confirmation_height_info () = default; + confirmation_height_info (uint64_t, nano::block_hash const &); + void serialize (nano::stream &) const; + bool deserialize (nano::stream &); + uint64_t height; + nano::block_hash frontier; +}; + using vote_blocks_vec_iter = std::vector, nano::block_hash>>::const_iterator; class iterate_vote_blocks_as_hash final { diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 9d560ec576..de5a4616a3 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -704,7 +704,7 @@ check_bootstrap_weights (true) { for (auto i (store.confirmation_height_begin (transaction)), n (store.confirmation_height_end ()); i != n; ++i) { - cache.cemented_count += i->second; + cache.cemented_count += i->second.height; } } @@ -887,11 +887,11 @@ bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano auto error (false); while (!error && store.block_exists (transaction_a, block_a)) { - uint64_t confirmation_height; - auto latest_error = store.confirmation_height_get (transaction_a, account_l, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + auto latest_error = store.confirmation_height_get (transaction_a, account_l, confirmation_height_info); assert (!latest_error); (void)latest_error; - if (block_account_height > confirmation_height) + if (block_account_height > confirmation_height_info.height) { latest_error = store.account_get (transaction_a, account_l, account_info); assert (!latest_error); @@ -1042,7 +1042,7 @@ void nano::ledger::change_latest (nano::write_transaction const & transaction_a, if (old_a.head.is_zero () && new_a.open_block == new_a.head) { assert (!store.confirmation_height_exists (transaction_a, account_a)); - store.confirmation_height_put (transaction_a, account_a, 0); + store.confirmation_height_put (transaction_a, account_a, { 0, nano::block_hash (0) }); } if (!old_a.head.is_zero () && old_a.epoch () != new_a.epoch ()) { @@ -1116,9 +1116,9 @@ bool nano::ledger::block_confirmed (nano::transaction const & transaction_a, nan auto block_height (store.block_account_height (transaction_a, hash_a)); if (block_height > 0) // 0 indicates that the block doesn't exist { - uint64_t confirmation_height; - release_assert (!store.confirmation_height_get (transaction_a, account (transaction_a, hash_a), confirmation_height)); - confirmed = (confirmation_height >= block_height); + nano::confirmation_height_info confirmation_height_info; + release_assert (!store.confirmation_height_get (transaction_a, account (transaction_a, hash_a), confirmation_height_info)); + confirmed = (confirmation_height_info.height >= block_height); } return confirmed; } diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index ac1da66af1..60e7106069 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -15,7 +15,7 @@ using tally_t = std::map, std::gre class ledger final { public: - ledger (nano::block_store &, nano::stat &, nano::generate_cache const & = nano::generate_cache()); + ledger (nano::block_store &, nano::stat &, nano::generate_cache const & = nano::generate_cache ()); nano::account account (nano::transaction const &, nano::block_hash const &) const; nano::uint128_t amount (nano::transaction const &, nano::account const &); nano::uint128_t amount (nano::transaction const &, nano::block_hash const &); diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 21ff60ca64..cf1c2a678a 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -170,7 +170,7 @@ TEST (store, load) { nano::account account; nano::random_pool::generate_block (account.bytes.data (), account.bytes.size ()); - system.nodes[0]->store.confirmation_height_put (transaction, account, 0); + system.nodes[0]->store.confirmation_height_put (transaction, account, { 0, nano::block_hash (0) }); system.nodes[0]->store.account_put (transaction, account, nano::account_info ()); } } @@ -515,9 +515,9 @@ TEST (confirmation_height, many_accounts_single_confirmation) auto & account = i->first; auto & account_info = i->second; auto count = (account != last_keypair.pub) ? 2 : 1; - uint64_t confirmation_height; - ASSERT_FALSE (node->store.confirmation_height_get (transaction, account, confirmation_height)); - ASSERT_EQ (count, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, account, confirmation_height_info)); + ASSERT_EQ (count, confirmation_height_info.height); ASSERT_EQ (count, account_info.block_count); } @@ -651,13 +651,14 @@ TEST (confirmation_height, long_chains) nano::account_info account_info; ASSERT_FALSE (node->store.account_get (transaction, nano::test_genesis_key.pub, account_info)); uint64_t confirmation_height; - ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height)); - ASSERT_EQ (num_blocks + 2, confirmation_height); + nano::confirmation_height_info confirmation_height_info; + ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height_info)); + ASSERT_EQ (num_blocks + 2, confirmation_height_info.height); ASSERT_EQ (num_blocks + 3, account_info.block_count); // Includes the unpocketed send ASSERT_FALSE (node->store.account_get (transaction, key1.pub, account_info)); - ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height)); - ASSERT_EQ (num_blocks + 1, confirmation_height); + ASSERT_FALSE (node->store.confirmation_height_get (transaction, key1.pub, confirmation_height_info)); + ASSERT_EQ (num_blocks + 1, confirmation_height_info.height); ASSERT_EQ (num_blocks + 1, account_info.block_count); ASSERT_EQ (node->ledger.stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in), num_blocks * 2 + 2); From f5e95912a19424f3916023196dcd918683806d16 Mon Sep 17 00:00:00 2001 From: wezrule Date: Wed, 15 Jan 2020 11:42:15 +0000 Subject: [PATCH 2/6] Stein review comments --- nano/core_test/block_store.cpp | 6 +++++- nano/node/lmdb/lmdb.cpp | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index bcc1add1e0..e4807ed51a 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1731,8 +1731,9 @@ TEST (mdb_block_store, upgrade_v15_v16) nano::work_pool pool (std::numeric_limits::max ()); nano::state_block block1 (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); nano::state_block block2 (nano::test_genesis_key.pub, block1.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio - 1, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (block1.hash ())); + nano::state_block block3 (nano::test_genesis_key.pub, block2.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio - 2, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (block2.hash ())); - auto code = [block1, block2](auto confirmation_height, nano::block_hash const & expected_cemented_frontier) { + auto code = [block1, block2, block3](auto confirmation_height, nano::block_hash const & expected_cemented_frontier) { auto path (nano::unique_path ()); nano::mdb_val value; { @@ -1745,6 +1746,7 @@ TEST (mdb_block_store, upgrade_v15_v16) store.initialize (transaction, genesis, ledger.cache); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block3).code); // The representation table should get removed after, so readd it so that we can later confirm this actually happens auto txn = store.env.tx (transaction); ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); @@ -1778,9 +1780,11 @@ TEST (mdb_block_store, upgrade_v15_v16) ASSERT_EQ (store.representation, 0); }; + code (0, nano::block_hash (0)); code (1, genesis.hash ()); code (2, block1.hash ()); code (3, block2.hash ()); + code (4, block3.hash ()); } TEST (mdb_block_store, upgrade_backup) diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index fff2d68ab9..cd79fcece3 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -81,6 +81,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable) nano::network_constants network_constants; if (needs_vacuuming && !network_constants.is_test_network ()) { + logger.always_log ("Preparing vacuum..."); auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_max_dbs); logger.always_log (vacuum_success ? "Vacuum succeeded." : "Failed to vacuum. (Optional) Ensure enough disk space is available for a copy of the database and try to vacuum after shutting down the node"); } @@ -673,11 +674,13 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_ } version_put (transaction_a, 15); - logger.always_log ("Finished epoch merge upgrade. Preparing vacuum..."); + logger.always_log ("Finished epoch merge upgrade"); } void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_a) { + logger.always_log ("Preparing v15 to v16 upgrade..."); + // Representation table is no longer used assert (representation != 0); if (representation != 0) @@ -692,7 +695,8 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ // Set the confirmed frontier for each account in the confirmation height table std::vector> confirmation_height_infos; - for (nano::mdb_iterator i (transaction_a, confirmation_height), n (nano::mdb_iterator{}); i != n; ++i, ++account_info_i) + auto num = 0u; + for (nano::mdb_iterator i (transaction_a, confirmation_height), n (nano::mdb_iterator{}); i != n; ++i, ++account_info_i, ++num) { nano::account account (i->first); uint64_t confirmation_height (i->second); @@ -708,7 +712,7 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ } else { - if (account_info_i->second.block_count / 2 > confirmation_height) + if (account_info_i->second.block_count / 2 >= confirmation_height) { // The confirmation height of the account to closer to the bottom of the chain, so start there and work up nano::block_sideband sideband; @@ -741,6 +745,13 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ confirmation_height_infos.emplace_back (account, confirmation_height_info{ confirmation_height, block->hash () }); } } + + // Every so often output to the log to indicate progress (every 200k accounts) + constexpr auto output_cutoff = 200000; + if (num % output_cutoff == 0 && num != 0) + { + logger.always_log (boost::str (boost::format ("Confirmation height frontier set for %1%00k accounts") % ((num / output_cutoff) * 2))); + } } // Clear it then append @@ -753,6 +764,7 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ } version_put (transaction_a, 16); + logger.always_log ("Finished upgrading confirmation height frontiers"); } /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ From 9a8dd111776d903ea7c9c481ca762d61c443ef0e Mon Sep 17 00:00:00 2001 From: wezrule Date: Wed, 15 Jan 2020 11:47:08 +0000 Subject: [PATCH 3/6] Add //clang-format block around lambda --- nano/core_test/block_store.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index e4807ed51a..232dc29d22 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1733,6 +1733,7 @@ TEST (mdb_block_store, upgrade_v15_v16) nano::state_block block2 (nano::test_genesis_key.pub, block1.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio - 1, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (block1.hash ())); nano::state_block block3 (nano::test_genesis_key.pub, block2.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio - 2, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (block2.hash ())); + // clang-format off auto code = [block1, block2, block3](auto confirmation_height, nano::block_hash const & expected_cemented_frontier) { auto path (nano::unique_path ()); nano::mdb_val value; @@ -1779,6 +1780,7 @@ TEST (mdb_block_store, upgrade_v15_v16) ASSERT_NE (MDB_SUCCESS, error_get_representation); ASSERT_EQ (store.representation, 0); }; + // clang-format on code (0, nano::block_hash (0)); code (1, genesis.hash ()); From ed2a1e4b941b2fdd6c3a6dd25f400faeccbbab47 Mon Sep 17 00:00:00 2001 From: wezrule Date: Wed, 15 Jan 2020 15:11:49 +0000 Subject: [PATCH 4/6] Update comment (thanks Gui) --- nano/node/lmdb/lmdb.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index cd79fcece3..a50bee764c 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -714,7 +714,7 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ { if (account_info_i->second.block_count / 2 >= confirmation_height) { - // The confirmation height of the account to closer to the bottom of the chain, so start there and work up + // The confirmation height of the account is closer to the bottom of the chain, so start there and work up nano::block_sideband sideband; auto block = block_get (transaction_a, account_info.open_block, &sideband); assert (block); @@ -732,7 +732,7 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ } else { - // The confirmation height of the account to closer to the top of the chain so start there and work down + // The confirmation height of the account is closer to the top of the chain so start there and work down nano::block_sideband sideband; auto block = block_get (transaction_a, account_info.head, &sideband); auto height = sideband.height; From 90d5168cf3c11e88b3b0d1a202da3b3ba651ea5b Mon Sep 17 00:00:00 2001 From: wezrule Date: Thu, 16 Jan 2020 17:03:27 +0000 Subject: [PATCH 5/6] Upgrade to v17 instead --- nano/core_test/block_store.cpp | 51 ++++++++++++++++++++++-------- nano/node/lmdb/lmdb.cpp | 31 +++++++++++++----- nano/node/lmdb/lmdb.hpp | 4 ++- nano/secure/blockstore_partial.hpp | 2 +- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 232dc29d22..af37dc61f0 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1726,6 +1726,43 @@ TEST (mdb_block_store, upgrade_v14_v15) } TEST (mdb_block_store, upgrade_v15_v16) +{ + auto path (nano::unique_path ()); + nano::mdb_val value; + { + nano::genesis genesis; + nano::logger_mt logger; + nano::mdb_store store (logger, path); + nano::stat stats; + nano::ledger ledger (store, stats); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, genesis, ledger.cache); + // The representation table should get removed after, so readd it so that we can later confirm this actually happens + auto txn = store.env.tx (transaction); + ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); + auto weight = ledger.cache.rep_weights.representation_get (nano::genesis_account); + ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.representation, nano::mdb_val (nano::genesis_account), nano::mdb_val (nano::uint128_union (weight)), 0)); + // Lower the database to the previous version + store.version_put (transaction, 15); + // Confirm the rep weight exists in the database + ASSERT_EQ (MDB_SUCCESS, mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); + store.confirmation_height_del (transaction, nano::genesis_account); + } + + // Now do the upgrade + nano::logger_mt logger; + auto error (false); + nano::mdb_store store (logger, path); + ASSERT_FALSE (error); + auto transaction (store.tx_begin_read ()); + + // The representation table should now be deleted + auto error_get_representation (mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); + ASSERT_NE (MDB_SUCCESS, error_get_representation); + ASSERT_EQ (store.representation, 0); +} + +TEST (mdb_block_store, upgrade_v16_v17) { nano::genesis genesis; nano::work_pool pool (std::numeric_limits::max ()); @@ -1748,17 +1785,10 @@ TEST (mdb_block_store, upgrade_v15_v16) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block1).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block2).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block3).code); - // The representation table should get removed after, so readd it so that we can later confirm this actually happens - auto txn = store.env.tx (transaction); - ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); - auto weight = ledger.cache.rep_weights.representation_get (nano::genesis_account); - ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.representation, nano::mdb_val (nano::genesis_account), nano::mdb_val (nano::uint128_union (weight)), 0)); modify_confirmation_height_to_v15 (store, transaction, nano::genesis_account, confirmation_height); // Lower the database to the previous version - store.version_put (transaction, 15); - // Confirm the rep weight exists in the database - ASSERT_EQ (MDB_SUCCESS, mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); + store.version_put (transaction, 16); } // Now do the upgrade @@ -1774,11 +1804,6 @@ TEST (mdb_block_store, upgrade_v15_v16) // Check confirmation height frontier is correct ASSERT_EQ (confirmation_height_info.frontier, expected_cemented_frontier); - - // The representation table should now be deleted - auto error_get_representation (mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); - ASSERT_NE (MDB_SUCCESS, error_get_representation); - ASSERT_EQ (store.representation, 0); }; // clang-format on diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index a50bee764c..1835ff8e0b 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -49,9 +49,11 @@ txn_tracking_enabled (txn_tracking_config_a.enable) if (!error) { auto is_fully_upgraded (false); + auto is_fresh_db (false); { auto transaction (tx_begin_read ()); auto err = mdb_dbi_open (env.tx (transaction), "meta", 0, &meta); + is_fresh_db = err != MDB_SUCCESS; if (err == MDB_SUCCESS) { is_fully_upgraded = (version_get (transaction) == version); @@ -64,9 +66,17 @@ txn_tracking_enabled (txn_tracking_config_a.enable) // (can be a few minutes with the --fast_bootstrap flag for instance) if (!is_fully_upgraded) { - if (backup_before_upgrade) + nano::network_constants network_constants; + if (!is_fresh_db) { - create_backup_file (env, path_a, logger_a); + if (!network_constants.is_test_network ()) + { + std::cout << "Upgrade in progress..." << std::endl; + } + if (backup_before_upgrade) + { + create_backup_file (env, path_a, logger_a); + } } auto needs_vacuuming = false; { @@ -78,7 +88,6 @@ txn_tracking_enabled (txn_tracking_config_a.enable) } } - nano::network_constants network_constants; if (needs_vacuuming && !network_constants.is_test_network ()) { logger.always_log ("Preparing vacuum..."); @@ -187,7 +196,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & if (version_get (transaction_a) < 16) { - // The representation database is no longer used, but need opening so that it can be deleted during an upgrade + // The representation database is no longer used, but needs opening so that it can be deleted during an upgrade error_a |= mdb_dbi_open (env.tx (transaction_a), "representation", flags, &representation) != 0; } @@ -244,6 +253,8 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool case 15: upgrade_v15_to_v16 (transaction_a); case 16: + upgrade_v16_to_v17 (transaction_a); + case 17: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -677,10 +688,8 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_ logger.always_log ("Finished epoch merge upgrade"); } -void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_a) +void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction const & transaction_a) { - logger.always_log ("Preparing v15 to v16 upgrade..."); - // Representation table is no longer used assert (representation != 0); if (representation != 0) @@ -689,6 +698,12 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ release_assert (status == MDB_SUCCESS); representation = 0; } + version_put (transaction_a, 16); +} + +void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v16 to v17 upgrade..."); auto account_info_i = latest_begin (transaction_a); auto account_info_n = latest_end (); @@ -763,7 +778,7 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction & transaction_ mdb_put (env.tx (transaction_a), confirmation_height, nano::mdb_val (confirmation_height_info_pair.first), nano::mdb_val (confirmation_height_info_pair.second), MDB_APPEND); } - version_put (transaction_a, 16); + version_put (transaction_a, 17); logger.always_log ("Finished upgrading confirmation height frontiers"); } diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 77f1df266a..009deaf011 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -240,7 +240,9 @@ class mdb_store : public block_store_partial void upgrade_v12_to_v13 (nano::write_transaction &, size_t); void upgrade_v13_to_v14 (nano::write_transaction const &); void upgrade_v14_to_v15 (nano::write_transaction &); - void upgrade_v15_to_v16 (nano::write_transaction &); + void upgrade_v15_to_v16 (nano::write_transaction const &); + void upgrade_v16_to_v17 (nano::write_transaction const &); + void open_databases (bool &, nano::transaction const &, unsigned); int drop (nano::write_transaction const & transaction_a, tables table_a) override; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index 0826db59fd..e4d716ba70 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -771,7 +771,7 @@ class block_store_partial : public block_store nano::network_params network_params; std::unordered_map> vote_cache_l1; std::unordered_map> vote_cache_l2; - static int constexpr version{ 16 }; + static int constexpr version{ 17 }; template std::shared_ptr block_random (nano::transaction const & transaction_a, tables table_a) From 8f5b1cf7cfab4f45351370b200895c2456fecff3 Mon Sep 17 00:00:00 2001 From: wezrule Date: Thu, 16 Jan 2020 18:52:26 +0000 Subject: [PATCH 6/6] Update comments --- nano/core_test/block_store.cpp | 6 ++++++ nano/node/lmdb/lmdb.cpp | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index af37dc61f0..6435e643f0 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1760,6 +1760,9 @@ TEST (mdb_block_store, upgrade_v15_v16) auto error_get_representation (mdb_get (store.env.tx (transaction), store.representation, nano::mdb_val (nano::genesis_account), value)); ASSERT_NE (MDB_SUCCESS, error_get_representation); ASSERT_EQ (store.representation, 0); + + // Version should be correct + ASSERT_LT (15, store.version_get (transaction)); } TEST (mdb_block_store, upgrade_v16_v17) @@ -1804,6 +1807,9 @@ TEST (mdb_block_store, upgrade_v16_v17) // Check confirmation height frontier is correct ASSERT_EQ (confirmation_height_info.frontier, expected_cemented_frontier); + + // Version should be correct + ASSERT_LT (16, store.version_get (transaction)); }; // clang-format on diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 1835ff8e0b..4fef14f349 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -251,6 +251,7 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool upgrade_v14_to_v15 (transaction_a); needs_vacuuming = true; case 15: + // Upgrades to v16 & v17 are both part of the v21 node release upgrade_v15_to_v16 (transaction_a); case 16: upgrade_v16_to_v17 (transaction_a); @@ -572,7 +573,7 @@ void nano::mdb_store::upgrade_v13_to_v14 (nano::write_transaction const & transa void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_a) { - logger.always_log ("Preparing v14 to v15 upgrade..."); + logger.always_log ("Preparing v14 to v15 database upgrade..."); std::vector> account_infos; upgrade_counters account_counters (count (transaction_a, accounts_v0), count (transaction_a, accounts_v1)); @@ -703,7 +704,7 @@ void nano::mdb_store::upgrade_v15_to_v16 (nano::write_transaction const & transa void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transaction_a) { - logger.always_log ("Preparing v16 to v17 upgrade..."); + logger.always_log ("Preparing v16 to v17 database upgrade..."); auto account_info_i = latest_begin (transaction_a); auto account_info_n = latest_end ();