rpcdaemon: walk/for_prefix as functions

silkworm/rpc/commands/debug_api.cpp

@@ -41,8 +41,8 @@
 #include <silkworm/rpc/core/evm_debug.hpp>
 #include <silkworm/rpc/core/evm_executor.hpp>
 #include <silkworm/rpc/core/rawdb/chain.hpp>
-#include <silkworm/rpc/core/state_reader.hpp>
 #include <silkworm/rpc/core/storage_walker.hpp>
+#include <silkworm/rpc/ethdb/walk.hpp>
 #include <silkworm/rpc/json/types.hpp>
 #include <silkworm/rpc/types/block.hpp>
 #include <silkworm/rpc/types/call.hpp>
@@ -648,7 +648,7 @@ Task<std::set<evmc::address>> get_modified_accounts(ethdb::Transaction& tx, Bloc
         const auto key = silkworm::db::block_key(start_block_number);
         SILK_TRACE << "Ready to walk starting from key: " << silkworm::to_hex(key);
 
-        co_await tx.walk(db::table::kAccountChangeSetName, key, 0, walker);
+        co_await walk(tx, db::table::kAccountChangeSetName, key, 0, walker);
     }
 
     co_return addresses;

silkworm/rpc/core/logs_walker.cpp

@@ -29,6 +29,7 @@
 #include <silkworm/rpc/core/rawdb/chain.hpp>
 #include <silkworm/rpc/ethdb/bitmap.hpp>
 #include <silkworm/rpc/ethdb/cbor.hpp>
+#include <silkworm/rpc/ethdb/walk.hpp>
 
 namespace silkworm::rpc {
 
@@ -121,7 +122,7 @@ Task<void> LogsWalker::get_logs(std::uint64_t start, std::uint64_t end,
         filtered_block_logs.clear();
         const auto block_key = silkworm::db::block_key(block_to_match);
         SILK_DEBUG << "block_to_match: " << block_to_match << " block_key: " << silkworm::to_hex(block_key);
-        co_await tx_.for_prefix(db::table::kLogsName, block_key, [&](const silkworm::Bytes& k, const silkworm::Bytes& v) {
+        co_await for_prefix(tx_, db::table::kLogsName, block_key, [&](const silkworm::Bytes& k, const silkworm::Bytes& v) {
             chunk_logs.clear();
             const bool decoding_ok{cbor_decode(v, chunk_logs)};
             if (!decoding_ok) {

silkworm/rpc/core/rawdb/chain.cpp

@@ -32,6 +32,7 @@
 #include <silkworm/rpc/common/util.hpp>
 #include <silkworm/rpc/core/blocks.hpp>
 #include <silkworm/rpc/ethdb/cbor.hpp>
+#include <silkworm/rpc/ethdb/walk.hpp>
 #include <silkworm/rpc/json/types.hpp>
 #include <silkworm/rpc/types/receipt.hpp>
 
@@ -203,7 +204,7 @@ Task<std::optional<Receipts>> read_raw_receipts(ethdb::Transaction& tx, BlockNum
         SILK_DEBUG << "#receipts[" << tx_id << "].logs: " << receipts[tx_id].logs.size();
         return true;
     };
-    co_await tx.walk(db::table::kLogsName, log_key, 8 * CHAR_BIT, walker);
+    co_await walk(tx, db::table::kLogsName, log_key, 8 * CHAR_BIT, walker);
 
     co_return receipts;
 }

silkworm/rpc/core/rawdb/chain_test.cpp

@@ -31,12 +31,14 @@
 #include <silkworm/db/tables.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 #include <silkworm/rpc/core/blocks.hpp>
+#include <silkworm/rpc/test_util/mock_cursor.hpp>
 #include <silkworm/rpc/test_util/mock_transaction.hpp>
 
 namespace silkworm::rpc::core::rawdb {
 
 using Catch::Matchers::Message;
 using testing::_;
+using testing::InSequence;
 using testing::Invoke;
 using testing::InvokeWithoutArgs;
 using testing::Unused;
@@ -278,8 +280,14 @@ TEST_CASE("read_raw_receipts") {
 
     SECTION("one receipt") {  // https://goerli.etherscan.io/block/3529600
         const uint64_t block_number{3'529'600};
-        EXPECT_CALL(transaction, get_one(db::table::kBlockReceiptsName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<silkworm::Bytes> { co_return *silkworm::from_hex("818400f6011a0004a0c8"); }));
-        EXPECT_CALL(transaction, walk(db::table::kLogsName, _, _, _)).WillOnce(Invoke([](Unused, Unused, Unused, auto w) -> Task<void> {
+        EXPECT_CALL(transaction, get_one(db::table::kBlockReceiptsName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<silkworm::Bytes> {
+            co_return *silkworm::from_hex("818400f6011a0004a0c8");
+        }));
+        auto cursor{std::make_shared<test::MockCursor>()};
+        EXPECT_CALL(transaction, cursor(db::table::kLogsName)).WillOnce(Invoke(
+            // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
+            [&cursor](Unused) -> Task<std::shared_ptr<ethdb::Cursor>> { co_return cursor; }));
+        EXPECT_CALL(*cursor, seek(_)).WillOnce(Invoke([](Unused) -> Task<KeyValue> {
             silkworm::Bytes key{*silkworm::from_hex("000000000035db8000000000")};
             silkworm::Bytes value{*silkworm::from_hex(
                 "8683547753cfad258efbc52a9a1452e42ffbce9be486cb835820ddf252ad1be2c89b69c2b068fc"
@@ -304,18 +312,24 @@ TEST_CASE("read_raw_receipts") {
                 "1898bd2ef0835820efaf768237c22e140a862d5d375ad5c153479fac3f8bcf8b580a1651fd62c3"
                 "ef5820000000000000000000000000f13c666056048634109c1ecca6893da293c70da458200000"
                 "000000000000000000000214281cf15c1a66b51990e2e65e1f7b7c363318f6")};
-            w(key, value);
-            co_return;
+            co_return KeyValue{std::move(key), std::move(value)};
         }));
+        EXPECT_CALL(*cursor, next()).WillOnce(Invoke([]() -> Task<KeyValue> { co_return KeyValue{}; }));
         auto result = boost::asio::co_spawn(pool, read_raw_receipts(transaction, block_number), boost::asio::use_future);
         // CHECK(result.get() == Receipts{Receipt{...}}); // TODO(canepat): provide operator== and operator!= for Receipt type
         CHECK(result.get().value().size() == Receipts{Receipt{}}.size());
     }
 
     SECTION("many receipts") {  // https://goerli.etherscan.io/block/3529600
         const uint64_t block_number{3'529'604};
-        EXPECT_CALL(transaction, get_one(db::table::kBlockReceiptsName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<silkworm::Bytes> { co_return *silkworm::from_hex("828400f6011a0003be508400f6011a0008b89a"); }));
-        EXPECT_CALL(transaction, walk(db::table::kLogsName, _, _, _)).WillOnce(Invoke([](Unused, Unused, Unused, auto w) -> Task<void> {
+        EXPECT_CALL(transaction, get_one(db::table::kBlockReceiptsName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<silkworm::Bytes> {
+            co_return *silkworm::from_hex("828400f6011a0003be508400f6011a0008b89a");
+        }));
+        auto cursor{std::make_shared<test::MockCursor>()};
+        EXPECT_CALL(transaction, cursor(db::table::kLogsName)).WillOnce(Invoke(
+            // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
+            [&cursor](Unused) -> Task<std::shared_ptr<ethdb::Cursor>> { co_return cursor; }));
+        EXPECT_CALL(*cursor, seek(_)).WillOnce(Invoke([](Unused) -> Task<KeyValue> {
             silkworm::Bytes key1{*silkworm::from_hex("000000000035db8400000000")};
             silkworm::Bytes value1{*silkworm::from_hex(
                 "8383547977d4f555fbee46303682b17e72e3d94339b4418258206155cfd0fd028b0ca77e8495a6"
@@ -331,7 +345,10 @@ TEST_CASE("read_raw_receipts") {
                 "eed6e23cead48c8cec7a58b6e7195820d76aaac3ecd5ced13bbab3b240a426352f76a6fffd583c"
                 "3b15f4ddae2b754e4e5840000000000000000000000000000000000000000000000000015c2a7b"
                 "13fd0000000000000000000000000000000000000000000000000000000000005f7cd33d")};
-            w(key1, value1);
+            co_return KeyValue{std::move(key1), std::move(value1)};
+        }));
+        InSequence following_calls_in_specific_order;
+        EXPECT_CALL(*cursor, next()).WillOnce(Invoke([]() -> Task<KeyValue> {
             silkworm::Bytes key2{*silkworm::from_hex("000000000035db8400000001")};
             silkworm::Bytes value2{*silkworm::from_hex(
                 "82835407b39f4fde4a38bace212b546dac87c58dfe3fdc815820649bbc62d0e31342afea4e5cd8"
@@ -354,18 +371,24 @@ TEST_CASE("read_raw_receipts") {
                 "725881c2a4290b02cd153d6599fd484f0d4e6062c361e740fbbe39e7ad61425820000000000000"
                 "000000000000000000000000000000000000000000000000000258200000000000000000000000"
                 "00000000000000000000000000000000005f7cd33d")};
-            w(key2, value2);
-            co_return;
+            co_return KeyValue{std::move(key2), std::move(value2)};
         }));
+        EXPECT_CALL(*cursor, next()).WillOnce(Invoke([]() -> Task<KeyValue> { co_return KeyValue{}; }));
         auto result = boost::asio::co_spawn(pool, read_raw_receipts(transaction, block_number), boost::asio::use_future);
         // CHECK(result.get() == Receipts{Receipt{...}, Receipt{...}}); // TODO(canepat): provide operator== and operator!= for Receipt type
         CHECK(result.get().value().size() == Receipts{Receipt{}, Receipt{}}.size());
     }
 
     SECTION("invalid receipt log") {  // https://goerli.etherscan.io/block/3529600
         const uint64_t block_number{3'529'600};
-        EXPECT_CALL(transaction, get_one(db::table::kBlockReceiptsName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<silkworm::Bytes> { co_return *silkworm::from_hex("818400f6011a0004a0c8"); }));
-        EXPECT_CALL(transaction, walk(db::table::kLogsName, _, _, _)).WillOnce(Invoke([](Unused, Unused, Unused, auto w) -> Task<void> {
+        EXPECT_CALL(transaction, get_one(db::table::kBlockReceiptsName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<silkworm::Bytes> {
+            co_return *silkworm::from_hex("818400f6011a0004a0c8");
+        }));
+        auto cursor{std::make_shared<test::MockCursor>()};
+        EXPECT_CALL(transaction, cursor(db::table::kLogsName)).WillOnce(Invoke(
+            // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
+            [&cursor](Unused) -> Task<std::shared_ptr<ethdb::Cursor>> { co_return cursor; }));
+        EXPECT_CALL(*cursor, seek(_)).WillOnce(Invoke([](Unused) -> Task<KeyValue> {
             silkworm::Bytes key{};
             silkworm::Bytes value{*silkworm::from_hex(
                 "8683547753cfad258efbc52a9a1452e42ffbce9be486cb835820ddf252ad1be2c89b69c2b068fc"
@@ -390,8 +413,7 @@ TEST_CASE("read_raw_receipts") {
                 "1898bd2ef0835820efaf768237c22e140a862d5d375ad5c153479fac3f8bcf8b580a1651fd62c3"
                 "ef5820000000000000000000000000f13c666056048634109c1ecca6893da293c70da458200000"
                 "000000000000000000000214281cf15c1a66b51990e2e65e1f7b7c363318f6")};
-            w(key, value);
-            co_return;
+            co_return KeyValue{std::move(key), std::move(value)};
         }));
         auto result = boost::asio::co_spawn(pool, read_raw_receipts(transaction, block_number), boost::asio::use_future);
         // TODO(canepat): this case should fail instead of providing 1 receipt with 0 logs

silkworm/rpc/ethdb/base_transaction.cpp

@@ -49,56 +49,6 @@ Task<std::optional<Bytes>> BaseTransaction::get_both_range(const std::string& ta
     co_return value.substr(subkey.length());
 }
 
-Task<void> BaseTransaction::walk(const std::string& table, ByteView start_key, uint32_t fixed_bits, Walker w) {
-    const auto fixed_bytes = (fixed_bits + 7) / CHAR_BIT;
-    SILK_TRACE << "BaseTransaction::walk fixed_bits: " << fixed_bits << " fixed_bytes: " << fixed_bytes;
-    const auto shift_bits = fixed_bits & 7;
-    uint8_t mask{0xff};
-    if (shift_bits != 0) {
-        mask = static_cast<uint8_t>(0xff << (CHAR_BIT - shift_bits));
-    }
-    SILK_TRACE << "mask: " << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(mask) << std::dec;
-
-    const auto new_cursor = co_await cursor(table);
-    SILK_TRACE << "BaseTransaction::walk cursor_id: " << new_cursor->cursor_id();
-    auto kv_pair = co_await new_cursor->seek(start_key);
-    auto k = kv_pair.key;
-    auto v = kv_pair.value;
-    SILK_TRACE << "k: " << k << " v: " << v;
-    while (
-        !k.empty() &&
-        k.size() >= fixed_bytes &&
-        (fixed_bits == 0 || (k.compare(0, fixed_bytes - 1, start_key, 0, fixed_bytes - 1) == 0 && (k[fixed_bytes - 1] & mask) == (start_key[fixed_bytes - 1] & mask)))) {
-        const auto go_on = w(k, v);
-        if (!go_on) {
-            break;
-        }
-        kv_pair = co_await new_cursor->next();
-        k = kv_pair.key;
-        v = kv_pair.value;
-    }
-}
-
-Task<void> BaseTransaction::for_prefix(const std::string& table, ByteView prefix, Walker w) {
-    const auto new_cursor = co_await cursor(table);
-    SILK_TRACE << "BaseTransaction::for_prefix cursor_id: " << new_cursor->cursor_id() << " prefix: " << silkworm::to_hex(prefix);
-    auto kv_pair = co_await new_cursor->seek(prefix);
-    auto k = kv_pair.key;
-    auto v = kv_pair.value;
-    SILK_TRACE << "BaseTransaction::for_prefix k: " << k << " v: " << v;
-    while (k.substr(0, prefix.size()) == prefix) {
-        const auto go_on = w(k, v);
-        if (!go_on) {
-            break;
-        }
-        kv_pair = co_await new_cursor->next();
-        k = kv_pair.key;
-        v = kv_pair.value;
-        SILK_TRACE << "BaseTransaction::for_prefix k: " << k << " v: " << v;
-    }
-    co_return;
-}
-
 Task<silkworm::Bytes> BaseTransaction::get_one_impl_with_cache(const std::string& table, ByteView key) {
     if (state_cache_) {
         // Just PlainState and Code tables are present in state cache

silkworm/rpc/ethdb/base_transaction.hpp

@@ -35,10 +35,6 @@ class BaseTransaction : public Transaction {
 
     Task<std::optional<Bytes>> get_both_range(const std::string& table, ByteView key, ByteView subkey) override;
 
-    Task<void> walk(const std::string& table, ByteView start_key, uint32_t fixed_bits, Walker w) override;
-
-    Task<void> for_prefix(const std::string& table, ByteView prefix, Walker w) override;
-
   private:
     Task<silkworm::Bytes> get_one_impl_no_cache(const std::string& table, ByteView key);
     Task<silkworm::Bytes> get_one_impl_with_cache(const std::string& table, ByteView key);

silkworm/rpc/ethdb/bitmap.cpp

@@ -25,6 +25,7 @@
 
 #include <silkworm/core/types/evmc_bytes32.hpp>
 #include <silkworm/infra/common/log.hpp>
+#include <silkworm/rpc/ethdb/walk.hpp>
 
 namespace silkworm::rpc::ethdb::bitmap {
 
@@ -66,7 +67,7 @@ Task<Roaring> get(
         auto block = endian::load_big_u32(&k[k.size() - sizeof(uint32_t)]);
         return block < to_block;
     };
-    co_await tx.walk(table, from_key, gsl::narrow<uint32_t>(key.size() * CHAR_BIT), walker);
+    co_await walk(tx, table, from_key, gsl::narrow<uint32_t>(key.size() * CHAR_BIT), walker);
 
     auto result{fast_or(chunks.size(), chunks)};
     SILK_DEBUG << "result: " << result.toString();

silkworm/rpc/ethdb/transaction.hpp

@@ -63,10 +63,6 @@ class Transaction {
     virtual Task<silkworm::Bytes> get_one(const std::string& table, ByteView key) = 0;
 
     virtual Task<std::optional<Bytes>> get_both_range(const std::string& table, ByteView key, ByteView subkey) = 0;
-
-    virtual Task<void> walk(const std::string& table, ByteView start_key, uint32_t fixed_bits, Walker w) = 0;
-
-    virtual Task<void> for_prefix(const std::string& table, ByteView prefix, Walker w) = 0;
 };
 
 }  // namespace silkworm::rpc::ethdb

silkworm/rpc/ethdb/walk.cpp

@@ -0,0 +1,72 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "walk.hpp"
+
+#include <silkworm/infra/common/log.hpp>
+
+namespace silkworm::rpc::ethdb {
+
+Task<void> walk(Transaction& tx, const std::string& table, ByteView start_key, uint32_t fixed_bits, Walker w) {
+    const auto fixed_bytes = (fixed_bits + 7) / CHAR_BIT;
+    SILK_TRACE << "rpc::ethdb::walk fixed_bits: " << fixed_bits << " fixed_bytes: " << fixed_bytes;
+    const auto shift_bits = fixed_bits & 7;
+    uint8_t mask{0xff};
+    if (shift_bits != 0) {
+        mask = static_cast<uint8_t>(0xff << (CHAR_BIT - shift_bits));
+    }
+    SILK_TRACE << "mask: " << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(mask) << std::dec;
+
+    const auto new_cursor = co_await tx.cursor(table);
+    SILK_TRACE << "rpc::ethdb::walk cursor_id: " << new_cursor->cursor_id();
+    auto kv_pair = co_await new_cursor->seek(start_key);
+    auto k = kv_pair.key;
+    auto v = kv_pair.value;
+    SILK_TRACE << "k: " << k << " v: " << v;
+    while (
+        !k.empty() &&
+        k.size() >= fixed_bytes &&
+        (fixed_bits == 0 || (k.compare(0, fixed_bytes - 1, start_key, 0, fixed_bytes - 1) == 0 && (k[fixed_bytes - 1] & mask) == (start_key[fixed_bytes - 1] & mask)))) {
+        const auto go_on = w(k, v);
+        if (!go_on) {
+            break;
+        }
+        kv_pair = co_await new_cursor->next();
+        k = kv_pair.key;
+        v = kv_pair.value;
+    }
+}
+
+Task<void> for_prefix(Transaction& tx, const std::string& table, ByteView prefix, Walker w) {
+    const auto new_cursor = co_await tx.cursor(table);
+    SILK_TRACE << "rpc::ethdb::for_prefix cursor_id: " << new_cursor->cursor_id() << " prefix: " << silkworm::to_hex(prefix);
+    auto kv_pair = co_await new_cursor->seek(prefix);
+    auto k = kv_pair.key;
+    auto v = kv_pair.value;
+    SILK_TRACE << "rpc::ethdb::for_prefix k: " << k << " v: " << v;
+    while (k.substr(0, prefix.size()) == prefix) {
+        const auto go_on = w(k, v);
+        if (!go_on) {
+            break;
+        }
+        kv_pair = co_await new_cursor->next();
+        k = kv_pair.key;
+        v = kv_pair.value;
+        SILK_TRACE << "rpc::ethdb::for_prefix k: " << k << " v: " << v;
+    }
+}
+
+}  // namespace silkworm::rpc::ethdb

silkworm/rpc/ethdb/walk.hpp

@@ -0,0 +1,34 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <string>
+
+#include <silkworm/infra/concurrency/task.hpp>
+
+#include <silkworm/core/common/util.hpp>
+#include <silkworm/rpc/ethdb/transaction.hpp>
+
+namespace silkworm::rpc::ethdb {
+
+using Walker = std::function<bool(silkworm::Bytes&, silkworm::Bytes&)>;
+
+Task<void> walk(Transaction& tx, const std::string& table, ByteView start_key, uint32_t fixed_bits, Walker w);
+
+Task<void> for_prefix(Transaction& tx, const std::string& table, ByteView prefix, Walker w);
+
+}  // namespace silkworm::rpc::ethdb

silkworm/rpc/test_util/mock_transaction.hpp

@@ -46,8 +46,6 @@ class MockTransaction : public ethdb::Transaction {
     MOCK_METHOD((Task<silkworm::Bytes>), get_one, (const std::string&, silkworm::ByteView), (override));
     MOCK_METHOD((Task<std::optional<silkworm::Bytes>>), get_both_range,
                 (const std::string&, silkworm::ByteView, silkworm::ByteView), (override));
-    MOCK_METHOD((Task<void>), walk, (const std::string&, silkworm::ByteView, uint32_t, Walker), (override));
-    MOCK_METHOD((Task<void>), for_prefix, (const std::string&, silkworm::ByteView, Walker), (override));
 };
 
 }  // namespace silkworm::rpc::test