snapshots: index builder refactorings

cmd/dev/snapshots.cpp

@@ -33,10 +33,14 @@
 #include <silkworm/core/types/evmc_bytes32.hpp>
 #include <silkworm/db/snapshot_sync.hpp>
 #include <silkworm/db/snapshots/bittorrent/client.hpp>
-#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/snapshots/body_index.hpp>
+#include <silkworm/db/snapshots/header_index.hpp>
+#include <silkworm/db/snapshots/index_builder.hpp>
 #include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/db/snapshots/seg/seg_zip.hpp>
 #include <silkworm/db/snapshots/snapshot.hpp>
+#include <silkworm/db/snapshots/txn_index.hpp>
+#include <silkworm/db/snapshots/txn_to_block_index.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/common/log.hpp>
 
@@ -283,18 +287,22 @@ void create_index(const SnapSettings& settings, int repetitions) {
         for (int i{0}; i < repetitions; ++i) {
             switch (snap_file->type()) {
                 case SnapshotType::headers: {
-                    HeaderIndex index{*snap_file};
+                    auto index = HeaderIndex::make(*snap_file);
                     index.build();
                     break;
                 }
                 case SnapshotType::bodies: {
-                    BodyIndex index{*snap_file};
+                    auto index = BodyIndex::make(*snap_file);
                     index.build();
                     break;
                 }
                 case SnapshotType::transactions: {
-                    TransactionIndex index{*snap_file};
+                    auto bodies_segment_path = TransactionIndex::bodies_segment_path(*snap_file);
+                    auto index = TransactionIndex::make(bodies_segment_path, *snap_file);
                     index.build();
+
+                    auto index_hash_to_block = TransactionToBlockIndex::make(bodies_segment_path, *snap_file);
+                    index_hash_to_block.build();
                     break;
                 }
                 default: {

silkworm/capi/silkworm.cpp

@@ -19,6 +19,7 @@
 #include <charconv>
 #include <chrono>
 #include <memory>
+#include <optional>
 #include <string>
 #include <string_view>
 #include <utility>
@@ -33,7 +34,11 @@
 #include <silkworm/core/execution/execution.hpp>
 #include <silkworm/db/access_layer.hpp>
 #include <silkworm/db/buffer.hpp>
-#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/snapshots/body_index.hpp>
+#include <silkworm/db/snapshots/header_index.hpp>
+#include <silkworm/db/snapshots/index_builder.hpp>
+#include <silkworm/db/snapshots/txn_index.hpp>
+#include <silkworm/db/snapshots/txn_to_block_index.hpp>
 #include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/bounded_buffer.hpp>
 #include <silkworm/infra/common/directories.hpp>
@@ -232,7 +237,7 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc
         return SILKWORM_INVALID_HANDLE;
     }
 
-    std::vector<std::shared_ptr<snapshots::Index>> needed_indexes;
+    std::vector<std::shared_ptr<snapshots::IndexBuilder>> needed_indexes;
     for (size_t i = 0; i < len; i++) {
         struct SilkwormMemoryMappedFile* snapshot = snapshots[i];
         if (!snapshot) {
@@ -245,25 +250,41 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc
             return SILKWORM_INVALID_PATH;
         }
 
-        std::shared_ptr<snapshots::Index> index;
+        std::shared_ptr<snapshots::IndexBuilder> index;
         switch (snapshot_path->type()) {
             case snapshots::SnapshotType::headers: {
-                index = std::make_shared<snapshots::HeaderIndex>(*snapshot_path, snapshot_region);
+                index = std::make_shared<snapshots::IndexBuilder>(snapshots::HeaderIndex::make(*snapshot_path, snapshot_region));
+                needed_indexes.push_back(index);
                 break;
             }
             case snapshots::SnapshotType::bodies: {
-                index = std::make_shared<snapshots::BodyIndex>(*snapshot_path, snapshot_region);
+                index = std::make_shared<snapshots::IndexBuilder>(snapshots::BodyIndex::make(*snapshot_path, snapshot_region));
+                needed_indexes.push_back(index);
                 break;
             }
             case snapshots::SnapshotType::transactions: {
-                index = std::make_shared<snapshots::TransactionIndex>(*snapshot_path, snapshot_region);
+                auto bodies_segment_path = snapshots::TransactionIndex::bodies_segment_path(*snapshot_path);
+                auto bodies_file = std::find_if(snapshots, snapshots + len, [&](SilkwormMemoryMappedFile* file) -> bool {
+                    return snapshots::SnapshotPath::parse(file->file_path) == bodies_segment_path;
+                });
+
+                if (bodies_file < snapshots + len) {
+                    auto bodies_segment_region = make_region(**bodies_file);
+
+                    index = std::make_shared<snapshots::IndexBuilder>(snapshots::TransactionIndex::make(
+                        bodies_segment_path, bodies_segment_region, *snapshot_path, snapshot_region));
+                    needed_indexes.push_back(index);
+
+                    index = std::make_shared<snapshots::IndexBuilder>(snapshots::TransactionToBlockIndex::make(
+                        bodies_segment_path, bodies_segment_region, *snapshot_path, snapshot_region));
+                    needed_indexes.push_back(index);
+                }
                 break;
             }
             default: {
                 SILKWORM_ASSERT(false);
             }
         }
-        needed_indexes.push_back(index);
     }
 
     if (needed_indexes.size() < kNeededIndexesToBuildInParallel) {

silkworm/capi/silkworm_test.cpp

@@ -23,9 +23,14 @@
 
 #include <silkworm/core/trie/vector_root.hpp>
 #include <silkworm/db/mdbx/mdbx.hpp>
-#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/snapshots/body_index.hpp>
+#include <silkworm/db/snapshots/header_index.hpp>
+#include <silkworm/db/snapshots/index_builder.hpp>
 #include <silkworm/db/snapshots/snapshot.hpp>
 #include <silkworm/db/snapshots/test_util/common.hpp>
+#include <silkworm/db/snapshots/txn_index.hpp>
+#include <silkworm/db/snapshots/txn_to_block_index.hpp>
+#include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/test_util/log.hpp>
 #include <silkworm/rpc/test/api_test_database.hpp>
 
@@ -34,6 +39,8 @@ namespace silkworm {
 namespace snapshot_test = snapshots::test_util;
 
 struct CApiTest : public rpc::test::TestDatabaseContext {
+    TemporaryDirectory tmp_dir;
+
   private:
     // TODO(canepat) remove test_util::StreamSwap objects when C API settings include log level
     std::stringstream string_cout, string_cerr;
@@ -635,25 +642,29 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_perpetual multiple bloc
 }
 
 TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_snapshot", "[silkworm][capi]") {
-    snapshot_test::SampleHeaderSnapshotFile valid_header_snapshot{};
+    snapshot_test::SampleHeaderSnapshotFile valid_header_snapshot{tmp_dir.path()};
     snapshot_test::SampleHeaderSnapshotPath header_snapshot_path{valid_header_snapshot.path()};
-    snapshot_test::SampleBodySnapshotFile valid_body_snapshot{};
+    snapshot_test::SampleBodySnapshotFile valid_body_snapshot{tmp_dir.path()};
     snapshot_test::SampleBodySnapshotPath body_snapshot_path{valid_body_snapshot.path()};
-    snapshot_test::SampleTransactionSnapshotFile valid_tx_snapshot{};
+    snapshot_test::SampleTransactionSnapshotFile valid_tx_snapshot{tmp_dir.path()};
     snapshot_test::SampleTransactionSnapshotPath tx_snapshot_path{valid_tx_snapshot.path()};
 
-    snapshots::HeaderIndex header_index{header_snapshot_path};
+    auto header_index = snapshots::HeaderIndex::make(header_snapshot_path);
     REQUIRE_NOTHROW(header_index.build());
     snapshots::HeaderSnapshot header_snapshot{header_snapshot_path};
     header_snapshot.reopen_segment();
     header_snapshot.reopen_index();
-    snapshots::BodyIndex body_index{body_snapshot_path};
+
+    auto body_index = snapshots::BodyIndex::make(body_snapshot_path);
     REQUIRE_NOTHROW(body_index.build());
     snapshots::BodySnapshot body_snapshot{body_snapshot_path};
     body_snapshot.reopen_segment();
     body_snapshot.reopen_index();
-    snapshots::TransactionIndex tx_index{tx_snapshot_path};
-    REQUIRE_NOTHROW(tx_index.build());
+
+    auto tx_index = snapshots::TransactionIndex::make(body_snapshot_path, tx_snapshot_path);
+    tx_index.build();
+    auto tx_index_hash_to_block = snapshots::TransactionToBlockIndex::make(body_snapshot_path, tx_snapshot_path);
+    tx_index_hash_to_block.build();
     snapshots::TransactionSnapshot tx_snapshot{tx_snapshot_path};
     tx_snapshot.reopen_segment();
     tx_snapshot.reopen_index();

silkworm/db/snapshot_sync.cpp

@@ -24,7 +24,7 @@
 #include <silkworm/core/types/hash.hpp>
 #include <silkworm/db/mdbx/etl_mdbx_collector.hpp>
 #include <silkworm/db/snapshots/config.hpp>
-#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/snapshots/index_builder.hpp>
 #include <silkworm/db/snapshots/path.hpp>
 #include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/ensure.hpp>

silkworm/db/snapshots/body_index.cpp

@@ -0,0 +1,29 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "body_index.hpp"
+
+#include <silkworm/db/snapshots/seg/common/varint.hpp>
+
+namespace silkworm::snapshots {
+
+Bytes BodyIndex::KeyFactory::make(ByteView /*key_data*/, uint64_t i) {
+    Bytes uint64_buffer;
+    seg::varint::encode(uint64_buffer, i);
+    return uint64_buffer;
+}
+
+}  // namespace silkworm::snapshots

silkworm/db/snapshots/body_index.hpp

@@ -0,0 +1,54 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+
+#include <silkworm/core/common/bytes.hpp>
+#include <silkworm/infra/common/memory_mapped_file.hpp>
+
+#include "index_builder.hpp"
+#include "path.hpp"
+
+namespace silkworm::snapshots {
+
+class BodyIndex {
+  public:
+    static IndexBuilder make(SnapshotPath segment_path, std::optional<MemoryMappedRegion> segment_region = std::nullopt) {
+        auto descriptor = make_descriptor(segment_path);
+        auto query = std::make_unique<DecompressorIndexInputDataQuery>(std::move(segment_path), segment_region);
+        return IndexBuilder{std::move(descriptor), std::move(query)};
+    }
+
+    struct KeyFactory : IndexKeyFactory {
+        ~KeyFactory() override = default;
+        Bytes make(ByteView key_data, uint64_t i) override;
+    };
+
+  private:
+    static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
+        return {
+            .index_file = segment_path.index_file(),
+            .key_factory = std::make_unique<KeyFactory>(),
+            .base_data_id = segment_path.block_from(),
+        };
+    }
+};
+
+}  // namespace silkworm::snapshots

silkworm/db/snapshots/header_index.cpp

@@ -0,0 +1,35 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "header_index.hpp"
+
+#include <silkworm/core/common/util.hpp>
+#include <silkworm/infra/common/ensure.hpp>
+
+namespace silkworm::snapshots {
+
+Bytes HeaderIndex::KeyFactory::make(ByteView key_data, uint64_t i) {
+    auto word = key_data;
+    ensure(!word.empty(), [&]() { return "HeaderIndex: word empty i=" + std::to_string(i); });
+    const uint8_t first_hash_byte{word[0]};
+    const ByteView rlp_encoded_header{word.data() + 1, word.size() - 1};
+    const ethash::hash256 hash = keccak256(rlp_encoded_header);
+    ensure(hash.bytes[0] == first_hash_byte,
+           [&]() { return "HeaderIndex: invalid prefix=" + to_hex(first_hash_byte) + " hash=" + to_hex(hash.bytes); });
+    return Bytes{ByteView{hash.bytes}};
+}
+
+}  // namespace silkworm::snapshots

silkworm/db/snapshots/header_index.hpp

@@ -0,0 +1,54 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+
+#include <silkworm/core/common/bytes.hpp>
+#include <silkworm/infra/common/memory_mapped_file.hpp>
+
+#include "index_builder.hpp"
+#include "path.hpp"
+
+namespace silkworm::snapshots {
+
+class HeaderIndex {
+  public:
+    static IndexBuilder make(SnapshotPath segment_path, std::optional<MemoryMappedRegion> segment_region = std::nullopt) {
+        auto descriptor = make_descriptor(segment_path);
+        auto query = std::make_unique<DecompressorIndexInputDataQuery>(std::move(segment_path), segment_region);
+        return IndexBuilder{std::move(descriptor), std::move(query)};
+    }
+
+    struct KeyFactory : IndexKeyFactory {
+        ~KeyFactory() override = default;
+        Bytes make(ByteView key_data, uint64_t i) override;
+    };
+
+  private:
+    static IndexDescriptor make_descriptor(const SnapshotPath& segment_path) {
+        return {
+            .index_file = segment_path.index_file(),
+            .key_factory = std::make_unique<KeyFactory>(),
+            .base_data_id = segment_path.block_from(),
+        };
+    }
+};
+
+}  // namespace silkworm::snapshots