db: move db to the root level (#1894)

cmake/copyright.cmake

@@ -62,7 +62,7 @@ list(FILTER SRC EXCLUDE REGEX [[silkworm/core/crypto/kzg\.cpp$]])
 list(FILTER SRC EXCLUDE REGEX [[silkworm/infra/concurrency/thread_pool\.hpp$]])
 list(FILTER SRC EXCLUDE REGEX [[silkworm/interfaces/]])
 list(FILTER SRC EXCLUDE REGEX [[silkworm/node/common/preverified_hashes_[a-z]+\.cpp$]])
-list(FILTER SRC EXCLUDE REGEX [[silkworm/snapshots/config/[a-z_]+\.hpp$]])
+list(FILTER SRC EXCLUDE REGEX [[silkworm/db/snapshots/config/[a-z_]+\.hpp$]])
 
 foreach(F IN LISTS SRC)
   check("${F}")

cmd/capi/execute.cpp

@@ -31,12 +31,12 @@
 #pragma GCC diagnostic pop
 
 #include <silkworm/capi/silkworm.h>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/mdbx.hpp>
 #include <silkworm/rpc/daemon.hpp>
-#include <silkworm/snapshots/repository.hpp>
 
 #include "../common/common.hpp"
 

cmd/common/db_checklist.cpp

@@ -19,10 +19,10 @@
 #include <regex>
 
 #include <silkworm/core/chain/genesis.hpp>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/genesis.hpp>
+#include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/log.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/genesis.hpp>
-#include <silkworm/node/db/stages.hpp>
 
 namespace silkworm::cmd::common {
 

cmd/common/db_max_readers_option.cpp

@@ -16,7 +16,7 @@
 
 #include "db_max_readers_option.hpp"
 
-#include <silkworm/node/db/mdbx.hpp>
+#include <silkworm/db/mdbx.hpp>
 
 namespace silkworm::cmd::common {
 

cmd/common/settings.hpp

@@ -16,11 +16,11 @@
 
 #pragma once
 
+#include <silkworm/db/snapshots/settings.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/node/settings.hpp>
 #include <silkworm/rpc/settings.hpp>
 #include <silkworm/sentry/settings.hpp>
-#include <silkworm/snapshots/settings.hpp>
 
 namespace silkworm::cmd::common {
 

cmd/common/snapshot_options.hpp

@@ -18,7 +18,7 @@
 
 #include <CLI/CLI.hpp>
 
-#include <silkworm/snapshots/settings.hpp>
+#include <silkworm/db/snapshots/settings.hpp>
 
 namespace silkworm::cmd::common {
 

cmd/dev/backend_kv_server.cpp

@@ -31,15 +31,15 @@
 #include <silkworm/buildinfo.h>
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/head_info.hpp>
+#include <silkworm/db/mdbx.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/awaitable_wait_for_one.hpp>
 #include <silkworm/infra/grpc/client/client_context_pool.hpp>
 #include <silkworm/node/backend/ethereum_backend.hpp>
 #include <silkworm/node/backend/remote/backend_kv_server.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/head_info.hpp>
-#include <silkworm/node/db/mdbx.hpp>
 #include <silkworm/sentry/eth/status_data_provider.hpp>
 #include <silkworm/sentry/grpc/client/sentry_client.hpp>
 #include <silkworm/sentry/multi_sentry_client.hpp>

cmd/dev/check_blockhashes.cpp

@@ -19,10 +19,10 @@
 #include <CLI/CLI.hpp>
 
 #include <silkworm/core/common/endian.hpp>
+#include <silkworm/db/tables.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/common/stopwatch.hpp>
-#include <silkworm/node/db/tables.hpp>
 
 using namespace silkworm;
 

cmd/dev/check_changes.cpp

@@ -23,12 +23,12 @@
 #include <silkworm/core/execution/processor.hpp>
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/buffer.hpp>
+#include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/buffer.hpp>
-#include <silkworm/snapshots/repository.hpp>
 
 using namespace evmc::literals;
 using namespace silkworm;

cmd/dev/check_hashstate.cpp

@@ -18,10 +18,10 @@
 
 #include <CLI/CLI.hpp>
 
+#include <silkworm/db/tables.hpp>
+#include <silkworm/db/util.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
-#include <silkworm/node/db/tables.hpp>
-#include <silkworm/node/db/util.hpp>
 
 using namespace silkworm;
 

cmd/dev/check_log_indices.cpp

@@ -29,15 +29,15 @@
 #include <silkworm/core/common/bytes_to_string.hpp>
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/bitmap.hpp>
+#include <silkworm/db/log_cbor.hpp>
+#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/tables.hpp>
+#include <silkworm/db/util.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
-#include <silkworm/node/db/bitmap.hpp>
-#include <silkworm/node/db/log_cbor.hpp>
-#include <silkworm/node/db/mdbx.hpp>
-#include <silkworm/node/db/tables.hpp>
-#include <silkworm/node/db/util.hpp>
 
 #include "../common/common.hpp"
 

cmd/dev/check_pow.cpp

@@ -24,12 +24,12 @@
 #include <silkworm/core/chain/config.hpp>
 #include <silkworm/core/common/endian.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/common/stopwatch.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/stages.hpp>
 
 namespace fs = std::filesystem;
 using namespace silkworm;

cmd/dev/check_senders.cpp

@@ -25,11 +25,11 @@
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/block_body_for_storage.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/decoding_exception.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
-#include <silkworm/node/db/stages.hpp>
 
 #include "../common/common.hpp"
 

cmd/dev/check_tx_lookup.cpp

@@ -19,12 +19,12 @@
 #include <silkworm/core/common/endian.hpp>
 #include <silkworm/core/common/util.hpp>
 #include <silkworm/core/types/block_body_for_storage.hpp>
+#include <silkworm/db/etl/collector.hpp>
+#include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/decoding_exception.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
-#include <silkworm/node/db/etl/collector.hpp>
-#include <silkworm/node/db/stages.hpp>
 
 using namespace silkworm;
 

cmd/dev/db_toolbox.cpp

@@ -41,16 +41,16 @@
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/block_body_for_storage.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/genesis.hpp>
+#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/prune_mode.hpp>
+#include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/decoding_exception.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/common/stopwatch.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
-#include <silkworm/node/db/genesis.hpp>
-#include <silkworm/node/db/mdbx.hpp>
-#include <silkworm/node/db/prune_mode.hpp>
-#include <silkworm/node/db/stages.hpp>
 #include <silkworm/node/stagedsync/execution_pipeline.hpp>
 #include <silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.hpp>
 

cmd/dev/embed_toml.cpp

@@ -85,7 +85,7 @@ int main(int argc, char* argv[]) {
             output << "#pragma once\n\n";
             output << "#include <array>\n";
             output << "#include <string_view>\n\n";
-            output << "#include <silkworm/snapshots/entry.hpp>\n\n";
+            output << "#include <silkworm/db/snapshots/entry.hpp>\n\n";
             output << "namespace silkworm::snapshots {\n\n";
             output << "inline constexpr std::array k" << snapshot_name << "Snapshots{\n";
             for (auto&& [key, value] : table) {

cmd/dev/scan_txs.cpp

@@ -21,9 +21,9 @@
 
 #include <silkworm/core/execution/execution.hpp>
 #include <silkworm/core/protocol/rule_set.hpp>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/buffer.hpp>
 #include <silkworm/infra/common/directories.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/buffer.hpp>
 
 int main(int argc, char* argv[]) {
     CLI::App app{"Executes Ethereum blocks and scans txs for errored txs"};

cmd/dev/snapshots.cpp

@@ -31,14 +31,14 @@
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/block_body_for_storage.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
+#include <silkworm/db/snapshot_sync.hpp>
+#include <silkworm/db/snapshots/bittorrent/client.hpp>
+#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/snapshots/repository.hpp>
+#include <silkworm/db/snapshots/seg/seg_zip.hpp>
+#include <silkworm/db/snapshots/snapshot.hpp>
 #include <silkworm/infra/common/ensure.hpp>
 #include <silkworm/infra/common/log.hpp>
-#include <silkworm/node/snapshot_sync.hpp>
-#include <silkworm/snapshots/bittorrent/client.hpp>
-#include <silkworm/snapshots/index.hpp>
-#include <silkworm/snapshots/repository.hpp>
-#include <silkworm/snapshots/seg/seg_zip.hpp>
-#include <silkworm/snapshots/snapshot.hpp>
 
 #include "../common/common.hpp"
 #include "../common/shutdown_signal.hpp"

cmd/silkworm.cpp

@@ -29,11 +29,11 @@
 #include <boost/asio/use_future.hpp>
 
 #include <silkworm/buildinfo.h>
+#include <silkworm/db/head_info.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/concurrency/awaitable_wait_for_all.hpp>
 #include <silkworm/infra/concurrency/awaitable_wait_for_one.hpp>
 #include <silkworm/infra/grpc/client/client_context_pool.hpp>
-#include <silkworm/node/db/head_info.hpp>
 #include <silkworm/node/node.hpp>
 #include <silkworm/sentry/eth/status_data_provider.hpp>
 #include <silkworm/sentry/sentry_client_factory.hpp>

cmd/test/backend_kv_test.cpp

@@ -33,12 +33,12 @@
 #include <silkworm/core/common/assert.hpp>
 #include <silkworm/core/common/util.hpp>
 #include <silkworm/core/types/address.hpp>
+#include <silkworm/db/tables.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/grpc/common/conversion.hpp>
 #include <silkworm/infra/grpc/common/util.hpp>
 #include <silkworm/interfaces/remote/ethbackend.grpc.pb.h>
 #include <silkworm/interfaces/remote/kv.grpc.pb.h>
-#include <silkworm/node/db/tables.hpp>
 
 #include "../common/shutdown_signal.hpp"
 

docs/CONTRIBUTING.md

@@ -77,7 +77,7 @@ If you need to update the list of builtin snapshots in Silkworm, the following p
 * update `erigon-snapshot` submodule to the new commit
 * generate the embedded C++ code bindings for predefined snapshots by executing from project home folder:
 ```
-<build_folder>/cmd/dev/embed_toml -i third_party/erigon-snapshot -o silkworm/snapshots/config
+<build_folder>/cmd/dev/embed_toml -i third_party/erigon-snapshot -o silkworm/db/snapshots/config
 ```
 
 

silkworm/CMakeLists.txt

@@ -20,11 +20,11 @@ add_subdirectory(core)
 if(NOT SILKWORM_CORE_ONLY)
   add_subdirectory(interfaces)
   add_subdirectory(infra)
+  add_subdirectory(db)
+  add_subdirectory(rpc)
   add_subdirectory(sentry)
-  add_subdirectory(snapshots)
   add_subdirectory(sync)
   add_subdirectory(node)
-  add_subdirectory(rpc)
   add_subdirectory(capi)
 endif()
 

silkworm/capi/instance.hpp

@@ -22,9 +22,9 @@
 
 #include <boost/asio/cancellation_signal.hpp>
 
+#include <silkworm/db/snapshots/repository.hpp>
 #include <silkworm/infra/concurrency/context_pool_settings.hpp>
 #include <silkworm/rpc/daemon.hpp>
-#include <silkworm/snapshots/repository.hpp>
 
 struct SilkwormInstance {
     silkworm::concurrency::ContextPoolSettings context_pool_settings;

silkworm/capi/silkworm.cpp

@@ -31,16 +31,16 @@
 #include <silkworm/core/chain/config.hpp>
 #include <silkworm/core/execution/call_tracer.hpp>
 #include <silkworm/core/execution/execution.hpp>
+#include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/buffer.hpp>
+#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/bounded_buffer.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/infra/common/stopwatch.hpp>
 #include <silkworm/infra/concurrency/signal_handler.hpp>
 #include <silkworm/infra/concurrency/thread_pool.hpp>
-#include <silkworm/node/db/access_layer.hpp>
-#include <silkworm/node/db/buffer.hpp>
-#include <silkworm/node/db/stages.hpp>
-#include <silkworm/snapshots/index.hpp>
 
 #include "instance.hpp"
 

silkworm/capi/silkworm_test.cpp

@@ -22,12 +22,12 @@
 #include <catch2/catch.hpp>
 
 #include <silkworm/core/trie/vector_root.hpp>
+#include <silkworm/db/mdbx.hpp>
+#include <silkworm/db/snapshots/index.hpp>
+#include <silkworm/db/snapshots/snapshot.hpp>
+#include <silkworm/db/snapshots/test_util/common.hpp>
 #include <silkworm/infra/test_util/log.hpp>
-#include <silkworm/node/db/mdbx.hpp>
 #include <silkworm/rpc/test/api_test_database.hpp>
-#include <silkworm/snapshots/index.hpp>
-#include <silkworm/snapshots/snapshot.hpp>
-#include <silkworm/snapshots/test_util/common.hpp>
 
 namespace silkworm {
 

silkworm/node/db/CMakeLists.txt → silkworm/db/CMakeLists.txt

@@ -17,8 +17,10 @@
 include("${SILKWORM_MAIN_DIR}/cmake/common/targets.cmake")
 
 add_subdirectory(etl)
+add_subdirectory(snapshots)
 
 find_package(absl REQUIRED)
+find_package(magic_enum REQUIRED)
 find_package(Microsoft.GSL REQUIRED)
 find_package(nlohmann_json REQUIRED)
 find_package(roaring REQUIRED)
@@ -36,8 +38,10 @@ set(LIBS_PUBLIC
 # cmake-format: off
 set(LIBS_PRIVATE
     cborcpp
+    magic_enum::magic_enum
     Microsoft.GSL::GSL
     nlohmann_json::nlohmann_json
+    silkworm_db_etl
     silkworm_infra
     silkworm_snapshots
 )
@@ -47,5 +51,5 @@ silkworm_library(
   silkworm_db
   PUBLIC ${LIBS_PUBLIC}
   PRIVATE ${LIBS_PRIVATE}
-  EXCLUDE_REGEX "node/db/etl"
+  EXCLUDE_REGEX "db/etl" "db/snapshots"
 )