feat: GroveDB visualizer

.github/workflows/grovedb.yml

@@ -19,17 +19,23 @@ jobs:
           access_token: ${{ github.token }}
 
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
 
       - name: Setup Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
+          target: wasm32-unknown-unknown
 
       - name: Enable Rust cache
         uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: "false"
 
+      - name: Setup Trunk
+        uses: jetli/trunk-action@v0.5.0
+
       - run: cargo test --workspace --all-features
 
 
@@ -44,19 +50,25 @@ jobs:
 
       - name: Check out repo
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
 
       - name: Setup Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           default: true
           components: clippy
+          target: wasm32-unknown-unknown
 
       - name: Enable Rust cache
         uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: "false"
 
+      - name: Setup Trunk
+        uses: jetli/trunk-action@v0.5.0
+
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "grovedb/grovedbg"]
+	path = grovedb/grovedbg
+	url = https://github.com/dashpay/grovedbg

Cargo.toml

@@ -1,10 +1,13 @@
 [workspace]
+resolver = "2"
+exclude = ["grovedb/grovedbg"]
 members = [
     "costs",
     "grovedb",
     "merk",
     "node-grove",
     "storage",
     "visualize",
-    "path"
+    "path",
+    "grovedbg-types",
 ]

README.md

@@ -232,6 +232,24 @@ From here we can build:
 
 ```cargo build```
 
+## grovedbg
+
+There is a work in progress implementation of a debugger layer for GroveDB. To use this library with
+these capabilities enabled one needs to set a dependency with `grovedbg` feature.
+
+At build time this requires two environment dependencies:
+1. `wasm32-unknown-unknown` Rust toolchain;
+2. [trunk](https://trunkrs.dev/) utility.
+
+Then, to launch visualizer tool to observe the database structure inside of your browser on a port,
+let's say 10000, the following snippet should do:
+
+```rust
+    let db = Arc::new(GroveDb::open("db").unwrap());
+    db.start_visualzier(10000);
+```
+
+Just remember to use Arc because the HTTP server might outlast the GroveDB instance.
 
 ## Performance
 

grovedb/Cargo.toml

@@ -10,7 +10,6 @@ repository = "https://github.com/dashpay/grovedb"
 readme = "../README.md"
 documentation = "https://docs.rs/grovedb"
 
-
 [dependencies]
 grovedb-merk = { version = "1.0.0-rc.2", path = "../merk", optional = true, default-features = false }
 thiserror = { version = "1.0.59", optional = true }
@@ -26,8 +25,13 @@ nohash-hasher = { version = "0.2.0", optional = true }
 indexmap = { version = "2.2.6", optional = true }
 intmap = { version = "2.0.0", optional = true }
 grovedb-path = { version = "1.0.0-rc.2", path = "../path" }
+grovedbg-types = { path = "../grovedbg-types", optional = true }
+tokio = { version = "1.37.0", features = ["rt-multi-thread", "net"], optional = true }
+axum = { version = "0.7.5", features = ["macros"], optional = true }
+tower-http = { version = "0.5.2", features = ["fs"], optional = true }
 blake3 = "1.4.0"
 bitvec = "1"
+zip-extensions = { version ="0.6.2", optional = true }
 
 [dev-dependencies]
 rand = "0.8.5"
@@ -65,3 +69,16 @@ verify = [
     "integer-encoding",
 ]
 estimated_costs = ["full"]
+grovedbg = [
+    "grovedbg-types",
+    "tokio",
+    "full",
+    "grovedb-merk/grovedbg",
+    "axum",
+    "tower-http",
+    "zip-extensions",
+    "tempfile"
+]
+
+[build-dependencies]
+zip-extensions = "0.6.2"

grovedb/build.rs

@@ -0,0 +1,31 @@
+#[cfg(feature = "grovedbg")]
+fn main() {
+    use std::{
+        env,
+        path::PathBuf,
+        process::{Command, ExitStatus},
+    };
+
+    let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap());
+
+    if !Command::new("trunk")
+        .arg("build")
+        .arg("--release")
+        .arg("--dist")
+        .arg(&out_dir)
+        .arg("grovedbg/index.html")
+        .status()
+        .as_ref()
+        .map(ExitStatus::success)
+        .unwrap_or(false)
+    {
+        panic!("Error running `trunk build --release`");
+    }
+
+    let zip_file = out_dir.join("grovedbg.zip");
+    zip_extensions::write::zip_create_from_directory(&zip_file, &out_dir)
+        .expect("can't create a grovedbg zip archive");
+}
+
+#[cfg(not(feature = "grovedbg"))]
+fn main() {}

grovedb/src/debugger.rs

@@ -0,0 +1,170 @@
+//! GroveDB debugging support module.
+
+use std::{fs, net::Ipv4Addr, sync::Weak};
+
+use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Json, Router};
+use grovedb_merk::debugger::NodeDbg;
+use grovedb_path::SubtreePath;
+use grovedbg_types::{NodeFetchRequest, NodeUpdate, Path};
+use tokio::sync::mpsc::{self, Sender};
+use tower_http::services::ServeDir;
+
+use crate::{reference_path::ReferencePathType, GroveDb};
+
+const GROVEDBG_ZIP: [u8; include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")).len()] =
+    *include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip"));
+
+pub(super) fn start_visualizer(grovedb: Weak<GroveDb>, port: u16) {
+    std::thread::spawn(move || {
+        let grovedbg_tmp =
+            tempfile::tempdir().expect("cannot create tempdir for grovedbg contents");
+        let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip");
+        let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www");
+
+        fs::write(&grovedbg_zip, GROVEDBG_ZIP).expect("cannot crate grovedbg.zip");
+        zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www)
+            .expect("cannot extract grovedbg contents");
+
+        let (shutdown_send, mut shutdown_receive) = mpsc::channel::<()>(1);
+        let app = Router::new()
+            .route("/fetch_node", post(fetch_node))
+            .route("/fetch_root_node", post(fetch_root_node))
+            .fallback_service(ServeDir::new(grovedbg_www))
+            .with_state((shutdown_send, grovedb));
+
+        tokio::runtime::Runtime::new()
+            .unwrap()
+            .block_on(async move {
+                let listener = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, port))
+                    .await
+                    .expect("can't bind visualizer port");
+                axum::serve(listener, app)
+                    .with_graceful_shutdown(async move {
+                        shutdown_receive.recv().await;
+                    })
+                    .await
+                    .unwrap()
+            });
+    });
+}
+
+enum AppError {
+    Closed,
+    Any(String),
+}
+
+impl IntoResponse for AppError {
+    fn into_response(self) -> axum::response::Response {
+        match self {
+            AppError::Closed => {
+                (StatusCode::SERVICE_UNAVAILABLE, "GroveDB is closed").into_response()
+            }
+            AppError::Any(e) => (StatusCode::INTERNAL_SERVER_ERROR, e).into_response(),
+        }
+    }
+}
+
+impl<E: std::error::Error> From<E> for AppError {
+    fn from(err: E) -> Self {
+        Self::Any(err.to_string())
+    }
+}
+
+async fn fetch_node(
+    State((shutdown, grovedb)): State<(Sender<()>, Weak<GroveDb>)>,
+    Json(NodeFetchRequest { path, key }): Json<NodeFetchRequest>,
+) -> Result<Json<Option<NodeUpdate>>, AppError> {
+    let Some(db) = grovedb.upgrade() else {
+        shutdown.send(()).await.ok();
+        return Err(AppError::Closed);
+    };
+
+    let merk = db
+        .open_non_transactional_merk_at_path(path.as_slice().into(), None)
+        .unwrap()?;
+    let node = merk.get_node_dbg(&key)?;
+
+    if let Some(node) = node {
+        let node_update: NodeUpdate = node_to_update(path, node)?;
+        Ok(Json(Some(node_update)))
+    } else {
+        Ok(None.into())
+    }
+}
+
+async fn fetch_root_node(
+    State((shutdown, grovedb)): State<(Sender<()>, Weak<GroveDb>)>,
+) -> Result<Json<Option<NodeUpdate>>, AppError> {
+    let Some(db) = grovedb.upgrade() else {
+        shutdown.send(()).await.ok();
+        return Err(AppError::Closed);
+    };
+
+    let merk = db
+        .open_non_transactional_merk_at_path(SubtreePath::empty(), None)
+        .unwrap()?;
+
+    let node = merk.get_root_node_dbg()?;
+
+    if let Some(node) = node {
+        let node_update: NodeUpdate = node_to_update(Vec::new(), node)?;
+        Ok(Json(Some(node_update)))
+    } else {
+        Ok(None.into())
+    }
+}
+
+fn node_to_update(
+    path: Path,
+    NodeDbg {
+        key,
+        value,
+        left_child,
+        right_child,
+    }: NodeDbg,
+) -> Result<NodeUpdate, crate::Error> {
+    let grovedb_element = crate::Element::deserialize(&value)?;
+
+    let element = match grovedb_element {
+        crate::Element::Item(value, ..) => grovedbg_types::Element::Item { value },
+        crate::Element::Tree(root_key, ..) => grovedbg_types::Element::Subtree { root_key },
+        crate::Element::Reference(ReferencePathType::AbsolutePathReference(path), ..) => {
+            grovedbg_types::Element::AbsolutePathReference { path }
+        }
+        crate::Element::Reference(
+            ReferencePathType::UpstreamRootHeightReference(n_keep, path_append),
+            ..,
+        ) => grovedbg_types::Element::UpstreamRootHeightReference {
+            n_keep: n_keep.into(),
+            path_append,
+        },
+        crate::Element::Reference(
+            ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append),
+            ..,
+        ) => grovedbg_types::Element::UpstreamFromElementHeightReference {
+            n_remove: n_remove.into(),
+            path_append,
+        },
+        crate::Element::Reference(ReferencePathType::CousinReference(swap_parent), ..) => {
+            grovedbg_types::Element::CousinReference { swap_parent }
+        }
+        crate::Element::Reference(ReferencePathType::RemovedCousinReference(swap_parent), ..) => {
+            grovedbg_types::Element::RemovedCousinReference { swap_parent }
+        }
+        crate::Element::Reference(ReferencePathType::SiblingReference(sibling_key), ..) => {
+            grovedbg_types::Element::SiblingReference { sibling_key }
+        }
+        crate::Element::SumItem(value, _) => grovedbg_types::Element::SumItem { value },
+        crate::Element::SumTree(root_key, sum, _) => {
+            grovedbg_types::Element::Sumtree { root_key, sum }
+        }
+    };
+
+    Ok(NodeUpdate {
+        path,
+        key,
+        element,
+        left_child,
+        right_child,
+    })
+}

grovedb/src/lib.rs

@@ -140,11 +140,10 @@
 //! [Architectural Decision Records](https://github.com/dashpay/grovedb/tree/master/adr) or
 //! [Tutorial](https://www.grovedb.org/tutorials.html)
 
-#[cfg(feature = "full")]
-extern crate core;
-
 #[cfg(feature = "full")]
 pub mod batch;
+#[cfg(feature = "grovedbg")]
+pub mod debugger;
 #[cfg(any(feature = "full", feature = "verify"))]
 pub mod element;
 #[cfg(any(feature = "full", feature = "verify"))]
@@ -165,13 +164,18 @@
 mod tests;
 #[cfg(feature = "full")]
 mod util;
+#[cfg(any(feature = "full", feature = "verify"))]
 mod versioning;
 #[cfg(feature = "full")]
 mod visualize;
 
+#[cfg(feature = "grovedbg")]
+use std::sync::Arc;
 #[cfg(feature = "full")]
 use std::{collections::HashMap, option::Option::None, path::Path};
 
+#[cfg(feature = "grovedbg")]
+use debugger::start_visualizer;
 #[cfg(any(feature = "full", feature = "verify"))]
 pub use element::Element;
 #[cfg(feature = "full")]
@@ -222,6 +226,7 @@
 pub use crate::error::Error;
 #[cfg(feature = "full")]
 use crate::util::{root_merk_optional_tx, storage_context_optional_tx};
+#[cfg(any(feature = "full", feature = "verify"))]
 use crate::Error::MerkError;
 
 #[cfg(feature = "full")]
@@ -250,6 +255,13 @@
         Ok(GroveDb { db })
     }
 
+    #[cfg(feature = "grovedbg")]
+    // Start visualizer server for the GroveDB instance
+    pub fn start_visualzier(self: &Arc<Self>, port: u16) {
+        let weak = Arc::downgrade(self);
+        start_visualizer(weak, port);
+    }
+
     /// Uses raw iter to delete GroveDB key values pairs from rocksdb
     pub fn wipe(&self) -> Result<(), Error> {
         self.db.wipe()?;
@@ -842,7 +854,7 @@
     pub fn verify_grovedb(
         &self,
         transaction: TransactionArg,
     ) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
         if let Some(transaction) = transaction {
             let root_merk = self
                 .open_transactional_merk_at_path(SubtreePath::empty(), transaction, None)
@@ -868,7 +880,7 @@
         merk: Merk<S>,
         path: &SubtreePath<B>,
         batch: Option<&'db StorageBatch>,
     ) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
         let mut all_query = Query::new();
         all_query.insert_all();
 
@@ -938,7 +950,7 @@
         path: &SubtreePath<B>,
         batch: Option<&'db StorageBatch>,
         transaction: &Transaction,
     ) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
         let mut all_query = Query::new();
         all_query.insert_all();