From 04fbf2562f48a71ed76bccae7374fe455bc93e6f Mon Sep 17 00:00:00 2001 From: Jordan Santell Date: Tue, 5 Sep 2023 13:17:13 -0700 Subject: [PATCH] feat: Initial MVP of RocksDbStorage. --- Cargo.lock | 179 +++++++++++++++++- rust/noosphere-cli/Cargo.toml | 5 +- rust/noosphere-storage/Cargo.toml | 6 + .../src/implementation/mod.rs | 6 + .../src/implementation/rocks_db.rs | 139 ++++++++++++++ rust/noosphere/Cargo.toml | 2 + rust/noosphere/src/platform.rs | 45 +++-- rust/noosphere/src/storage.rs | 29 +-- 8 files changed, 378 insertions(+), 33 deletions(-) create mode 100644 rust/noosphere-storage/src/implementation/rocks_db.rs diff --git a/Cargo.lock b/Cargo.lock index 49de2def0..7dc7dcd2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -566,6 +566,27 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "prettyplease 0.2.12", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.28", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -715,6 +736,17 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cbor4ii" version = "0.2.14" @@ -730,6 +762,7 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ + "jobserver", "libc", ] @@ -744,6 +777,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -829,6 +871,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.3.19" @@ -1867,6 +1920,12 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.13" @@ -2407,6 +2466,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.64" @@ -2434,6 +2502,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.147" @@ -2530,6 +2604,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libm" version = "0.1.4" @@ -2933,6 +3017,33 @@ dependencies = [ "yamux", ] +[[package]] +name = "librocksdb-sys" +version = "0.11.0+8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2985,6 +3096,16 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "mac" version = "0.1.1" @@ -3653,6 +3774,7 @@ dependencies = [ "libipld-cbor", "libipld-core", "rexie", + "rocksdb", "serde", "sled", "tokio", @@ -3924,6 +4046,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem" version = "1.1.1" @@ -4020,6 +4148,12 @@ dependencies = [ "spki 0.7.2", ] +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "pkg-version" version = "1.0.0" @@ -4112,6 +4246,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prettyplease" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +dependencies = [ + "proc-macro2", + "syn 2.0.28", +] + [[package]] name = "primeorder" version = "0.13.2" @@ -4525,6 +4669,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "rocksdb" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rsa" version = "0.9.2" @@ -4747,7 +4901,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b08f58cf71a58bda5734758eb20051cdb66c06c9243badbc45092ced1be834df" dependencies = [ "macro_rules_attribute", - "prettyplease", + "prettyplease 0.1.25", "proc-macro2", "quote", "syn 1.0.109", @@ -5040,6 +5194,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -5970,6 +6130,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -6765,3 +6931,14 @@ dependencies = [ "quote", "syn 2.0.28", ] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/rust/noosphere-cli/Cargo.toml b/rust/noosphere-cli/Cargo.toml index afec7ad12..b4e6a772b 100644 --- a/rust/noosphere-cli/Cargo.toml +++ b/rust/noosphere-cli/Cargo.toml @@ -17,7 +17,10 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [features] +default = [] test_kubo = [] +experimental-rocksdb = ["noosphere/experimental-rocksdb"] +experimental-rocksdb-mt = ["noosphere/experimental-rocksdb-mt"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -66,4 +69,4 @@ libipld-core = { workspace = true } libipld-cbor = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] -wasm-bindgen = { workspace = true } +wasm-bindgen = { workspace = true } \ No newline at end of file diff --git a/rust/noosphere-storage/Cargo.toml b/rust/noosphere-storage/Cargo.toml index b3ad41872..e2a602c6b 100644 --- a/rust/noosphere-storage/Cargo.toml +++ b/rust/noosphere-storage/Cargo.toml @@ -40,6 +40,7 @@ wasm-bindgen-test = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] sled = "~0.34" tokio = { workspace = true, features = ["full"] } +rocksdb = { version = "0.21.0", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] tokio = { workspace = true, features = ["sync", "macros"] } @@ -53,3 +54,8 @@ features = [ "Window", "DedicatedWorkerGlobalScope", ] + +[features] +default = [] +rocksdb = ["dep:rocksdb"] +rocksdb-mt = ["dep:rocksdb"] diff --git a/rust/noosphere-storage/src/implementation/mod.rs b/rust/noosphere-storage/src/implementation/mod.rs index f55567d7a..f1d8fca79 100644 --- a/rust/noosphere-storage/src/implementation/mod.rs +++ b/rust/noosphere-storage/src/implementation/mod.rs @@ -10,6 +10,12 @@ mod sled; #[cfg(not(target_arch = "wasm32"))] pub use self::sled::*; +#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] +mod rocks_db; + +#[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] +pub use rocks_db::*; + #[cfg(target_arch = "wasm32")] mod indexed_db; diff --git a/rust/noosphere-storage/src/implementation/rocks_db.rs b/rust/noosphere-storage/src/implementation/rocks_db.rs new file mode 100644 index 000000000..4b6dddba1 --- /dev/null +++ b/rust/noosphere-storage/src/implementation/rocks_db.rs @@ -0,0 +1,139 @@ +use crate::{storage::Storage, store::Store, SPHERE_DB_STORE_NAMES}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use rocksdb::{ColumnFamilyDescriptor, DBWithThreadMode, Options}; +use std::{path::PathBuf, sync::Arc}; + +#[cfg(not(feature = "rocksdb-mt"))] +type DbInner = DBWithThreadMode; +#[cfg(not(feature = "rocksdb-mt"))] +type ColumnType<'a> = &'a rocksdb::ColumnFamily; +#[cfg(feature = "rocksdb-mt")] +type DbInner = DBWithThreadMode; +#[cfg(feature = "rocksdb-mt")] +type ColumnType<'a> = Arc>; + +/// A RocksDB implementation of [Storage]. +/// +/// Caveats: +/// * Values are limited to 4GB(?) [https://github.com/facebook/rocksdb/wiki/Basic-Operations#reads] +/// * Potential workarounds of cf_handles not being [Sync] [https://github.com/rust-rocksdb/rust-rocksdb/issues/407] +#[derive(Clone, Debug)] +pub struct RocksDbStorage { + db: Arc, +} + +impl RocksDbStorage { + pub fn new(path: PathBuf) -> Result { + let db = Arc::new(RocksDbStorage::open_db(path)?); + Ok(RocksDbStorage { db }) + } + + async fn get_store(&self, name: &str) -> Result { + if SPHERE_DB_STORE_NAMES + .iter() + .find(|val| **val == name) + .is_none() + { + return Err(anyhow!("No such store named {}", name)); + } + + RocksDbStore::new(self.db.clone(), name.to_owned()) + } + + fn open_db(path: PathBuf) -> Result { + std::fs::create_dir_all(&path)?; + let mut cfs: Vec = Vec::with_capacity(SPHERE_DB_STORE_NAMES.len()); + + for store_name in SPHERE_DB_STORE_NAMES { + // https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + let cf_opts = Options::default(); + cfs.push(ColumnFamilyDescriptor::new(*store_name, cf_opts)); + } + + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + + Ok(DbInner::open_cf_descriptors(&db_opts, path, cfs)?) + } +} + +#[async_trait] +impl Storage for RocksDbStorage { + type BlockStore = RocksDbStore; + type KeyValueStore = RocksDbStore; + + async fn get_block_store(&self, name: &str) -> Result { + self.get_store(name).await + } + + async fn get_key_value_store(&self, name: &str) -> Result { + self.get_store(name).await + } +} + +#[derive(Clone)] +pub struct RocksDbStore { + name: String, + db: Arc, +} + +impl RocksDbStore { + pub fn new(db: Arc, name: String) -> Result { + Ok(RocksDbStore { db, name }) + } + + /// Returns the column family handle. Unfortunately generated on every call + /// due to not being `Sync`, potentially `unsafe` alternatives: + /// https://github.com/rust-rocksdb/rust-rocksdb/issues/407 + fn cf_handle<'a>(&'a self) -> Result { + self.db + .cf_handle(&self.name) + .ok_or_else(move || anyhow!("Could not open handle for {}", self.name)) + } +} + +#[async_trait] +impl Store for RocksDbStore { + async fn read(&self, key: &[u8]) -> Result>> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-mt")] + let cf = &cf; + Ok(self.db.get_cf(cf, key)?) + } + + async fn write(&mut self, key: &[u8], bytes: &[u8]) -> Result>> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-mt")] + let cf = &cf; + // @TODO atomicize this + let old_bytes = self.db.get_cf(cf, key)?; + self.db.put_cf(cf, key, bytes)?; + Ok(old_bytes) + } + + async fn remove(&mut self, key: &[u8]) -> Result>> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-mt")] + let cf = &cf; + // @TODO atomicize this + let old_bytes = self.db.get_cf(cf, key)?; + self.db.delete_cf(cf, key)?; + Ok(old_bytes) + } + + async fn flush(&self) -> Result<()> { + let cf = self.cf_handle()?; + #[cfg(feature = "rocksdb-mt")] + let cf = &cf; + self.db.flush_cf(cf)?; + Ok(()) + } +} + +impl Drop for RocksDbStorage { + fn drop(&mut self) { + let _ = self.db.flush(); + } +} diff --git a/rust/noosphere/Cargo.toml b/rust/noosphere/Cargo.toml index 5d0227fdc..3ab3fe549 100644 --- a/rust/noosphere/Cargo.toml +++ b/rust/noosphere/Cargo.toml @@ -19,6 +19,8 @@ crate-type = ["rlib", "staticlib", "cdylib"] default = [] headers = ["safer-ffi/headers"] ipfs-storage = ["noosphere-ipfs"] +experimental-rocksdb = ["noosphere-storage/rocksdb"] +experimental-rocksdb-mt = ["noosphere-storage/rocksdb-mt"] [dependencies] anyhow = { workspace = true } diff --git a/rust/noosphere/src/platform.rs b/rust/noosphere/src/platform.rs index 4579bd8fb..dd0fb2d90 100644 --- a/rust/noosphere/src/platform.rs +++ b/rust/noosphere/src/platform.rs @@ -3,12 +3,17 @@ ///! secure key management. This module lays out the concrete strategies we will ///! use on a per-platform basis. +#[cfg(all( + feature = "experimental-rocksdb-mt", + not(feature = "experimental-rocksdb") +))] +compile_error!("feature \"experimental-rocksdb-mt\" requires feature \"experimental-rocksdb\""); + #[cfg(all( any(target_arch = "aarch64", target_arch = "x86_64"), target_vendor = "apple" ))] mod inner { - use noosphere_storage::SledStorage; use ucan_key_support::ed25519::Ed25519KeyMaterial; use crate::key::InsecureKeyStorage; @@ -18,14 +23,16 @@ mod inner { pub type PlatformKeyMaterial = Ed25519KeyMaterial; pub type PlatformKeyStorage = InsecureKeyStorage; - #[cfg(not(feature = "ipfs-storage"))] - pub type PlatformStorage = SledStorage; - - #[cfg(feature = "ipfs-storage")] - use noosphere_ipfs::{IpfsStorage, KuboClient}; + #[cfg(not(any(feature = "experimental-rocksdb")))] + pub(crate) type PrimitiveStorage = noosphere_storage::SledStorage; + #[cfg(feature = "experimental-rocksdb")] + pub(crate) type PrimitiveStorage = noosphere_storage::RocksDbStorage; + #[cfg(not(feature = "ipfs-storage"))] + pub type PlatformStorage = PrimitiveStorage; #[cfg(feature = "ipfs-storage")] - pub type PlatformStorage = IpfsStorage; + pub type PlatformStorage = + noosphere_ipfs::IpfsStorage; #[cfg(test)] use anyhow::Result; @@ -61,12 +68,14 @@ mod inner { use noosphere_storage::IndexedDbStorage; + pub(crate) type PrimitiveStorage = IndexedDbStorage; + #[cfg(feature = "ipfs-storage")] pub type PlatformStorage = - noosphere_ipfs::IpfsStorage; + noosphere_ipfs::IpfsStorage; #[cfg(not(feature = "ipfs-storage"))] - pub type PlatformStorage = IndexedDbStorage; + pub type PlatformStorage = PrimitiveStorage; #[cfg(test)] use anyhow::Result; @@ -104,22 +113,22 @@ mod inner { )) ))] mod inner { - use noosphere_storage::SledStorage; - use ucan_key_support::ed25519::Ed25519KeyMaterial; - use crate::key::InsecureKeyStorage; + use ucan_key_support::ed25519::Ed25519KeyMaterial; pub type PlatformKeyMaterial = Ed25519KeyMaterial; pub type PlatformKeyStorage = InsecureKeyStorage; - #[cfg(not(feature = "ipfs-storage"))] - pub type PlatformStorage = SledStorage; - - #[cfg(feature = "ipfs-storage")] - use noosphere_ipfs::{IpfsStorage, KuboClient}; + #[cfg(not(any(feature = "experimental-rocksdb")))] + pub(crate) type PrimitiveStorage = noosphere_storage::SledStorage; + #[cfg(feature = "experimental-rocksdb")] + pub(crate) type PrimitiveStorage = noosphere_storage::RocksDbStorage; + #[cfg(not(feature = "ipfs-storage"))] + pub type PlatformStorage = PrimitiveStorage; #[cfg(feature = "ipfs-storage")] - pub type PlatformStorage = IpfsStorage; + pub type PlatformStorage = + noosphere_ipfs::IpfsStorage; #[cfg(test)] use anyhow::Result; diff --git a/rust/noosphere/src/storage.rs b/rust/noosphere/src/storage.rs index aed049645..ae213530d 100644 --- a/rust/noosphere/src/storage.rs +++ b/rust/noosphere/src/storage.rs @@ -1,11 +1,11 @@ +use crate::platform::PrimitiveStorage; +use anyhow::Result; +use noosphere_core::data::Did; use std::{ fmt::Display, path::{Path, PathBuf}, }; -use anyhow::Result; -use noosphere_core::data::Did; - #[cfg(doc)] use noosphere_storage::Storage; @@ -41,23 +41,26 @@ impl From for PathBuf { } } -#[cfg(not(target_arch = "wasm32"))] -use noosphere_storage::{SledStorage, SledStorageInit}; - #[cfg(not(target_arch = "wasm32"))] impl StorageLayout { - pub async fn to_storage(&self) -> Result { - SledStorage::new(SledStorageInit::Path(PathBuf::from(self))) + pub async fn to_storage(&self) -> Result { + #[cfg(not(feature = "experimental-rocksdb"))] + { + noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path( + PathBuf::from(self), + )) + } + #[cfg(feature = "experimental-rocksdb")] + { + noosphere_storage::RocksDbStorage::new(PathBuf::from(self)) + } } } -#[cfg(target_arch = "wasm32")] -use noosphere_storage::IndexedDbStorage; - #[cfg(target_arch = "wasm32")] impl StorageLayout { - pub async fn to_storage(&self) -> Result { - IndexedDbStorage::new(&self.to_string()).await + pub async fn to_storage(&self) -> Result { + noosphere_storage::IndexedDbStorage::new(&self.to_string()).await } }