From 9d44417756b36a38518bdd28e34b4d3082956b36 Mon Sep 17 00:00:00 2001 From: Jordan Santell Date: Thu, 12 Oct 2023 09:53:49 -0700 Subject: [PATCH] feat: Introduce `--storage-memory-cache-limit` (#671) * feat: Introduce "--storage-memory-cache-limit" for orb gateway to to limit size of memory cache in storage providers. Co-authored-by: Christopher Joel <240083+cdata@users.noreply.github.com> --- .vscode/settings.json | 2 +- images/orb/start.sh | 1 + rust/noosphere-cli/src/native/cli.rs | 5 ++ .../src/native/commands/serve.rs | 10 +-- .../src/native/commands/sphere/mod.rs | 4 +- rust/noosphere-cli/src/native/helpers/cli.rs | 19 ++--- .../src/native/helpers/workspace.rs | 9 +- rust/noosphere-cli/src/native/mod.rs | 68 +++++++++++---- rust/noosphere-cli/src/native/paths.rs | 2 +- rust/noosphere-cli/src/native/workspace.rs | 28 ++++--- rust/noosphere-storage/examples/bench/main.rs | 6 +- rust/noosphere-storage/src/config.rs | 25 ++++++ rust/noosphere-storage/src/helpers.rs | 4 +- .../src/implementation/indexed_db.rs | 4 +- .../src/implementation/rocks_db.rs | 83 +++++++++++++------ .../src/implementation/sled.rs | 67 ++++++++++----- rust/noosphere-storage/src/lib.rs | 7 +- rust/noosphere/src/ffi/noosphere.rs | 8 +- rust/noosphere/src/lib.rs | 8 +- rust/noosphere/src/noosphere.rs | 36 +++++--- rust/noosphere/src/sphere/builder/create.rs | 1 + rust/noosphere/src/sphere/builder/join.rs | 1 + rust/noosphere/src/sphere/builder/mod.rs | 42 ++++------ rust/noosphere/src/sphere/builder/open.rs | 1 + rust/noosphere/src/sphere/builder/recover.rs | 1 + rust/noosphere/src/storage.rs | 65 +++++++++------ rust/noosphere/src/wasm/noosphere.rs | 8 +- rust/noosphere/tests/cli.rs | 11 ++- rust/noosphere/tests/sphere_channel.rs | 12 +-- 29 files changed, 345 insertions(+), 193 deletions(-) create mode 100644 rust/noosphere-storage/src/config.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 62025d3e2..7f295ee8b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,4 +14,4 @@ "test-kubo", "helpers" ] -} \ No newline at end of file +} diff --git a/images/orb/start.sh b/images/orb/start.sh index 807d94cdd..0799d2a8d 100755 --- a/images/orb/start.sh +++ b/images/orb/start.sh @@ -18,6 +18,7 @@ orb sphere config set counterpart $COUNTERPART ARGS="-i 0.0.0.0" ARGS="${ARGS} --ipfs-api ${IPFS_API}" +ARGS="${ARGS} --storage-memory-cache-limit 50000000" # ~50MB storage memory cache limit if ! [ -z "$NS_API" ]; then ARGS="${ARGS} --name-resolver-api ${NS_API}" diff --git a/rust/noosphere-cli/src/native/cli.rs b/rust/noosphere-cli/src/native/cli.rs index 350e11405..c6c1d2747 100644 --- a/rust/noosphere-cli/src/native/cli.rs +++ b/rust/noosphere-cli/src/native/cli.rs @@ -61,6 +61,11 @@ pub enum OrbCommand { /// The port that the gateway should listen on #[clap(short, long, default_value = "4433")] port: u16, + + /// If set, the amount of memory that the storage provider may use + /// for caching in bytes. + #[clap(long)] + storage_memory_cache_limit: Option, }, } diff --git a/rust/noosphere-cli/src/native/commands/serve.rs b/rust/noosphere-cli/src/native/commands/serve.rs index df3d38191..150aa380e 100644 --- a/rust/noosphere-cli/src/native/commands/serve.rs +++ b/rust/noosphere-cli/src/native/commands/serve.rs @@ -1,16 +1,12 @@ //! Concrete implementations of subcommands related to running a Noosphere //! gateway server +use crate::native::workspace::Workspace; use anyhow::Result; - +use noosphere_gateway::{start_gateway, GatewayScope}; use std::net::{IpAddr, TcpListener}; - use url::Url; -use crate::native::workspace::Workspace; - -use noosphere_gateway::{start_gateway, GatewayScope}; - /// Start a Noosphere gateway server pub async fn serve( interface: IpAddr, @@ -18,7 +14,7 @@ pub async fn serve( ipfs_api: Url, name_resolver_api: Url, cors_origin: Option, - workspace: &Workspace, + workspace: &mut Workspace, ) -> Result<()> { workspace.ensure_sphere_initialized()?; diff --git a/rust/noosphere-cli/src/native/commands/sphere/mod.rs b/rust/noosphere-cli/src/native/commands/sphere/mod.rs index 131161a8d..423e11515 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/mod.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/mod.rs @@ -42,7 +42,7 @@ use url::Url; pub async fn sphere_create(owner_key: &str, workspace: &mut Workspace) -> Result<(Did, Mnemonic)> { workspace.ensure_sphere_uninitialized()?; - let sphere_paths = SpherePaths::intialize(workspace.working_directory()).await?; + let sphere_paths = SpherePaths::initialize(workspace.working_directory()).await?; let sphere_context_artifacts = SphereContextBuilder::default() .create_sphere() @@ -124,7 +124,7 @@ Type or paste the code here and press enter:"# let cid = Cid::from_str(cid_string.trim()) .map_err(|_| anyhow!("Could not parse the authorization identity as a CID"))?; - let sphere_paths = SpherePaths::intialize(workspace.working_directory()).await?; + let sphere_paths = SpherePaths::initialize(workspace.working_directory()).await?; { let mut sphere_context = Arc::new(Mutex::new( diff --git a/rust/noosphere-cli/src/native/helpers/cli.rs b/rust/noosphere-cli/src/native/helpers/cli.rs index a2f477b19..94098b506 100644 --- a/rust/noosphere-cli/src/native/helpers/cli.rs +++ b/rust/noosphere-cli/src/native/helpers/cli.rs @@ -1,4 +1,6 @@ +use crate::{cli::Cli, invoke_cli, CliContext}; use anyhow::Result; +use clap::Parser; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::Mutex; @@ -6,9 +8,6 @@ use tempfile::TempDir; use tracing::{field, Level, Subscriber}; use tracing_subscriber::{prelude::*, Layer}; -use crate::{cli::Cli, invoke_cli, workspace::Workspace}; -use clap::Parser; - #[derive(Default)] struct InfoCaptureVisitor { message: String, @@ -161,22 +160,22 @@ impl CliSimulator { ) -> Result>> { let cli = self.parse_orb_command(command)?; - let workspace = Workspace::new( - &self.current_working_directory, - Some(self.noosphere_directory.path()), - )?; - debug!( "In {}: orb {}", self.current_working_directory.display(), command.join(" ") ); + let context = CliContext { + cwd: self.current_working_directory.clone(), + global_config_directory: Some(self.noosphere_directory.path()), + }; + let future = invoke_cli(cli, &context); if capture_output { - let info = run_and_capture_info(invoke_cli(cli, workspace))?; + let info = run_and_capture_info(future)?; Ok(Some(info)) } else { - invoke_cli(cli, workspace).await?; + future.await?; Ok(None) } } diff --git a/rust/noosphere-cli/src/native/helpers/workspace.rs b/rust/noosphere-cli/src/native/helpers/workspace.rs index 34ba10efd..a4ac4538f 100644 --- a/rust/noosphere-cli/src/native/helpers/workspace.rs +++ b/rust/noosphere-cli/src/native/helpers/workspace.rs @@ -12,7 +12,7 @@ use crate::{ }; use noosphere::{ NoosphereContext, NoosphereContextConfiguration, NoosphereNetwork, NoosphereSecurity, - NoosphereStorage, + NoosphereStorage, NoosphereStorageConfig, NoosphereStoragePath, }; use noosphere_core::{ context::HasSphereContext, @@ -37,7 +37,7 @@ pub fn temporary_workspace() -> Result<(Workspace, (TempDir, TempDir))> { let global_root = TempDir::new()?; Ok(( - Workspace::new(root.path(), Some(global_root.path()))?, + Workspace::new(root.path(), Some(global_root.path()), None)?, (root, global_root), )) } @@ -156,8 +156,9 @@ impl SphereData { /// state of this [SphereData] pub async fn as_noosphere_context(&self) -> Result { NoosphereContext::new(NoosphereContextConfiguration { - storage: NoosphereStorage::Unscoped { - path: self.sphere_root().to_owned(), + storage: NoosphereStorage { + path: NoosphereStoragePath::Unscoped(self.sphere_root().to_owned()), + config: NoosphereStorageConfig::default(), }, security: NoosphereSecurity::Insecure { path: self.global_root().to_owned(), diff --git a/rust/noosphere-cli/src/native/mod.rs b/rust/noosphere-cli/src/native/mod.rs index 727304719..dfc924264 100644 --- a/rust/noosphere-cli/src/native/mod.rs +++ b/rust/noosphere-cli/src/native/mod.rs @@ -11,11 +11,7 @@ pub mod workspace; #[cfg(any(test, feature = "helpers"))] pub mod helpers; -use anyhow::Result; - -use noosphere_core::tracing::initialize_tracing; - -use clap::Parser; +use std::path::{Path, PathBuf}; use self::{ cli::{AuthCommand, Cli, ConfigCommand, FollowCommand, KeyCommand, OrbCommand, SphereCommand}, @@ -29,24 +25,61 @@ use self::{ }, workspace::Workspace, }; +use anyhow::Result; +use clap::Parser; +use noosphere_core::tracing::initialize_tracing; +use noosphere_storage::StorageConfig; + +/// Additional context used to invoke a [Cli] command. +pub struct CliContext<'a> { + /// Path to the current working directory. + cwd: PathBuf, + /// Path to the global configuration directory, if provided. + global_config_directory: Option<&'a Path>, +} #[cfg(not(doc))] #[allow(missing_docs)] pub async fn main() -> Result<()> { initialize_tracing(None); + let context = CliContext { + cwd: std::env::current_dir()?, + global_config_directory: None, + }; + invoke_cli(Cli::parse(), &context).await +} - let workspace = Workspace::new(&std::env::current_dir()?, None)?; +/// Invoke the CLI implementation imperatively. +/// +/// This is the entrypoint used by orb when handling a command line invocation. +/// The [Cli] is produced by parsing the command line arguments, and internally +/// creates a new [Workspace] from the current working directory. +/// +/// Use [invoke_cli_with_workspace] if using your own [Workspace]. +pub async fn invoke_cli<'a>(cli: Cli, context: &CliContext<'a>) -> Result<()> { + let storage_config = if let OrbCommand::Serve { + storage_memory_cache_limit, + .. + } = &cli.command + { + Some(StorageConfig { + memory_cache_limit: *storage_memory_cache_limit, + }) + } else { + None + }; + let workspace = Workspace::new( + &context.cwd, + context.global_config_directory, + storage_config, + )?; - invoke_cli(Cli::parse(), workspace).await + invoke_cli_with_workspace(cli, workspace).await } -/// Invoke the CLI implementation imperatively. This is the entrypoint used by -/// orb when handling a command line invocation. The [Cli] is produced by -/// parsing the command line arguments, and the [Workspace] is initialized from -/// the current working directory. The effect of invoking the CLI this way will -/// depend on the [Cli] and [Workspace] provided (results may vary significantly -/// depending on those inputs). -pub async fn invoke_cli(cli: Cli, mut workspace: Workspace) -> Result<()> { +/// Same as [invoke_cli], but enables the caller to provide their own +/// initialized [Workspace] +pub async fn invoke_cli_with_workspace(cli: Cli, mut workspace: Workspace) -> Result<()> { match cli.command { OrbCommand::Key { command } => match command { KeyCommand::Create { name } => key_create(&name, &workspace).await?, @@ -117,6 +150,7 @@ pub async fn invoke_cli(cli: Cli, mut workspace: Workspace) -> Result<()> { name_resolver_api, interface, port, + .. } => { serve( interface, @@ -124,11 +158,11 @@ pub async fn invoke_cli(cli: Cli, mut workspace: Workspace) -> Result<()> { ipfs_api, name_resolver_api, cors_origin, - &workspace, + &mut workspace, ) - .await? + .await?; } - }; + } Ok(()) } diff --git a/rust/noosphere-cli/src/native/paths.rs b/rust/noosphere-cli/src/native/paths.rs index fbee996bd..234306fa7 100644 --- a/rust/noosphere-cli/src/native/paths.rs +++ b/rust/noosphere-cli/src/native/paths.rs @@ -93,7 +93,7 @@ impl SpherePaths { /// Initialize [SpherePaths] for a given root path. This has the effect of /// creating the "private" directory hierarchy (starting from /// [SPHERE_DIRECTORY] inside the root). - pub async fn intialize(root: &Path) -> Result { + pub async fn initialize(root: &Path) -> Result { if !root.is_absolute() { return Err(anyhow!( "Must use an absolute path to initialize sphere directories; got {:?}", diff --git a/rust/noosphere-cli/src/native/workspace.rs b/rust/noosphere-cli/src/native/workspace.rs index 4298aa99d..52626a229 100644 --- a/rust/noosphere-cli/src/native/workspace.rs +++ b/rust/noosphere-cli/src/native/workspace.rs @@ -9,7 +9,7 @@ use noosphere_core::context::{ SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL, }; use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld}; -use noosphere_storage::{KeyValueStore, SphereDb}; +use noosphere_storage::{KeyValueStore, SphereDb, StorageConfig}; use serde_json::Value; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -39,6 +39,7 @@ pub struct Workspace { key_storage: InsecureKeyStorage, sphere_context: OnceCell>>, working_directory: PathBuf, + storage_config: Option, } impl Workspace { @@ -53,15 +54,17 @@ impl Workspace { Ok(self .sphere_context .get_or_try_init(|| async { - Ok(Arc::new(Mutex::new( - SphereContextBuilder::default() - .open_sphere(None) - .at_storage_path(self.require_sphere_paths()?.root()) - .reading_keys_from(self.key_storage.clone()) - .build() - .await? - .into(), - ))) as Result>, anyhow::Error> + let mut builder = SphereContextBuilder::default() + .open_sphere(None) + .at_storage_path(self.require_sphere_paths()?.root()) + .reading_keys_from(self.key_storage.clone()); + + if let Some(storage_config) = self.storage_config.as_ref() { + builder = builder.with_storage_config(storage_config); + } + + Ok(Arc::new(Mutex::new(builder.build().await?.into()))) + as Result>, anyhow::Error> }) .await? .clone()) @@ -312,9 +315,13 @@ impl Workspace { /// - Windows: C:\Users\\AppData\Roaming\subconscious\noosphere\config /// /// On Linux, an $XDG_CONFIG_HOME environment variable will be respected if set. + /// + /// Additionally, a [StorageConfig] may be provided to configure the storage used + /// by the opened sphere. pub fn new( working_directory: &Path, custom_noosphere_directory: Option<&Path>, + storage_config: Option, ) -> Result { let sphere_paths = SpherePaths::discover(Some(working_directory)).map(Arc::new); @@ -340,6 +347,7 @@ impl Workspace { key_storage, sphere_context: OnceCell::new(), working_directory: working_directory.to_owned(), + storage_config, }; Ok(workspace) diff --git a/rust/noosphere-storage/examples/bench/main.rs b/rust/noosphere-storage/examples/bench/main.rs index 28e687a8b..36484568e 100644 --- a/rust/noosphere-storage/examples/bench/main.rs +++ b/rust/noosphere-storage/examples/bench/main.rs @@ -132,9 +132,7 @@ impl BenchmarkStorage { ))] let (storage, storage_name) = { ( - noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path( - storage_path.into(), - ))?, + noosphere_storage::SledStorage::new(&storage_path)?, "SledDbStorage", ) }; @@ -142,7 +140,7 @@ impl BenchmarkStorage { #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] let (storage, storage_name) = { ( - noosphere_storage::RocksDbStorage::new(storage_path.into())?, + noosphere_storage::RocksDbStorage::new(&storage_path)?, "RocksDbStorage", ) }; diff --git a/rust/noosphere-storage/src/config.rs b/rust/noosphere-storage/src/config.rs new file mode 100644 index 000000000..b0c77b8cd --- /dev/null +++ b/rust/noosphere-storage/src/config.rs @@ -0,0 +1,25 @@ +use crate::storage::Storage; +use anyhow::Result; +use async_trait::async_trait; +use noosphere_common::ConditionalSend; +use std::path::Path; + +/// Generalized configurations for [ConfigurableStorage]. +#[derive(Debug, Clone, Default)] +pub struct StorageConfig { + /// If set, the size limit in bytes of a memory-based cache. + pub memory_cache_limit: Option, +} + +/// [Storage] that can be customized via [StorageConfig]. +/// +/// Configurations are generalized across storage providers, +/// and may have differing underlying semantics. +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +pub trait ConfigurableStorage: Storage { + async fn open_with_config + ConditionalSend>( + path: P, + config: StorageConfig, + ) -> Result; +} diff --git a/rust/noosphere-storage/src/helpers.rs b/rust/noosphere-storage/src/helpers.rs index 5c531ef3c..2bdc9602a 100644 --- a/rust/noosphere-storage/src/helpers.rs +++ b/rust/noosphere-storage/src/helpers.rs @@ -2,7 +2,7 @@ use crate::Storage; use anyhow::Result; #[cfg(not(target_arch = "wasm32"))] -use crate::{SledStorage, SledStorageInit, SledStore}; +use crate::{SledStorage, SledStore}; #[cfg(not(target_arch = "wasm32"))] pub async fn make_disposable_store() -> Result { @@ -13,7 +13,7 @@ pub async fn make_disposable_store() -> Result { .into_iter() .map(String::from) .collect(); - let provider = SledStorage::new(SledStorageInit::Path(temp_dir.join(temp_name)))?; + let provider = SledStorage::new(temp_dir.join(temp_name))?; provider.get_block_store("foo").await } diff --git a/rust/noosphere-storage/src/implementation/indexed_db.rs b/rust/noosphere-storage/src/implementation/indexed_db.rs index f80113738..dfec3e215 100644 --- a/rust/noosphere-storage/src/implementation/indexed_db.rs +++ b/rust/noosphere-storage/src/implementation/indexed_db.rs @@ -20,7 +20,9 @@ pub struct IndexedDbStorage { impl Debug for IndexedDbStorage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("IndexedDbStorage").finish() + f.debug_struct("IndexedDbStorage") + .field("path", &self.name) + .finish() } } diff --git a/rust/noosphere-storage/src/implementation/rocks_db.rs b/rust/noosphere-storage/src/implementation/rocks_db.rs index cd2dd4bf0..d5441e875 100644 --- a/rust/noosphere-storage/src/implementation/rocks_db.rs +++ b/rust/noosphere-storage/src/implementation/rocks_db.rs @@ -1,6 +1,9 @@ -use crate::{storage::Storage, store::Store, SPHERE_DB_STORE_NAMES}; +use crate::{ + storage::Storage, store::Store, ConfigurableStorage, StorageConfig, SPHERE_DB_STORE_NAMES, +}; use anyhow::{anyhow, Result}; use async_trait::async_trait; +use noosphere_common::ConditionalSend; use rocksdb::{ColumnFamilyDescriptor, DBWithThreadMode, Options}; use std::{ path::{Path, PathBuf}, @@ -21,21 +24,45 @@ type ColumnType<'a> = Arc>; /// Caveats: /// * Values are limited to 4GB(?) [https://github.com/facebook/rocksdb/wiki/Basic-Operations#reads] /// TODO(#631): Further improvements to the implementation. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct RocksDbStorage { db: Arc, - #[allow(unused)] - path: PathBuf, + debug_data: Arc<(PathBuf, StorageConfig)>, } impl RocksDbStorage { - pub fn new(path: PathBuf) -> Result { - std::fs::create_dir_all(&path)?; - let canonicalized = path.canonicalize()?; - let db = Arc::new(RocksDbStorage::init_db(canonicalized.clone())?); + pub fn new>(path: P) -> Result { + Self::with_config(path, StorageConfig::default()) + } + + pub fn with_config>(path: P, storage_config: StorageConfig) -> Result { + std::fs::create_dir_all(path.as_ref())?; + let db_path = path.as_ref().canonicalize()?; + let db = { + let mut cfs: Vec = + Vec::with_capacity(SPHERE_DB_STORE_NAMES.len()); + + for store_name in SPHERE_DB_STORE_NAMES { + // https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + let cf_opts = Options::default(); + cfs.push(ColumnFamilyDescriptor::new(*store_name, cf_opts)); + } + + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + + if let Some(memory_cache_limit) = storage_config.memory_cache_limit { + // Amount of data to build up in memtables across all column families before writing to disk. + db_opts.set_db_write_buffer_size(memory_cache_limit); + } + + Arc::new(DbInner::open_cf_descriptors(&db_opts, path, cfs)?) + }; + Ok(RocksDbStorage { db, - path: canonicalized, + debug_data: Arc::new((db_path, storage_config)), }) } @@ -50,23 +77,6 @@ impl RocksDbStorage { RocksDbStore::new(self.db.clone(), name.to_owned()) } - - /// Configures a databasea at `path` and initializes the expected configurations. - fn init_db>(path: P) -> Result { - let mut cfs: Vec = Vec::with_capacity(SPHERE_DB_STORE_NAMES.len()); - - for store_name in SPHERE_DB_STORE_NAMES { - // https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide - let cf_opts = Options::default(); - cfs.push(ColumnFamilyDescriptor::new(*store_name, cf_opts)); - } - - let mut db_opts = Options::default(); - db_opts.create_if_missing(true); - db_opts.create_missing_column_families(true); - - Ok(DbInner::open_cf_descriptors(&db_opts, path, cfs)?) - } } #[async_trait] @@ -83,6 +93,25 @@ impl Storage for RocksDbStorage { } } +#[async_trait] +impl ConfigurableStorage for RocksDbStorage { + async fn open_with_config + ConditionalSend>( + path: P, + storage_config: StorageConfig, + ) -> Result { + Self::with_config(path, storage_config) + } +} + +impl std::fmt::Debug for RocksDbStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RocksDbStorage") + .field("path", &self.debug_data.0) + .field("config", &self.debug_data.1) + .finish() + } +} + #[derive(Clone)] pub struct RocksDbStore { name: String, @@ -142,6 +171,6 @@ impl Store for RocksDbStore { #[async_trait] impl crate::Space for RocksDbStorage { async fn get_space_usage(&self) -> Result { - crate::get_dir_size(&self.path).await + crate::get_dir_size(&self.debug_data.0).await } } diff --git a/rust/noosphere-storage/src/implementation/sled.rs b/rust/noosphere-storage/src/implementation/sled.rs index afab87527..4418fb928 100644 --- a/rust/noosphere-storage/src/implementation/sled.rs +++ b/rust/noosphere-storage/src/implementation/sled.rs @@ -1,37 +1,41 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::sync::Arc; -use crate::storage::Storage; use crate::store::Store; +use crate::StorageConfig; +use crate::{storage::Storage, ConfigurableStorage}; use anyhow::Result; use async_trait::async_trait; +use noosphere_common::ConditionalSend; use sled::{Db, Tree}; -pub enum SledStorageInit { - Path(PathBuf), - Db(Db), -} - -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct SledStorage { db: Db, - #[allow(unused)] - path: Option, + debug_data: Arc<(PathBuf, StorageConfig)>, } impl SledStorage { - pub fn new(init: SledStorageInit) -> Result { - let mut db_path = None; - let db: Db = match init { - SledStorageInit::Path(path) => { - std::fs::create_dir_all(&path)?; - db_path = Some(path.clone().canonicalize()?); - sled::open(path)? - } - SledStorageInit::Db(db) => db, - }; - - Ok(SledStorage { db, path: db_path }) + /// Open or create a database at directory `path`. + pub fn new>(path: P) -> Result { + Self::with_config(path, StorageConfig::default()) + } + + pub fn with_config>(path: P, config: StorageConfig) -> Result { + std::fs::create_dir_all(path.as_ref())?; + let db_path = path.as_ref().canonicalize()?; + + let mut sled_config = sled::Config::default(); + sled_config = sled_config.path(&db_path); + if let Some(memory_cache_limit) = config.memory_cache_limit { + // Maximum size in bytes for the system page cache. (default: 1GB) + sled_config = sled_config.cache_capacity(memory_cache_limit.try_into()?); + } + + let db = sled_config.open()?; + let debug_data = Arc::new((db_path, config)); + Ok(SledStorage { db, debug_data }) } async fn get_store(&self, name: &str) -> Result { @@ -54,6 +58,25 @@ impl Storage for SledStorage { } } +#[async_trait] +impl ConfigurableStorage for SledStorage { + async fn open_with_config + ConditionalSend>( + path: P, + config: StorageConfig, + ) -> Result { + SledStorage::with_config(path, config) + } +} + +impl std::fmt::Debug for SledStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SledStorage") + .field("path", &self.debug_data.0) + .field("config", &self.debug_data.1) + .finish() + } +} + #[derive(Clone)] pub struct SledStore { db: Tree, diff --git a/rust/noosphere-storage/src/lib.rs b/rust/noosphere-storage/src/lib.rs index 17835e0d8..d526447b7 100644 --- a/rust/noosphere-storage/src/lib.rs +++ b/rust/noosphere-storage/src/lib.rs @@ -7,11 +7,11 @@ extern crate tracing; mod block; -mod implementation; -mod key_value; - +mod config; mod db; mod encoding; +mod implementation; +mod key_value; mod retry; mod storage; mod store; @@ -20,6 +20,7 @@ mod ucan; pub use crate::ucan::*; pub use block::*; +pub use config::*; pub use db::*; pub use encoding::*; pub use implementation::*; diff --git a/rust/noosphere/src/ffi/noosphere.rs b/rust/noosphere/src/ffi/noosphere.rs index 0a8afaec7..7445e3cca 100644 --- a/rust/noosphere/src/ffi/noosphere.rs +++ b/rust/noosphere/src/ffi/noosphere.rs @@ -3,7 +3,8 @@ use std::{cell::OnceCell, time::Duration}; use crate::{ ffi::{NsError, TryOrInitialize}, noosphere::{NoosphereContext, NoosphereContextConfiguration}, - NoosphereNetwork, NoosphereSecurity, NoosphereStorage, + NoosphereNetwork, NoosphereSecurity, NoosphereStorage, NoosphereStorageConfig, + NoosphereStoragePath, }; use anyhow::{anyhow, Result}; use pkg_version::*; @@ -43,8 +44,9 @@ impl NsNoosphere { ) -> Result { Ok(NsNoosphere { inner: NoosphereContext::new(NoosphereContextConfiguration { - storage: NoosphereStorage::Scoped { - path: sphere_storage_path.into(), + storage: NoosphereStorage { + path: NoosphereStoragePath::Scoped(sphere_storage_path.into()), + config: NoosphereStorageConfig::default(), }, security: NoosphereSecurity::Insecure { path: global_storage_path.into(), diff --git a/rust/noosphere/src/lib.rs b/rust/noosphere/src/lib.rs index a3cf4e62a..0159fac45 100644 --- a/rust/noosphere/src/lib.rs +++ b/rust/noosphere/src/lib.rs @@ -7,7 +7,8 @@ //! //! ```rust,no_run //! # use noosphere::{ -//! # NoosphereStorage, NoosphereSecurity, NoosphereNetwork, NoosphereContextConfiguration, NoosphereContext, sphere::SphereReceipt +//! # NoosphereStorage, NoosphereStoragePath, NoosphereStorageConfig, NoosphereSecurity, NoosphereNetwork, +//! # NoosphereContextConfiguration, NoosphereContext, sphere::SphereReceipt //! # }; //! # use noosphere_core::{ //! # context::{ @@ -21,8 +22,9 @@ //! # #[tokio::main] //! # pub async fn main() -> Result<()> { //! let noosphere = NoosphereContext::new(NoosphereContextConfiguration { -//! storage: NoosphereStorage::Scoped { -//! path: "/path/to/block/storage".into(), +//! storage: NoosphereStorage { +//! path: NoosphereStoragePath::Scoped("/path/to/block/storage".into()), +//! config: NoosphereStorageConfig::default(), //! }, //! security: NoosphereSecurity::Insecure { //! path: "/path/to/key/storage".into(), diff --git a/rust/noosphere/src/noosphere.rs b/rust/noosphere/src/noosphere.rs index 2e82e28fa..60b946c5f 100644 --- a/rust/noosphere/src/noosphere.rs +++ b/rust/noosphere/src/noosphere.rs @@ -5,6 +5,7 @@ use noosphere_core::{ authority::Authorization, data::{Did, Mnemonic}, }; +use noosphere_storage::StorageConfig; use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; use noosphere_core::context::{SphereContext, SphereCursor}; @@ -17,25 +18,32 @@ use crate::{ sphere::{SphereChannel, SphereContextBuilder, SphereReceipt}, }; +/// Configurations for the Noosphere storage layer. +pub type NoosphereStorageConfig = StorageConfig; + /// An enum describing different storage stragies that may be interesting /// depending on the environment and implementation of Noosphere #[derive(Clone)] -pub enum NoosphereStorage { +pub enum NoosphereStoragePath { /// Scoped storage implies that the given path is a root and that spheres /// should be stored in a sub-path that includes the sphere identity at the /// trailing end - Scoped { - /// The path where storage should be rooted - path: PathBuf, - }, + Scoped(PathBuf), /// Unscoped storage implies that sphere data should be kept at the given /// path. Note that this is typically only appropriate when dealing with a /// single sphere. - Unscoped { - /// The path where storage should be rooted - path: PathBuf, - }, + Unscoped(PathBuf), +} + +/// Fields describing configuration of Noosphere's storage layer. +#[derive(Clone)] +pub struct NoosphereStorage { + /// The strategy and path of the storage layer. + pub path: NoosphereStoragePath, + /// Configurable features for the storage layer. + /// Storage providers may not support any or all configurations. + pub config: NoosphereStorageConfig, } /// This enum exists so that we can incrementally layer on support for secure @@ -123,9 +131,9 @@ impl NoosphereContext { } fn sphere_storage_path(&self) -> &PathBuf { - match &self.configuration.storage { - NoosphereStorage::Scoped { path } => path, - NoosphereStorage::Unscoped { path } => path, + match &self.configuration.storage.path { + NoosphereStoragePath::Scoped(path) => path, + NoosphereStoragePath::Unscoped(path) => path, } } @@ -180,6 +188,7 @@ impl NoosphereContext { let artifacts = SphereContextBuilder::default() .recover_sphere(sphere_id) .at_storage_path(self.sphere_storage_path()) + .with_storage_config(&self.configuration.storage.config) .using_scoped_storage_layout() .reading_keys_from(self.key_storage().await?) .using_mnemonic(Some(mnemonic)) @@ -208,6 +217,7 @@ impl NoosphereContext { let artifacts = SphereContextBuilder::default() .create_sphere() .at_storage_path(self.sphere_storage_path()) + .with_storage_config(&self.configuration.storage.config) .using_scoped_storage_layout() .reading_keys_from(self.key_storage().await?) .using_key(owner_key_name) @@ -249,6 +259,7 @@ impl NoosphereContext { let artifacts = SphereContextBuilder::default() .join_sphere(sphere_identity) .at_storage_path(self.sphere_storage_path()) + .with_storage_config(&self.configuration.storage.config) .using_scoped_storage_layout() .reading_keys_from(self.key_storage().await?) .using_key(local_key_name) @@ -287,6 +298,7 @@ impl NoosphereContext { let artifacts = SphereContextBuilder::default() .open_sphere(Some(sphere_identity)) .at_storage_path(self.sphere_storage_path()) + .with_storage_config(&self.configuration.storage.config) .using_scoped_storage_layout() .reading_keys_from(self.key_storage().await?) .syncing_to(self.gateway_api()) diff --git a/rust/noosphere/src/sphere/builder/create.rs b/rust/noosphere/src/sphere/builder/create.rs index c1a6331d6..2934253c0 100644 --- a/rust/noosphere/src/sphere/builder/create.rs +++ b/rust/noosphere/src/sphere/builder/create.rs @@ -45,6 +45,7 @@ pub async fn create_a_sphere( builder.scoped_storage_layout, Some(sphere_did.clone()), builder.ipfs_gateway_url, + builder.storage_config, ) .await?; diff --git a/rust/noosphere/src/sphere/builder/join.rs b/rust/noosphere/src/sphere/builder/join.rs index 563378bd1..e40d49bed 100644 --- a/rust/noosphere/src/sphere/builder/join.rs +++ b/rust/noosphere/src/sphere/builder/join.rs @@ -35,6 +35,7 @@ pub async fn join_a_sphere( builder.scoped_storage_layout, Some(sphere_identity.clone()), builder.ipfs_gateway_url.clone(), + builder.storage_config.clone(), ) .await?; diff --git a/rust/noosphere/src/sphere/builder/mod.rs b/rust/noosphere/src/sphere/builder/mod.rs index 86d513df3..fc2bb586f 100644 --- a/rust/noosphere/src/sphere/builder/mod.rs +++ b/rust/noosphere/src/sphere/builder/mod.rs @@ -8,29 +8,20 @@ use join::*; use open::*; use recover::*; -use std::path::{Path, PathBuf}; - +use crate::{ + platform::{PlatformKeyStorage, PlatformStorage}, + storage::{create_platform_storage, StorageLayout}, +}; use anyhow::{anyhow, Result}; - +use noosphere_core::context::SphereContext; use noosphere_core::{ authority::Authorization, data::{Did, Mnemonic}, }; - -#[cfg(all(target_arch = "wasm32", feature = "ipfs-storage"))] -use noosphere_ipfs::{GatewayClient, IpfsStorage}; - -use noosphere_storage::SphereDb; - +use noosphere_storage::{SphereDb, StorageConfig}; +use std::path::{Path, PathBuf}; use url::Url; -use noosphere_core::context::SphereContext; - -use crate::{ - platform::{PlatformKeyStorage, PlatformStorage}, - storage::StorageLayout, -}; - #[derive(Default, Clone)] enum SphereInitialization { #[default] @@ -97,6 +88,7 @@ pub struct SphereContextBuilder { pub(crate) key_storage: Option, pub(crate) key_name: Option, pub(crate) mnemonic: Option, + pub(crate) storage_config: Option, } impl SphereContextBuilder { @@ -181,6 +173,13 @@ impl SphereContextBuilder { self } + /// Specify a [StorageConfig] configuration to use when opening the underlying + /// storage layer. + pub fn with_storage_config(mut self, storage_config: &StorageConfig) -> Self { + self.storage_config = Some(storage_config.to_owned()); + self + } + /// Generate [SphereContextBuilderArtifacts] based on the given /// configuration of the [SphereContextBuilder]. The successful result of /// invoking this method will always include an activated [SphereContext]. @@ -247,6 +246,7 @@ impl Default for SphereContextBuilder { key_storage: None as Option, key_name: None, mnemonic: None, + storage_config: None, } } } @@ -273,17 +273,11 @@ pub(crate) async fn generate_db( scoped_storage_layout: bool, sphere_identity: Option, ipfs_gateway_url: Option, + storage_config: Option, ) -> Result> { let storage_layout: StorageLayout = (storage_path, scoped_storage_layout, sphere_identity).try_into()?; - - #[cfg(not(all(wasm, ipfs_storage)))] - let storage = storage_layout.to_storage().await?; - #[cfg(all(wasm, ipfs_storage))] - let storage = IpfsStorage::new( - storage_layout.to_storage().await?, - ipfs_gateway_url.map(|url| GatewayClient::new(url)), - ); + let storage = create_platform_storage(storage_layout, ipfs_gateway_url, storage_config).await?; SphereDb::new(&storage).await } diff --git a/rust/noosphere/src/sphere/builder/open.rs b/rust/noosphere/src/sphere/builder/open.rs index 549671c68..6ea0395a2 100644 --- a/rust/noosphere/src/sphere/builder/open.rs +++ b/rust/noosphere/src/sphere/builder/open.rs @@ -23,6 +23,7 @@ pub async fn open_a_sphere( builder.scoped_storage_layout, sphere_identity, builder.ipfs_gateway_url, + builder.storage_config, ) .await?; diff --git a/rust/noosphere/src/sphere/builder/recover.rs b/rust/noosphere/src/sphere/builder/recover.rs index b7810d87f..f69df056b 100644 --- a/rust/noosphere/src/sphere/builder/recover.rs +++ b/rust/noosphere/src/sphere/builder/recover.rs @@ -94,6 +94,7 @@ pub async fn recover_a_sphere( builder.scoped_storage_layout, Some(sphere_identity.clone()), builder.ipfs_gateway_url.clone(), + builder.storage_config.clone(), ) .await?; diff --git a/rust/noosphere/src/storage.rs b/rust/noosphere/src/storage.rs index e9631eaef..2c03f54a5 100644 --- a/rust/noosphere/src/storage.rs +++ b/rust/noosphere/src/storage.rs @@ -1,16 +1,21 @@ //! Intermediate constructs to normalize how storage is initialized -use crate::platform::PrimitiveStorage; +use crate::platform::PlatformStorage; use anyhow::Result; use noosphere_core::data::Did; +use noosphere_storage::StorageConfig; use std::{ fmt::Display, path::{Path, PathBuf}, }; +use url::Url; #[cfg(doc)] use noosphere_storage::Storage; +#[cfg(feature = "ipfs-storage")] +use noosphere_ipfs::{GatewayClient, IpfsStorage}; + /// [StorageLayout] represents the namespace that should be used depending on /// whether or not a sphere's DID should be included in the namespace. The enum /// is a convenience that can be directly transformed into a [Storage] @@ -46,31 +51,6 @@ impl From for PathBuf { } } -#[cfg(native)] -impl StorageLayout { - pub(crate) async fn to_storage(&self) -> Result { - #[cfg(sled)] - { - noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path( - PathBuf::from(self), - )) - } - #[cfg(rocksdb)] - { - noosphere_storage::RocksDbStorage::new(PathBuf::from(self)) - } - } -} - -#[cfg(wasm)] -impl StorageLayout { - /// Convert this [StorageLayout] to a [noosphere_storage::Storage] based on the - /// defaults configured for the current platform. - pub async fn to_storage(&self) -> Result { - noosphere_storage::IndexedDbStorage::new(&self.to_string()).await - } -} - fn get_scoped_path(path: &Path, scope: &Did) -> PathBuf { #[cfg(not(windows))] let path_buf = path.join(scope.as_str()); @@ -82,3 +62,36 @@ fn get_scoped_path(path: &Path, scope: &Did) -> PathBuf { path_buf } + +/// Construct [PlatformStorage] from a [StorageLayout] and [StorageConfig]. +/// +/// Takes a [Url] to an IPFS Gateway that is used when compiling with `ipfs-storage`. +pub async fn create_platform_storage( + layout: StorageLayout, + #[allow(unused)] ipfs_gateway_url: Option, + #[allow(unused)] storage_config: Option, +) -> Result { + #[cfg(any(sled, rocksdb))] + let storage = { + use noosphere_storage::ConfigurableStorage; + let path: PathBuf = layout.into(); + crate::platform::PrimitiveStorage::open_with_config( + &path, + storage_config.unwrap_or_default(), + ) + .await? + }; + + #[cfg(wasm)] + let storage = noosphere_storage::IndexedDbStorage::new(&layout.to_string()).await?; + + #[cfg(ipfs_storage)] + let storage = { + let maybe_client = ipfs_gateway_url.map(|url| GatewayClient::new(url)); + IpfsStorage::new(storage, maybe_client) + }; + + debug!("Created platform storage: {:#?}", storage); + + Ok(storage) +} diff --git a/rust/noosphere/src/wasm/noosphere.rs b/rust/noosphere/src/wasm/noosphere.rs index 1ca79f7f9..c189f425a 100644 --- a/rust/noosphere/src/wasm/noosphere.rs +++ b/rust/noosphere/src/wasm/noosphere.rs @@ -5,7 +5,8 @@ use wasm_bindgen::prelude::*; use crate::{ wasm::SphereContext, NoosphereContext as NoosphereContextImpl, NoosphereContextConfiguration, - NoosphereNetwork, NoosphereSecurity, NoosphereStorage, + NoosphereNetwork, NoosphereSecurity, NoosphereStorage, NoosphereStorageConfig, + NoosphereStoragePath, }; #[wasm_bindgen] @@ -79,8 +80,9 @@ impl NoosphereContext { }; let noosphere_context = NoosphereContextImpl::new(NoosphereContextConfiguration { - storage: NoosphereStorage::Scoped { - path: storage_namespace.into(), + storage: NoosphereStorage { + path: NoosphereStoragePath::Scoped(storage_namespace.into()), + config: NoosphereStorageConfig::default(), }, security: NoosphereSecurity::Opaque, network: NoosphereNetwork::Http { diff --git a/rust/noosphere/tests/cli.rs b/rust/noosphere/tests/cli.rs index 557c39314..4acba1574 100644 --- a/rust/noosphere/tests/cli.rs +++ b/rust/noosphere/tests/cli.rs @@ -103,12 +103,11 @@ async fn orb_can_enable_multiple_replicas_to_synchronize() -> Result<()> { .orb(&["sphere", "auth", "add", &second_replica_id]) .await?; - let second_replica_auth = match serde_json::from_str( - &first_replica - .orb_with_output(&["sphere", "auth", "list", "--as-json"]) - .await? - .join("\n"), - )? { + let auth_list = first_replica + .orb_with_output(&["sphere", "auth", "list", "--as-json"]) + .await? + .join("\n"); + let second_replica_auth = match serde_json::from_str(&auth_list)? { Value::Array(auths) => match auths .iter() .filter(|auth| { diff --git a/rust/noosphere/tests/sphere_channel.rs b/rust/noosphere/tests/sphere_channel.rs index 5e8222c71..85d8501cf 100644 --- a/rust/noosphere/tests/sphere_channel.rs +++ b/rust/noosphere/tests/sphere_channel.rs @@ -25,14 +25,15 @@ wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); use noosphere::{ sphere::SphereReceipt, NoosphereContext, NoosphereContextConfiguration, NoosphereNetwork, - NoosphereSecurity, NoosphereStorage, + NoosphereSecurity, NoosphereStorage, NoosphereStorageConfig, NoosphereStoragePath, }; #[cfg(target_arch = "wasm32")] fn platform_configuration() -> (NoosphereContextConfiguration, ()) { let configuration = NoosphereContextConfiguration { - storage: NoosphereStorage::Scoped { - path: "sphere-data".into(), + storage: NoosphereStorage { + path: NoosphereStoragePath::Scoped("sphere-data".into()), + config: NoosphereStorageConfig::default(), }, security: NoosphereSecurity::Opaque, network: NoosphereNetwork::Http { @@ -55,8 +56,9 @@ fn platform_configuration() -> ( let sphere_storage = TempDir::new().unwrap(); let configuration = NoosphereContextConfiguration { - storage: NoosphereStorage::Unscoped { - path: sphere_storage.path().to_path_buf(), + storage: NoosphereStorage { + path: NoosphereStoragePath::Unscoped(sphere_storage.path().to_path_buf()), + config: NoosphereStorageConfig::default(), }, security: NoosphereSecurity::Insecure { path: global_storage.path().to_path_buf(),