feat: Introduce --storage-memory-cache-limit (#671)

* feat: Introduce "--storage-memory-cache-limit" for orb gateway to
to limit size of memory cache in storage providers.

Co-authored-by: Christopher Joel <240083+cdata@users.noreply.github.com>

.vscode/settings.json

@@ -14,4 +14,4 @@
     "test-kubo",
     "helpers"
   ]
-}
+}

images/orb/start.sh

@@ -18,6 +18,7 @@ orb sphere config set counterpart $COUNTERPART
 
 ARGS="-i 0.0.0.0"
 ARGS="${ARGS} --ipfs-api ${IPFS_API}"
+ARGS="${ARGS} --storage-memory-cache-limit 50000000" # ~50MB storage memory cache limit
 
 if ! [ -z "$NS_API" ]; then
 	ARGS="${ARGS} --name-resolver-api ${NS_API}"

rust/noosphere-cli/src/native/cli.rs

@@ -61,6 +61,11 @@ pub enum OrbCommand {
         /// The port that the gateway should listen on
         #[clap(short, long, default_value = "4433")]
         port: u16,
+
+        /// If set, the amount of memory that the storage provider may use
+        /// for caching in bytes.
+        #[clap(long)]
+        storage_memory_cache_limit: Option<usize>,
     },
 }
 

rust/noosphere-cli/src/native/commands/serve.rs

@@ -1,24 +1,20 @@
 //! Concrete implementations of subcommands related to running a Noosphere
 //! gateway server
 
+use crate::native::workspace::Workspace;
 use anyhow::Result;
-
+use noosphere_gateway::{start_gateway, GatewayScope};
 use std::net::{IpAddr, TcpListener};
-
 use url::Url;
 
-use crate::native::workspace::Workspace;
-
-use noosphere_gateway::{start_gateway, GatewayScope};
-
 /// Start a Noosphere gateway server
 pub async fn serve(
     interface: IpAddr,
     port: u16,
     ipfs_api: Url,
     name_resolver_api: Url,
     cors_origin: Option<Url>,
-    workspace: &Workspace,
+    workspace: &mut Workspace,
 ) -> Result<()> {
     workspace.ensure_sphere_initialized()?;
 

rust/noosphere-cli/src/native/commands/sphere/mod.rs

@@ -42,7 +42,7 @@ use url::Url;
 pub async fn sphere_create(owner_key: &str, workspace: &mut Workspace) -> Result<(Did, Mnemonic)> {
     workspace.ensure_sphere_uninitialized()?;
 
-    let sphere_paths = SpherePaths::intialize(workspace.working_directory()).await?;
+    let sphere_paths = SpherePaths::initialize(workspace.working_directory()).await?;
 
     let sphere_context_artifacts = SphereContextBuilder::default()
         .create_sphere()
@@ -124,7 +124,7 @@ Type or paste the code here and press enter:"#
     let cid = Cid::from_str(cid_string.trim())
         .map_err(|_| anyhow!("Could not parse the authorization identity as a CID"))?;
 
-    let sphere_paths = SpherePaths::intialize(workspace.working_directory()).await?;
+    let sphere_paths = SpherePaths::initialize(workspace.working_directory()).await?;
 
     {
         let mut sphere_context = Arc::new(Mutex::new(

rust/noosphere-cli/src/native/helpers/cli.rs

@@ -1,14 +1,13 @@
+use crate::{cli::Cli, invoke_cli, CliContext};
 use anyhow::Result;
+use clap::Parser;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::sync::Mutex;
 use tempfile::TempDir;
 use tracing::{field, Level, Subscriber};
 use tracing_subscriber::{prelude::*, Layer};
 
-use crate::{cli::Cli, invoke_cli, workspace::Workspace};
-use clap::Parser;
-
 #[derive(Default)]
 struct InfoCaptureVisitor {
     message: String,
@@ -161,22 +160,22 @@ impl CliSimulator {
     ) -> Result<Option<Vec<String>>> {
         let cli = self.parse_orb_command(command)?;
 
-        let workspace = Workspace::new(
-            &self.current_working_directory,
-            Some(self.noosphere_directory.path()),
-        )?;
-
         debug!(
             "In {}: orb {}",
             self.current_working_directory.display(),
             command.join(" ")
         );
 
+        let context = CliContext {
+            cwd: self.current_working_directory.clone(),
+            global_config_directory: Some(self.noosphere_directory.path()),
+        };
+        let future = invoke_cli(cli, &context);
         if capture_output {
-            let info = run_and_capture_info(invoke_cli(cli, workspace))?;
+            let info = run_and_capture_info(future)?;
             Ok(Some(info))
         } else {
-            invoke_cli(cli, workspace).await?;
+            future.await?;
             Ok(None)
         }
     }

rust/noosphere-cli/src/native/helpers/workspace.rs

@@ -12,7 +12,7 @@ use crate::{
 };
 use noosphere::{
     NoosphereContext, NoosphereContextConfiguration, NoosphereNetwork, NoosphereSecurity,
-    NoosphereStorage,
+    NoosphereStorage, NoosphereStorageConfig, NoosphereStoragePath,
 };
 use noosphere_core::{
     context::HasSphereContext,
@@ -37,7 +37,7 @@ pub fn temporary_workspace() -> Result<(Workspace, (TempDir, TempDir))> {
     let global_root = TempDir::new()?;
 
     Ok((
-        Workspace::new(root.path(), Some(global_root.path()))?,
+        Workspace::new(root.path(), Some(global_root.path()), None)?,
         (root, global_root),
     ))
 }
@@ -156,8 +156,9 @@ impl SphereData {
     /// state of this [SphereData]
     pub async fn as_noosphere_context(&self) -> Result<NoosphereContext> {
         NoosphereContext::new(NoosphereContextConfiguration {
-            storage: NoosphereStorage::Unscoped {
-                path: self.sphere_root().to_owned(),
+            storage: NoosphereStorage {
+                path: NoosphereStoragePath::Unscoped(self.sphere_root().to_owned()),
+                config: NoosphereStorageConfig::default(),
             },
             security: NoosphereSecurity::Insecure {
                 path: self.global_root().to_owned(),

rust/noosphere-cli/src/native/mod.rs

@@ -11,11 +11,7 @@ pub mod workspace;
 #[cfg(any(test, feature = "helpers"))]
 pub mod helpers;
 
-use anyhow::Result;
-
-use noosphere_core::tracing::initialize_tracing;
-
-use clap::Parser;
+use std::path::{Path, PathBuf};
 
 use self::{
     cli::{AuthCommand, Cli, ConfigCommand, FollowCommand, KeyCommand, OrbCommand, SphereCommand},
@@ -29,24 +25,61 @@ use self::{
     },
     workspace::Workspace,
 };
+use anyhow::Result;
+use clap::Parser;
+use noosphere_core::tracing::initialize_tracing;
+use noosphere_storage::StorageConfig;
+
+/// Additional context used to invoke a [Cli] command.
+pub struct CliContext<'a> {
+    /// Path to the current working directory.
+    cwd: PathBuf,
+    /// Path to the global configuration directory, if provided.
+    global_config_directory: Option<&'a Path>,
+}
 
 #[cfg(not(doc))]
 #[allow(missing_docs)]
 pub async fn main() -> Result<()> {
     initialize_tracing(None);
+    let context = CliContext {
+        cwd: std::env::current_dir()?,
+        global_config_directory: None,
+    };
+    invoke_cli(Cli::parse(), &context).await
+}
 
-    let workspace = Workspace::new(&std::env::current_dir()?, None)?;
+/// Invoke the CLI implementation imperatively.
+///
+/// This is the entrypoint used by orb when handling a command line invocation.
+/// The [Cli] is produced by parsing the command line arguments, and internally
+/// creates a new [Workspace] from the current working directory.
+///
+/// Use [invoke_cli_with_workspace] if using your own [Workspace].
+pub async fn invoke_cli<'a>(cli: Cli, context: &CliContext<'a>) -> Result<()> {
+    let storage_config = if let OrbCommand::Serve {
+        storage_memory_cache_limit,
+        ..
+    } = &cli.command
+    {
+        Some(StorageConfig {
+            memory_cache_limit: *storage_memory_cache_limit,
+        })
+    } else {
+        None
+    };
+    let workspace = Workspace::new(
+        &context.cwd,
+        context.global_config_directory,
+        storage_config,
+    )?;
 
-    invoke_cli(Cli::parse(), workspace).await
+    invoke_cli_with_workspace(cli, workspace).await
 }
 
-/// Invoke the CLI implementation imperatively. This is the entrypoint used by
-/// orb when handling a command line invocation. The [Cli] is produced by
-/// parsing the command line arguments, and the [Workspace] is initialized from
-/// the current working directory. The effect of invoking the CLI this way will
-/// depend on the [Cli] and [Workspace] provided (results may vary significantly
-/// depending on those inputs).
-pub async fn invoke_cli(cli: Cli, mut workspace: Workspace) -> Result<()> {
+/// Same as [invoke_cli], but enables the caller to provide their own
+/// initialized [Workspace]
+pub async fn invoke_cli_with_workspace(cli: Cli, mut workspace: Workspace) -> Result<()> {
     match cli.command {
         OrbCommand::Key { command } => match command {
             KeyCommand::Create { name } => key_create(&name, &workspace).await?,
@@ -117,18 +150,19 @@ pub async fn invoke_cli(cli: Cli, mut workspace: Workspace) -> Result<()> {
             name_resolver_api,
             interface,
             port,
+            ..
         } => {
             serve(
                 interface,
                 port,
                 ipfs_api,
                 name_resolver_api,
                 cors_origin,
-                &workspace,
+                &mut workspace,
             )
-            .await?
+            .await?;
         }
-    };
+    }
 
     Ok(())
 }

rust/noosphere-cli/src/native/paths.rs

@@ -93,7 +93,7 @@ impl SpherePaths {
     /// Initialize [SpherePaths] for a given root path. This has the effect of
     /// creating the "private" directory hierarchy (starting from
     /// [SPHERE_DIRECTORY] inside the root).
-    pub async fn intialize(root: &Path) -> Result<Self> {
+    pub async fn initialize(root: &Path) -> Result<Self> {
         if !root.is_absolute() {
             return Err(anyhow!(
                 "Must use an absolute path to initialize sphere directories; got {:?}",

rust/noosphere-cli/src/native/workspace.rs

@@ -9,7 +9,7 @@ use noosphere_core::context::{
     SphereContentRead, SphereContext, SphereCursor, COUNTERPART, GATEWAY_URL,
 };
 use noosphere_core::data::{Did, Link, LinkRecord, MemoIpld};
-use noosphere_storage::{KeyValueStore, SphereDb};
+use noosphere_storage::{KeyValueStore, SphereDb, StorageConfig};
 use serde_json::Value;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -39,6 +39,7 @@ pub struct Workspace {
     key_storage: InsecureKeyStorage,
     sphere_context: OnceCell<Arc<Mutex<CliSphereContext>>>,
     working_directory: PathBuf,
+    storage_config: Option<StorageConfig>,
 }
 
 impl Workspace {
@@ -53,15 +54,17 @@ impl Workspace {
         Ok(self
             .sphere_context
             .get_or_try_init(|| async {
-                Ok(Arc::new(Mutex::new(
-                    SphereContextBuilder::default()
-                        .open_sphere(None)
-                        .at_storage_path(self.require_sphere_paths()?.root())
-                        .reading_keys_from(self.key_storage.clone())
-                        .build()
-                        .await?
-                        .into(),
-                ))) as Result<Arc<Mutex<CliSphereContext>>, anyhow::Error>
+                let mut builder = SphereContextBuilder::default()
+                    .open_sphere(None)
+                    .at_storage_path(self.require_sphere_paths()?.root())
+                    .reading_keys_from(self.key_storage.clone());
+
+                if let Some(storage_config) = self.storage_config.as_ref() {
+                    builder = builder.with_storage_config(storage_config);
+                }
+
+                Ok(Arc::new(Mutex::new(builder.build().await?.into())))
+                    as Result<Arc<Mutex<CliSphereContext>>, anyhow::Error>
             })
             .await?
             .clone())
@@ -312,9 +315,13 @@ impl Workspace {
     /// - Windows: C:\Users\<user>\AppData\Roaming\subconscious\noosphere\config
     ///
     /// On Linux, an $XDG_CONFIG_HOME environment variable will be respected if set.
+    ///
+    /// Additionally, a [StorageConfig] may be provided to configure the storage used
+    /// by the opened sphere.
     pub fn new(
         working_directory: &Path,
         custom_noosphere_directory: Option<&Path>,
+        storage_config: Option<StorageConfig>,
     ) -> Result<Self> {
         let sphere_paths = SpherePaths::discover(Some(working_directory)).map(Arc::new);
 
@@ -340,6 +347,7 @@ impl Workspace {
             key_storage,
             sphere_context: OnceCell::new(),
             working_directory: working_directory.to_owned(),
+            storage_config,
         };
 
         Ok(workspace)

rust/noosphere-storage/examples/bench/main.rs

@@ -132,17 +132,15 @@ impl BenchmarkStorage {
         ))]
         let (storage, storage_name) = {
             (
-                noosphere_storage::SledStorage::new(noosphere_storage::SledStorageInit::Path(
-                    storage_path.into(),
-                ))?,
+                noosphere_storage::SledStorage::new(&storage_path)?,
                 "SledDbStorage",
             )
         };
 
         #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))]
         let (storage, storage_name) = {
             (
-                noosphere_storage::RocksDbStorage::new(storage_path.into())?,
+                noosphere_storage::RocksDbStorage::new(&storage_path)?,
                 "RocksDbStorage",
             )
         };

rust/noosphere-storage/src/config.rs

@@ -0,0 +1,25 @@
+use crate::storage::Storage;
+use anyhow::Result;
+use async_trait::async_trait;
+use noosphere_common::ConditionalSend;
+use std::path::Path;
+
+/// Generalized configurations for [ConfigurableStorage].
+#[derive(Debug, Clone, Default)]
+pub struct StorageConfig {
+    /// If set, the size limit in bytes of a memory-based cache.
+    pub memory_cache_limit: Option<usize>,
+}
+
+/// [Storage] that can be customized via [StorageConfig].
+///
+/// Configurations are generalized across storage providers,
+/// and may have differing underlying semantics.
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait ConfigurableStorage: Storage {
+    async fn open_with_config<P: AsRef<Path> + ConditionalSend>(
+        path: P,
+        config: StorageConfig,
+    ) -> Result<Self>;
+}

rust/noosphere-storage/src/helpers.rs

@@ -2,7 +2,7 @@ use crate::Storage;
 use anyhow::Result;
 
 #[cfg(not(target_arch = "wasm32"))]
-use crate::{SledStorage, SledStorageInit, SledStore};
+use crate::{SledStorage, SledStore};
 
 #[cfg(not(target_arch = "wasm32"))]
 pub async fn make_disposable_store() -> Result<SledStore> {
@@ -13,7 +13,7 @@ pub async fn make_disposable_store() -> Result<SledStore> {
         .into_iter()
         .map(String::from)
         .collect();
-    let provider = SledStorage::new(SledStorageInit::Path(temp_dir.join(temp_name)))?;
+    let provider = SledStorage::new(temp_dir.join(temp_name))?;
     provider.get_block_store("foo").await
 }