feat: Introduce a streaming way to encode BodyChunkIpld. Fixes #498

* Introduce a `Scratch` trait, for `Storage` providers to provide a
  temporary read/write space that does not persist.
* Introduce streaming `BodyChunkIpld::encode` and
  `BodyChunkIpld::decode` methods.
* Streaming mechanisms store data in memory until a threshold is hit,
  which then stores to disk.
* Mark non-streaming functions `BodyChunkIpld::load_all_bytes` and
  `BodyChunkIpld::store_bytes` as deprecated.
* Remove `BodyChunkDecoder` (now implemented as `BodyChunkIpld::decode`.
* Promote `bytes` to a workspace dependency.

Cargo.toml

@@ -19,6 +19,7 @@ resolver = "2"
 [workspace.dependencies]
 anyhow = { version = "1" }
 axum = { version = "^0.6.18" }
+bytes = { version = "^1" }
 cid = { version = "0.10" }
 directories = { version = "5" }
 fastcdc = { version = "3.1" }
@@ -30,6 +31,7 @@ libipld = { version = "0.16" }
 libipld-core = { version = "0.16" }
 libipld-cbor = { version = "0.16" }
 pathdiff = { version = "0.2.1" }
+rand = { version = "0.8.5" }
 sentry-tracing = { version = "0.31.5" }
 serde = { version = "^1" }
 serde_json = { version = "^1" }

rust/noosphere-cli/src/native/content.rs

@@ -4,7 +4,7 @@ use anyhow::{anyhow, Result};
 use cid::Cid;
 use globset::{Glob, GlobSet, GlobSetBuilder};
 use noosphere_core::data::{BodyChunkIpld, ContentType};
-use noosphere_storage::{BlockStore, MemoryStore};
+use noosphere_storage::{BlockStore, MemoryStore, Scratch};
 use pathdiff::diff_paths;
 use std::collections::{BTreeMap, BTreeSet};
 use subtext::util::to_slug;
@@ -84,7 +84,10 @@ impl Content {
     /// provided store.
     // TODO(#556): This is slow; we could probably do a concurrent traversal
     // similar to how we traverse when rendering files to disk
-    pub async fn read_all<S: BlockStore>(paths: &SpherePaths, store: &mut S) -> Result<Content> {
+    pub async fn read_all<S: BlockStore + Scratch>(
+        paths: &SpherePaths,
+        store: &mut S,
+    ) -> Result<Content> {
         let root_path = paths.root();
         let mut directories = vec![(None, tokio::fs::read_dir(root_path).await?)];
 
@@ -144,7 +147,7 @@ impl Content {
                 };
 
                 let file_bytes = fs::read(path).await?;
-                let body_cid = BodyChunkIpld::store_bytes(&file_bytes, store).await?;
+                let body_cid = BodyChunkIpld::encode(file_bytes.as_ref(), store).await?;
 
                 content.matched.insert(
                     slug,
@@ -172,7 +175,6 @@ impl Content {
         let mut new_blocks = MemoryStore::default();
         let file_content =
             Content::read_all(workspace.require_sphere_paths()?, &mut new_blocks).await?;
-
         let sphere_context = workspace.sphere_context().await?;
         let walker = SphereWalker::from(&sphere_context);
 

rust/noosphere-core/Cargo.toml

@@ -23,6 +23,8 @@ sentry = ["dep:sentry-tracing"]
 helpers = []
 
 [dependencies]
+bytes = { workspace = true }
+tempfile = { workspace = true }
 tracing = { workspace = true }
 cid = { workspace = true }
 url = { workspace = true }
@@ -35,14 +37,14 @@ async-stream = "~0.3"
 async-once-cell = "~0.4"
 anyhow = "^1"
 thiserror = { workspace = true }
-fastcdc = { workspace = true }
+fastcdc = { workspace = true, features = ["tokio"] }
 futures = "~0.3"
 serde = { workspace = true }
 serde_json = { workspace = true }
 byteorder = "^1.4"
 base64 = "0.21"
 ed25519-zebra = "^3"
-rand = "~0.8"
+rand = { workspace = true }
 once_cell = "^1"
 tiny-bip39 = "^1"
 tokio-stream = "~0.1"