feat: Introduce web bindings and orb NPM package (#182)

Co-authored-by: Alexander Marks <aomarks@google.com>
Co-authored-by: Sebastian Zimmer <SebastianZimmer@users.noreply.github.com>

.github/workflows/run_test_suite.yaml

@@ -58,7 +58,7 @@ jobs:
           command: test
           args: --features test_kubo
 
-  run-test-suite-web:
+  run-test-suite-web-wasm:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -99,3 +99,52 @@ jobs:
         working-directory: ./rust
         run: CHROMEDRIVER=/usr/local/bin/chromedriver cargo test --target wasm32-unknown-unknown
         shell: bash
+
+  run-test-suite-web-typescript:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: Swatinem/rust-cache@v2
+      - uses: google/wireit@setup-github-actions-caching/v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: true
+          profile: minimal
+          toolchain: stable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: lts/*
+      - name: 'Install environment packages'
+        run: |
+          sudo apt-get update -qqy
+          sudo apt-get install jq protobuf-compiler cmake
+      - name: 'Install Rust/WASM test dependencies'
+        run: |
+          rustup target install wasm32-unknown-unknown
+          cargo install toml-cli
+          WASM_BINDGEN_VERSION=`toml get ./Cargo.lock . | jq '.package | map(select(.name == "wasm-bindgen"))[0].version' | xargs echo`
+          cargo install wasm-bindgen-cli --vers "$WASM_BINDGEN_VERSION"
+          cargo install wasm-opt --locked
+        shell: bash
+      # See: https://github.com/SeleniumHQ/selenium/blob/5d108f9a679634af0bbc387e7e3811bc1565912b/.github/actions/setup-chrome/action.yml
+      - name: 'Setup Chrome and chromedriver'
+        run: |
+          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+          echo "deb http://dl.google.com/linux/chrome/deb/ stable main" | sudo tee -a /etc/apt/sources.list.d/google-chrome.list
+          sudo apt-get update -qqy
+          sudo apt-get -qqy install google-chrome-stable
+          CHROME_VERSION=$(google-chrome-stable --version)
+          CHROME_FULL_VERSION=${CHROME_VERSION%%.*}
+          CHROME_MAJOR_VERSION=${CHROME_FULL_VERSION//[!0-9]}
+          sudo rm /etc/apt/sources.list.d/google-chrome.list
+          export CHROMEDRIVER_VERSION=`curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_MAJOR_VERSION%%.*}`
+          curl -L -O "https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip"
+          unzip chromedriver_linux64.zip && chmod +x chromedriver && sudo mv chromedriver /usr/local/bin
+          chromedriver -version
+        shell: bash
+      - name: 'Install NPM dependencies'
+        working-directory: ./typescript
+        run: npm ci
+      - name: 'Run TypeScript headless browser tests'
+        working-directory: ./typescript
+        run: npm run test

.gitignore

@@ -2,4 +2,8 @@ target
 optimized_wasm
 dist
 .DS_Store
-noosphere.h
+noosphere.h
+lib
+node_modules
+.wireit
+*.tsbuildinfo

rust/noosphere-cli/src/bin/orb.rs

@@ -8,6 +8,7 @@ pub async fn main() -> anyhow::Result<()> {
 }
 
 #[cfg(target_arch = "wasm32")]
+#[allow(unused_must_use)]
 pub fn main() {
     noosphere_cli::web::main();
 }

rust/noosphere-cli/src/native/commands/serve/ipfs/kubo.rs

@@ -106,9 +106,12 @@ mod tests {
     use url::Url;
 
     use super::{IpfsClient, KuboClient};
+    use crate::native::commands::serve::tracing::initialize_tracing;
 
     #[tokio::test]
     pub async fn it_can_interact_with_a_kubo_server() {
+        initialize_tracing();
+
         #[derive(Serialize, Deserialize)]
         struct SomeData {
             value: String,

rust/noosphere-cli/src/native/commands/sphere.rs

@@ -106,7 +106,7 @@ Type or paste the code here and press enter:"#,
         .at_storage_path(workspace.root_directory())
         .reading_keys_from(workspace.key_storage().clone())
         .using_key(local_key)
-        .authorized_by(&Authorization::Cid(cid))
+        .authorized_by(Some(&Authorization::Cid(cid)))
         .build()
         .await?;
 

rust/noosphere-core/src/authority/author.rs

@@ -17,7 +17,7 @@ use ucan_key_support::ed25519::Ed25519KeyMaterial;
 /// The level of access that a given user has to a related resource. Broadly,
 /// a user will always have either read/write access (to their own sphere) or
 /// else read-only access (to all other spheres).
-#[derive(PartialEq, Eq, Debug)]
+#[derive(PartialEq, Eq, Debug, Clone)]
 pub enum Access {
     ReadWrite,
     ReadOnly,

rust/noosphere-fs/Cargo.toml

@@ -32,6 +32,7 @@ cid = "~0.9"
 anyhow = "^1"
 libipld-core = "~0.15"
 libipld-cbor = "~0.15"
+tracing = "~0.1"
 
 futures-util = "~0.3"
 ucan = { version = "0.1.0" }

rust/noosphere-fs/src/decoder.rs

@@ -15,7 +15,7 @@ impl<'a, 'b, S: BlockStore> BodyChunkDecoder<'a, 'b, S> {
         let store = self.1.clone();
         Box::pin(try_stream! {
             while let Some(cid) = next {
-                println!("Unpacking block {}...", cid);
+                debug!("Unpacking block {}...", cid);
                 let chunk = store.load::<DagCborCodec, BodyChunkIpld>(&cid).await.map_err(|error| {
                     std::io::Error::new(std::io::ErrorKind::UnexpectedEof, error.to_string())
                 })?;

rust/noosphere-fs/src/fs.rs

@@ -1,4 +1,5 @@
 use anyhow::{anyhow, Result};
+use async_stream::try_stream;
 use libipld_cbor::DagCborCodec;
 use noosphere_core::{
     authority::{Access, Author},
@@ -8,6 +9,7 @@ use noosphere_core::{
 use noosphere_storage::{BlockStore, SphereDb, Storage};
 use once_cell::sync::OnceCell;
 use std::str::FromStr;
+use tokio_stream::Stream;
 use tokio_util::io::StreamReader;
 use ucan::crypto::KeyMaterial;
 
@@ -37,6 +39,23 @@ where
     mutation: OnceCell<SphereMutation>,
 }
 
+impl<S, K> Clone for SphereFs<S, K>
+where
+    S: Storage,
+    K: KeyMaterial + Clone + 'static,
+{
+    fn clone(&self) -> Self {
+        Self {
+            author: self.author.clone(),
+            access: self.access.clone(),
+            sphere_identity: self.sphere_identity.clone(),
+            sphere_revision: self.sphere_revision.clone(),
+            db: self.db.clone(),
+            mutation: OnceCell::new(),
+        }
+    }
+}
+
 impl<S, K> SphereFs<S, K>
 where
     S: Storage,
@@ -298,10 +317,51 @@ where
 
         Ok(new_sphere_revision)
     }
+
+    /// Get a stream that yields every slug in the namespace along with its
+    /// corresponding [SphereFile]. This is useful for iterating over sphere
+    /// content incrementally without having to load the entire index into
+    /// memory at once.
+    pub fn stream<'a>(
+        &'a self,
+    ) -> impl Stream<Item = Result<(String, SphereFile<impl AsyncRead + 'a>)>> {
+        try_stream! {
+            let sphere = Sphere::at(&self.sphere_revision, &self.db);
+            let links = sphere.try_get_links().await?;
+            let stream = links.stream().await?;
+
+            for await entry in stream {
+                let (key, revision) = entry?;
+                let file = self.get_file(revision).await?;
+
+                yield (key.clone(), file);
+            }
+        }
+    }
+
+    /// Same as `stream`, but consumes the [SphereFs]. This is useful in cases
+    /// where it would otherwise be necessary to borrow a reference to
+    /// [SphereFs] for a static lifetime.
+    pub fn into_stream(self) -> impl Stream<Item = Result<(String, SphereFile<impl AsyncRead>)>> {
+        try_stream! {
+            let sphere = Sphere::at(&self.sphere_revision, &self.db);
+            let links = sphere.try_get_links().await?;
+            let stream = links.stream().await?;
+
+            for await entry in stream {
+                let (key, revision) = entry?;
+                let file = self.get_file(revision).await?;
+
+                yield (key.clone(), file);
+            }
+        }
+    }
 }
 
 #[cfg(test)]
 pub mod tests {
+    use std::collections::BTreeSet;
+
     use noosphere_core::{
         authority::{generate_ed25519_key, Author},
         data::{ContentType, Header},
@@ -310,6 +370,7 @@ pub mod tests {
     use noosphere_storage::MemoryStorage;
     use noosphere_storage::SphereDb;
     use tokio::io::AsyncReadExt;
+    use tokio_stream::StreamExt;
     use ucan::crypto::KeyMaterial;
 
     #[cfg(target_arch = "wasm32")]
@@ -590,4 +651,63 @@ pub mod tests {
         assert!(result.is_err());
         assert_eq!(result.unwrap_err().to_string(), "No changes to save");
     }
+
+    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+    #[cfg_attr(not(target_arch = "wasm32"), tokio::test)]
+    async fn it_can_stream_the_whole_index() {
+        let storage = MemoryStorage::default();
+        let mut db = SphereDb::new(&storage).await.unwrap();
+
+        let owner_key = generate_ed25519_key();
+        let owner_did = owner_key.get_did().await.unwrap();
+
+        let (sphere, authorization, _) = Sphere::try_generate(&owner_did, &mut db).await.unwrap();
+
+        let sphere_identity = sphere.try_get_identity().await.unwrap();
+        let author = Author {
+            key: owner_key,
+            authorization: Some(authorization),
+        };
+
+        db.set_version(&sphere_identity, sphere.cid())
+            .await
+            .unwrap();
+
+        let mut fs = SphereFs::latest(&sphere_identity, &author, &db)
+            .await
+            .unwrap();
+
+        let expected = BTreeSet::<(String, String)>::from([
+            ("cats".into(), "Cats are awesome".into()),
+            ("dogs".into(), "Dogs are pretty cool".into()),
+            ("birds".into(), "Birds rights".into()),
+            ("mice".into(), "Mice like cookies".into()),
+        ]);
+
+        for (slug, content) in &expected {
+            fs.write(
+                slug.as_str(),
+                &ContentType::Subtext.to_string(),
+                content.as_ref(),
+                None,
+            )
+            .await
+            .unwrap();
+
+            fs.save(None).await.unwrap();
+        }
+
+        let mut actual = BTreeSet::new();
+        let stream = fs.stream();
+
+        tokio::pin!(stream);
+
+        while let Some(Ok((slug, mut file))) = stream.next().await {
+            let mut contents = String::new();
+            file.contents.read_to_string(&mut contents).await.unwrap();
+            actual.insert((slug, contents));
+        }
+
+        assert_eq!(expected, actual);
+    }
 }

rust/noosphere-fs/src/lib.rs

@@ -1,3 +1,6 @@
+#[macro_use]
+extern crate tracing;
+
 mod decoder;
 mod file;
 mod fs;

rust/noosphere-storage/Cargo.toml

@@ -17,6 +17,10 @@ repository = "https://github.com/subconsciousnetwork/noosphere"
 homepage = "https://github.com/subconsciousnetwork/noosphere"
 readme = "README.md"
 
+[features]
+default = ["kubo-storage"]
+kubo-storage = ["reqwest"]
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -32,6 +36,10 @@ libipld-core = "~0.15"
 libipld-cbor = "~0.15"
 serde = "^1"
 base64 = "=0.13.0"
+url = { version = "^2" }
+
+# Used with the 'kubo-storage' feature
+reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls"], optional = true }
 
 [dev-dependencies]
 witty-phrase-generator = "~0.2"
@@ -47,6 +55,7 @@ wasm-bindgen = "~0.2"
 rexie = { version = "~0.4" }
 js-sys = "~0.3"
 
+
 [target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
 version = "~0.3"
 features = [