chore!: Templatize the two IPFS HTTP APIs as noosphere_ipfs::IpfsClie…

…nt, and reconfigure KuboStorage as IpfsStorage, operating on IpfsClient rather than a URL. (#252)

rust/noosphere-ipfs/Cargo.toml

@@ -23,25 +23,26 @@ readme = "README.md"
 test_kubo = []
 
 [dependencies]
-
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 anyhow = "^1"
 async-compat = { version = "~0.2" }
 async-trait = "~0.1"
 cid = "~0.9"
-hyper = { version = "~0.14", features = ["full"] }
-hyper-multipart-rfc7578 = "~0.8"
-ipfs-api-prelude = "~0.5"
+reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls"] }
 serde = "^1"
 serde_json = "^1"
 tokio = { version = "^1", features = ["io-util"] }
 tracing = "0.1"
 url = { version = "^2", features = [ "serde" ] }
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+hyper = { version = "~0.14", features = ["full"] }
+hyper-multipart-rfc7578 = "~0.8"
+ipfs-api-prelude = "~0.5"
+
 [dev-dependencies]
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 iroh-car = { version = "0.1.3" }
 libipld-cbor = "~0.15"
 noosphere-storage = { version = "0.4.2", path = "../noosphere-storage" }
-noosphere-core = { version = "0.6.3", path = "../noosphere-core" }
+noosphere-core = { version = "0.6.3", path = "../noosphere-core" }

rust/noosphere-ipfs/src/client.rs

@@ -3,12 +3,23 @@ use async_trait::async_trait;
 use cid::Cid;
 use tokio::io::AsyncRead;
 
+#[cfg(not(target_arch = "wasm32"))]
+pub trait IpfsClientAsyncReadSendSync: AsyncRead + Send + Sync + 'static {}
+#[cfg(not(target_arch = "wasm32"))]
+impl<S> IpfsClientAsyncReadSendSync for S where S: AsyncRead + Send + Sync + 'static {}
+
+#[cfg(target_arch = "wasm32")]
+pub trait IpfsClientAsyncReadSendSync: AsyncRead {}
+#[cfg(target_arch = "wasm32")]
+impl<S> IpfsClientAsyncReadSendSync for S where S: AsyncRead {}
+
 /// A generic interface for interacting with an IPFS-like backend where it may
 /// be desirable to syndicate sphere data to. Although the interface was
 /// designed after a small subset of the capabilities of IPFS Kubo, it is
 /// intended to be general enough to apply to other IPFS implementations.
-#[async_trait]
-pub trait IpfsClient {
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+pub trait IpfsClient: Clone {
     /// Returns true if the block (referenced by [Cid]) is pinned by the IPFS
     /// server
     async fn block_is_pinned(&self, cid: &Cid) -> Result<bool>;
@@ -24,8 +35,11 @@ pub trait IpfsClient {
     /// descendents to be pinned by association.
     async fn syndicate_blocks<R>(&self, car: R) -> Result<()>
     where
-        R: AsyncRead + Send + Sync + 'static;
+        R: IpfsClientAsyncReadSendSync;
 
     /// Returns the associated block (referenced by [Cid]) if found.
-    async fn get_block(&self, cid: &Cid) -> Result<Vec<u8>>;
+    async fn get_block(&self, cid: &Cid) -> Result<Option<Vec<u8>>>;
+
+    /// Places the associated block with cid on the corresponding backend.
+    async fn put_block(&mut self, cid: &Cid, block: &[u8]) -> Result<()>;
 }

rust/noosphere-ipfs/src/gateway.rs

@@ -0,0 +1,116 @@
+use super::{IpfsClient, IpfsClientAsyncReadSendSync};
+use anyhow::Result;
+use async_trait::async_trait;
+use cid::Cid;
+use reqwest::Client;
+use reqwest::StatusCode;
+use std::str::FromStr;
+use url::Url;
+
+/// A high-level HTTP client for accessing IPFS
+/// [HTTP Gateway](https://docs.ipfs.tech/reference/http/gateway/) and normalizing
+/// their expected payloads to Noosphere-friendly formats.
+#[derive(Clone)]
+pub struct GatewayClient {
+    client: Client,
+    api_url: Url,
+}
+
+impl GatewayClient {
+    pub fn new(api_url: Url) -> Self {
+        let client = Client::new();
+        GatewayClient { client, api_url }
+    }
+
+    pub(crate) fn make_block_url(&self, cid: &Cid) -> Url {
+        let mut url = self.api_url.clone();
+
+        if let Some(domain) = url.domain() {
+            let mut parts = domain.split('.');
+
+            if let Some(fragment) = parts.nth(0) {
+                if Cid::from_str(fragment).is_ok() {
+                    let upper_domain = parts
+                        .map(|part| part.to_string())
+                        .collect::<Vec<String>>()
+                        .join(".");
+
+                    let mut host = format!("{}.{}", cid, upper_domain);
+
+                    if let Some(port) = url.port() {
+                        host = format!("{}:{}", host, port);
+                    }
+
+                    if let Ok(()) = url.set_host(Some(&host)) {
+                        return url;
+                    }
+                }
+            }
+        }
+
+        url.set_path(&format!("/ipfs/{}", cid));
+        url
+    }
+}
+
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+impl IpfsClient for GatewayClient {
+    async fn block_is_pinned(&self, _cid: &Cid) -> Result<bool> {
+        unimplemented!("IPFS HTTP Gateway does not have this capability.");
+    }
+
+    async fn server_identity(&self) -> Result<String> {
+        unimplemented!("IPFS HTTP Gateway does not have this capability.");
+    }
+
+    async fn syndicate_blocks<R>(&self, _car: R) -> Result<()>
+    where
+        R: IpfsClientAsyncReadSendSync,
+    {
+        unimplemented!("IPFS HTTP Gateway does not have this capability.");
+    }
+
+    async fn put_block(&mut self, _cid: &Cid, _block: &[u8]) -> Result<()> {
+        unimplemented!("IPFS HTTP Gateway does not have this capability.");
+    }
+
+    async fn get_block(&self, cid: &Cid) -> Result<Option<Vec<u8>>> {
+        let api_url = self.make_block_url(cid);
+        let response = self
+            .client
+            .get(api_url)
+            .header("Accept", "application/vnd.ipld.raw")
+            .send()
+            .await?;
+
+        match response.status() {
+            StatusCode::OK => Ok(Some(response.bytes().await?.into())),
+            _ => Ok(None),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_can_derive_a_block_url_for_subdomain_gateways() {
+        let gateway_url = Url::from_str(
+            "https://bafybeieh53mh2gt4khnrixfro7wvbvtrux4247cfwse642e36z67medkzq.ipfs.noo.pub",
+        )
+        .unwrap();
+        let test_cid =
+            Cid::from_str("bafy2bzacecsjls67zqx25dcvbu6p4z4rsdkm2k6hanhd5qowrvwmhtov2sjpo")
+                .unwrap();
+        let client = GatewayClient::new(gateway_url.clone());
+        let derived_url = client.make_block_url(&test_cid);
+        let expected_url = Url::from_str(
+            "https://bafy2bzacecsjls67zqx25dcvbu6p4z4rsdkm2k6hanhd5qowrvwmhtov2sjpo.ipfs.noo.pub",
+        )
+        .unwrap();
+
+        assert_eq!(derived_url, expected_url);
+    }
+}

rust/noosphere-ipfs/src/kubo.rs

@@ -1,4 +1,5 @@
-use super::IpfsClient;
+#![cfg(not(target_arch = "wasm32"))]
+use super::{IpfsClient, IpfsClientAsyncReadSendSync};
 use async_trait::async_trait;
 
 use anyhow::{anyhow, Result};
@@ -9,7 +10,6 @@ use hyper::{
 };
 use hyper_multipart_rfc7578::client::multipart::{Body as MultipartBody, Form};
 use ipfs_api_prelude::response::{IdResponse, PinLsResponse};
-use tokio::io::AsyncRead;
 use url::Url;
 
 /// A high-level HTTP client for accessing IPFS
@@ -21,6 +21,16 @@ pub struct KuboClient {
     api_url: Url,
 }
 
+impl KuboClient {
+    pub fn new(api_url: &Url) -> Result<Self> {
+        let client = hyper::Client::builder().build_http();
+        Ok(KuboClient {
+            client,
+            api_url: api_url.clone(),
+        })
+    }
+}
+
 #[async_trait]
 impl IpfsClient for KuboClient {
     async fn block_is_pinned(&self, cid: &Cid) -> Result<bool> {
@@ -63,7 +73,7 @@ impl IpfsClient for KuboClient {
 
     async fn syndicate_blocks<R>(&self, car: R) -> Result<()>
     where
-        R: AsyncRead + Send + Sync + 'static,
+        R: IpfsClientAsyncReadSendSync,
     {
         let mut api_url = self.api_url.clone();
         let mut form = Form::default();
@@ -83,7 +93,11 @@ impl IpfsClient for KuboClient {
         }
     }
 
-    async fn get_block(&self, cid: &Cid) -> Result<Vec<u8>> {
+    async fn put_block(&mut self, _cid: &Cid, _block: &[u8]) -> Result<()> {
+        unimplemented!();
+    }
+
+    async fn get_block(&self, cid: &Cid) -> Result<Option<Vec<u8>>> {
         let mut api_url = self.api_url.clone();
         api_url.set_path("/api/v0/dag/get");
         api_url
@@ -105,23 +119,13 @@ impl IpfsClient for KuboClient {
         match response.status() {
             StatusCode::OK => {
                 let body_bytes = hyper::body::to_bytes(response.into_body()).await?;
-                Ok(body_bytes.into())
+                Ok(Some(body_bytes.into()))
             }
             other_status => Err(anyhow!("Unexpected status code: {}", other_status)),
         }
     }
 }
 
-impl KuboClient {
-    pub fn new(api_url: &Url) -> Result<Self> {
-        let client = hyper::Client::builder().build_http();
-        Ok(KuboClient {
-            client,
-            api_url: api_url.clone(),
-        })
-    }
-}
-
 // Note that these tests require that there is a locally available IPFS Kubo
 // node running with the RPC API enabled
 #[cfg(all(test, feature = "test_kubo"))]
@@ -180,12 +184,12 @@ mod tests {
         assert!(kubo_client.block_is_pinned(&foo_cid).await.unwrap());
         assert!(kubo_client.block_is_pinned(&bar_cid).await.unwrap());
 
-        let foo_bytes = kubo_client.get_block(&foo_cid).await.unwrap();
+        let foo_bytes = kubo_client.get_block(&foo_cid).await.unwrap().unwrap();
         assert_eq!(
             block_deserialize::<DagCborCodec, SomeData>(&foo_bytes).unwrap(),
             foo
         );
-        let bar_bytes = kubo_client.get_block(&bar_cid).await.unwrap();
+        let bar_bytes = kubo_client.get_block(&bar_cid).await.unwrap().unwrap();
         assert_eq!(
             block_deserialize::<DagCborCodec, SomeData>(&bar_bytes).unwrap(),
             bar,

rust/noosphere-ipfs/src/lib.rs

@@ -1,13 +1,13 @@
-#![cfg(not(target_arch = "wasm32"))]
-///! IPFS integration for various backend implementations. Currently only Kubo
-///! has out-of-the-box support, but integration is based on the generalized
-///! [IpfsClient] trait, which opens the possibility for integration with
-///! alternative backends in the future. Integration is currently only one-way,
-///! but eventually this module will be the entrypoint for pulling blocks out of
-///! IPFS backends as well.
-
+///! IPFS integration for various backend implementations.
+///! Provides the generalized [IpfsClient] trait, and implementations
+///! for Kubo's HTTP RPC API, and a more limited IPFS HTTP Gateway.
 mod client;
-mod kubo;
+mod gateway;
 
-pub use client::*;
-pub use kubo::*;
+pub use client::{IpfsClient, IpfsClientAsyncReadSendSync};
+pub use gateway::GatewayClient;
+
+#[cfg(not(target_arch = "wasm32"))]
+mod kubo;
+#[cfg(not(target_arch = "wasm32"))]
+pub use kubo::KuboClient;

rust/noosphere-ns/tests/ns_test.rs

@@ -222,7 +222,6 @@ async fn test_name_system_validation() -> Result<()> {
 
     let sphere_1_cid_1 = derive_cid::<DagCborCodec>(b"00000000");
 
-    // Test propagating records from ns_1 to ns_2
     assert!(
         ns_1.ns
             .put_record(

rust/noosphere-storage/Cargo.toml

@@ -18,8 +18,8 @@ homepage = "https://github.com/subconsciousnetwork/noosphere"
 readme = "README.md"
 
 [features]
-default = ["kubo-storage"]
-kubo-storage = ["reqwest"]
+default = ["ipfs-storage"]
+ipfs-storage = ["noosphere-ipfs"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -37,9 +37,7 @@ libipld-cbor = "~0.15"
 serde = "^1"
 base64 = "=0.13.0"
 url = { version = "^2" }
-
-# Used with the 'kubo-storage' feature
-reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls"], optional = true }
+noosphere-ipfs = { version = "~0.1", path = "../noosphere-ipfs", optional = true }
 
 [dev-dependencies]
 witty-phrase-generator = "~0.2"

rust/noosphere-storage/src/db.rs

@@ -210,26 +210,6 @@ where
     }
 }
 
-#[cfg(all(target_arch = "wasm32", feature = "gateway-storage"))]
-use crate::{KuboStorage, KuboStore};
-#[cfg(all(target_arch = "wasm32", feature = "gateway-storage"))]
-use url::Url;
-
-#[cfg(all(target_arch = "wasm32", feature = "gateway-storage"))]
-impl<S> From<(SphereDb<S>, &Url)> for SphereDb<KuboStorage<S>>
-where
-    S: Storage,
-{
-    fn from((db, ipfs_api): (SphereDb<S>, &Url)) -> Self {
-        SphereDb {
-            block_store: KuboStore::new(db.block_store, Some(ipfs_api)),
-            link_store: db.link_store,
-            version_store: db.version_store,
-            metadata_store: db.metadata_store,
-        }
-    }
-}
-
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl<S> BlockStore for SphereDb<S>