From b811e6891aec648d9a856adaeda86335ae94cacb Mon Sep 17 00:00:00 2001 From: Christopher Joel <240083+cdata@users.noreply.github.com> Date: Thu, 10 Aug 2023 16:50:51 -0700 Subject: [PATCH] fix: Better handling of removed content in `orb` (#588) --- Cargo.lock | 115 ++++-------------- .../src/native/commands/sphere/history.rs | 19 ++- .../noosphere-cli/src/native/render/writer.rs | 8 ++ rust/noosphere-cli/tests/cli.rs | 15 ++- rust/noosphere-ipfs/Cargo.toml | 2 +- rust/noosphere-ipfs/src/client/kubo.rs | 19 ++- 6 files changed, 82 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 090e772d7..5917927f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -885,7 +885,7 @@ dependencies = [ "multihash 0.18.1", "serde", "serde_bytes", - "unsigned-varint 0.7.1", + "unsigned-varint", ] [[package]] @@ -2016,7 +2016,7 @@ dependencies = [ "indexmap 1.9.3", "slab", "tokio", - "tokio-util 0.7.8", + "tokio-util", "tracing", ] @@ -2441,9 +2441,9 @@ dependencies = [ [[package]] name = "ipfs-api-prelude" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbaf47fa129710ae041d5844a15b1365bdad8551673aee237449ba9dec6bcadc" +checksum = "9b74065805db266ba2c6edbd670b23c4714824a955628472b2e46cc9f3a869cb" dependencies = [ "async-trait", "bytes", @@ -2452,14 +2452,14 @@ dependencies = [ "dirs", "futures", "http", + "multiaddr", "multibase", - "parity-multiaddr", "serde", "serde_json", "serde_urlencoded", "thiserror", "tokio", - "tokio-util 0.6.10", + "tokio-util", "tracing", "walkdir", ] @@ -2482,7 +2482,7 @@ dependencies = [ "libipld", "thiserror", "tokio", - "unsigned-varint 0.7.1", + "unsigned-varint", ] [[package]] @@ -2735,7 +2735,7 @@ dependencies = [ "serde", "smallvec", "thiserror", - "unsigned-varint 0.7.1", + "unsigned-varint", "void", ] @@ -2781,7 +2781,7 @@ dependencies = [ "sha2 0.10.7", "smallvec", "thiserror", - "unsigned-varint 0.7.1", + "unsigned-varint", "void", "wasm-timer", ] @@ -2852,7 +2852,7 @@ dependencies = [ "smallvec", "thiserror", "uint", - "unsigned-varint 0.7.1", + "unsigned-varint", "void", ] @@ -3029,7 +3029,7 @@ dependencies = [ "thiserror", "tinytemplate", "tokio", - "tokio-util 0.7.8", + "tokio-util", "webrtc", ] @@ -3240,7 +3240,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint 0.7.1", + "unsigned-varint", "url", ] @@ -3255,17 +3255,6 @@ dependencies = [ "data-encoding-macro", ] -[[package]] -name = "multihash" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac63698b887d2d929306ea48b63760431ff8a24fac40ddb22f9c7f49fb7cab" -dependencies = [ - "generic-array", - "multihash-derive 0.7.2", - "unsigned-varint 0.5.1", -] - [[package]] name = "multihash" version = "0.17.0" @@ -3274,11 +3263,11 @@ checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" dependencies = [ "core2", "digest 0.10.7", - "multihash-derive 0.8.1", + "multihash-derive", "serde", "serde-big-array", "sha2 0.10.7", - "unsigned-varint 0.7.1", + "unsigned-varint", ] [[package]] @@ -3292,26 +3281,12 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive 0.8.1", + "multihash-derive", "serde", "serde-big-array", "sha2 0.10.7", "sha3", - "unsigned-varint 0.7.1", -] - -[[package]] -name = "multihash-derive" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424f6e86263cd5294cbd7f1e95746b95aca0e0d66bff31e5a40d6baa87b4aa99" -dependencies = [ - "proc-macro-crate", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", - "synstructure", + "unsigned-varint", ] [[package]] @@ -3339,7 +3314,7 @@ dependencies = [ "log", "pin-project", "smallvec", - "unsigned-varint 0.7.1", + "unsigned-varint", ] [[package]] @@ -3472,7 +3447,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.8", + "tokio-util", "tracing", "ucan", "ucan-key-support", @@ -3501,7 +3476,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.8", + "tokio-util", "tracing", "ucan", "ucan-key-support", @@ -3576,7 +3551,7 @@ dependencies = [ "sha2 0.10.7", "tokio", "tokio-stream", - "unsigned-varint 0.7.1", + "unsigned-varint", "wasm-bindgen-test", ] @@ -3683,7 +3658,7 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", - "tokio-util 0.7.8", + "tokio-util", "tower-http", "tracing", "ucan", @@ -3775,7 +3750,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.8", + "tokio-util", "tracing", "ucan", "ucan-key-support", @@ -3995,24 +3970,6 @@ dependencies = [ "libm 0.1.4", ] -[[package]] -name = "parity-multiaddr" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58341485071825827b7f03cf7efd1cb21e6a709bea778fb50227fd45d2f361b4" -dependencies = [ - "arrayref", - "bs58 0.4.0", - "byteorder", - "data-encoding", - "multihash 0.13.2", - "percent-encoding", - "serde", - "static_assertions", - "unsigned-varint 0.7.1", - "url", -] - [[package]] name = "parking" version = "2.1.0" @@ -4382,7 +4339,7 @@ dependencies = [ "bytes", "quick-protobuf", "thiserror", - "unsigned-varint 0.7.1", + "unsigned-varint", ] [[package]] @@ -4617,7 +4574,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-rustls", - "tokio-util 0.7.8", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -5665,20 +5622,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.8" @@ -5772,7 +5715,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tokio", - "tokio-util 0.7.8", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -5977,7 +5920,7 @@ dependencies = [ "serde_json", "strum 0.24.1", "strum_macros", - "unsigned-varint 0.7.1", + "unsigned-varint", "url", ] @@ -6086,12 +6029,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsigned-varint" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fdeedbf205afadfe39ae559b75c3240f24e257d0ca27e85f85cb82aa19ac35" - [[package]] name = "unsigned-varint" version = "0.7.1" diff --git a/rust/noosphere-cli/src/native/commands/sphere/history.rs b/rust/noosphere-cli/src/native/commands/sphere/history.rs index d1cbf5a35..12e783be2 100644 --- a/rust/noosphere-cli/src/native/commands/sphere/history.rs +++ b/rust/noosphere-cli/src/native/commands/sphere/history.rs @@ -47,14 +47,29 @@ Author: {author}"# if !content_changes.is_empty() { let mut removed = Vec::new(); + let mut any_content_added = false; + for change in content_changes { match change { - MapOperation::Add { key, .. } => info!("{: >12} /{key}", "Modified"), + MapOperation::Add { key, .. } => { + any_content_added = true; + info!("{: >12} /{key}", "Modified") + } MapOperation::Remove { key } => removed.push(key), } } - info!("") + if any_content_added { + info!(""); + } + + for slug in &removed { + info!("{: >12} /{slug}", "Removed"); + } + + if !removed.is_empty() { + info!("") + } } let petname_changes = mutation.identities().changes(); diff --git a/rust/noosphere-cli/src/native/render/writer.rs b/rust/noosphere-cli/src/native/render/writer.rs index 98f0237e2..01179e0d7 100644 --- a/rust/noosphere-cli/src/native/render/writer.rs +++ b/rust/noosphere-cli/src/native/render/writer.rs @@ -120,6 +120,14 @@ impl SphereWriter { pub async fn remove_content(&self, slug: &str) -> Result<()> { if self.is_root_writer() { let slug_path = self.paths.slug(slug)?; + + if !slug_path.exists() { + trace!( + "No slug link found at '{}', skipping removal of '{slug}'...", + slug_path.display() + ); + } + let file_path = tokio::fs::read_link(&slug_path).await?; let _ = remove_symlink_file(slug_path); diff --git a/rust/noosphere-cli/tests/cli.rs b/rust/noosphere-cli/tests/cli.rs index ea4fe50b2..426c8972f 100644 --- a/rust/noosphere-cli/tests/cli.rs +++ b/rust/noosphere-cli/tests/cli.rs @@ -355,10 +355,21 @@ mod multiplayer { .spawn(move |mut ctx| async move { // Give the graph state the opportunity to "settle" wait(2).await; + + // Change up petnames a bit ctx.set_petname("peer3", None).await?; ctx.set_petname("peer2", None).await?; ctx.set_petname("peer2-renamed", Some(sphere_2_id)).await?; + + // Add some new content... + ctx.write("never-seen", "text/plain", "boo".as_bytes(), None) + .await?; ctx.save(None).await?; + + // ...and remove it again: + ctx.remove("never-seen").await?; + ctx.save(None).await?; + ctx.sync(SyncRecovery::Retry(3)).await?; wait(2).await; let version = ctx.sync(SyncRecovery::Retry(3)).await?; @@ -410,6 +421,8 @@ mod multiplayer { "@peer2-renamed/@peer3-of-peer2/@peer4-of-peer3/@peer5/content5.txt"; let unexpected_content = [ + // Content added and removed remotely before local sync + "never-seen", // Peer removed "@peer3/content3.txt", // Peer renamed @@ -430,7 +443,7 @@ mod multiplayer { wait(1).await; // Sync again, but with a greater render depth - cli.orb(&["sphere", "sync", "--auto-retry", "3", "--render-depth", "4"]) + cli.orb(&["sphere", "sync", "--auto-retry", "3", "--render-depth", "5"]) .await?; // Previously omitted peer should be rendered now diff --git a/rust/noosphere-ipfs/Cargo.toml b/rust/noosphere-ipfs/Cargo.toml index c3e3fb982..d440446a8 100644 --- a/rust/noosphere-ipfs/Cargo.toml +++ b/rust/noosphere-ipfs/Cargo.toml @@ -45,7 +45,7 @@ ucan = { workspace = true, optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] hyper = { version = "^0.14.27", features = ["full"] } hyper-multipart-rfc7578 = "~0.8" -ipfs-api-prelude = "~0.5" +ipfs-api-prelude = "0.6" [dev-dependencies] diff --git a/rust/noosphere-ipfs/src/client/kubo.rs b/rust/noosphere-ipfs/src/client/kubo.rs index a93ac2e66..41b52c584 100644 --- a/rust/noosphere-ipfs/src/client/kubo.rs +++ b/rust/noosphere-ipfs/src/client/kubo.rs @@ -10,9 +10,11 @@ use hyper::{ client::connect::dns::GaiResolver, client::HttpConnector, Body, Client, Request, StatusCode, }; use hyper_multipart_rfc7578::client::multipart::{Body as MultipartBody, Form}; -use ipfs_api_prelude::response::{IdResponse, PinLsResponse}; +// TODO(#587): Remove dependency on `ipfs-api` crate +use ipfs_api_prelude::response::PinLsResponse; use libipld_cbor::DagCborCodec; use libipld_core::raw::RawCodec; +use serde_json::Value; use url::Url; /// Maps a codec defined in a [Cid] to a string @@ -83,8 +85,19 @@ impl IpfsClient for KuboClient { let body_bytes = hyper::body::to_bytes(response.into_body()).await?; - let IdResponse { public_key, .. } = serde_json::from_slice(body_bytes.as_ref())?; - Ok(public_key) + let identity_response: Value = serde_json::from_slice(body_bytes.as_ref())?; + + if let Some(identity) = identity_response.as_object() { + if let Some(public_key_value) = identity.get("PublicKey") { + if let Some(public_key) = public_key_value.as_str() { + return Ok(public_key.into()); + } + } + } + + Err(anyhow!( + "Could not discover public key from /api/v0/id response" + )) } #[instrument(skip(self, car), level = "trace")]