From eaa82f9d48318290f0fdfe9c9cb9474410ad5134 Mon Sep 17 00:00:00 2001 From: Jordan Santell Date: Tue, 19 Sep 2023 13:00:22 -0700 Subject: [PATCH] feat: Replace wnfs-namefilter with deterministic-bloom. Fixes #626 (#644) --- Cargo.lock | 143 +++++------------- Cargo.toml | 3 +- rust/noosphere-gateway/Cargo.toml | 2 +- .../src/worker/syndication.rs | 4 +- 4 files changed, 44 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ce473c59..186850924 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,21 +159,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "0.5.0" @@ -822,18 +807,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "chrono" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "windows-targets", -] - [[package]] name = "cid" version = "0.10.1" @@ -1347,6 +1320,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deterministic-bloom" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12a3873e91e360aee2403cbafd2beb42f02ace06da9b053574518f003aa2490d" +dependencies = [ + "bitvec", + "miette", + "rand_core 0.6.4", + "serde", + "thiserror", + "tracing", + "xxhash-rust", +] + [[package]] name = "digest" version = "0.9.0" @@ -2256,29 +2244,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "iana-time-zone" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows 0.48.0", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -2332,7 +2297,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows 0.34.0", + "windows", ] [[package]] @@ -3217,6 +3182,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "mime" version = "0.3.17" @@ -3668,6 +3656,7 @@ dependencies = [ "axum", "bytes", "cid", + "deterministic-bloom", "iroh-car", "libipld-cbor", "libipld-core", @@ -3691,7 +3680,6 @@ dependencies = [ "ucan-key-support", "url", "wasm-bindgen", - "wnfs-namefilter", ] [[package]] @@ -6629,15 +6617,6 @@ dependencies = [ "windows_x86_64_msvc 0.34.0", ] -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -6783,48 +6762,6 @@ dependencies = [ "rand", ] -[[package]] -name = "wnfs-common" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64f5fae2a182d0264a424b8be448b7c9798ba1eff2fa260b479da54d86d6bbe" -dependencies = [ - "anyhow", - "async-once-cell", - "async-trait", - "bytes", - "chrono", - "futures", - "libipld", - "multihash 0.18.1", - "once_cell", - "rand_core 0.6.4", - "serde", - "thiserror", -] - -[[package]] -name = "wnfs-namefilter" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523d5a2c7f5fd4977ddef3287bb00349a842adc68f5f0de6dfe79ea7c351b70d" -dependencies = [ - "anyhow", - "async-once-cell", - "async-trait", - "bitvec", - "chrono", - "futures", - "libipld", - "multihash 0.18.1", - "rand_core 0.6.4", - "serde", - "sha3", - "thiserror", - "wnfs-common", - "xxhash-rust", -] - [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 46fe037e0..bdbd9c49e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ async-stream = { version = "0.3" } axum = { version = "^0.6.18" } bytes = { version = "^1" } cid = { version = "0.10" } +deterministic-bloom = { version = "0.1.0" } directories = { version = "5" } fastcdc = { version = "3.1" } futures = { version = "0.3" } @@ -64,8 +65,6 @@ wasm-bindgen-test = { version = "^0.3" } wasm-bindgen-futures = { version = "^0.4" } wasm-streams = { version = "0.3.0" } web-sys = { version = "0.3" } -# TODO(#626): Migrate off of this crate -wnfs-namefilter = { version = "0.1.23" } [profile.release] opt-level = 'z' diff --git a/rust/noosphere-gateway/Cargo.toml b/rust/noosphere-gateway/Cargo.toml index 20aee3916..755192c51 100644 --- a/rust/noosphere-gateway/Cargo.toml +++ b/rust/noosphere-gateway/Cargo.toml @@ -36,7 +36,7 @@ tower = { workspace = true } tower-http = { workspace = true, features = ["cors", "trace"] } async-trait = "~0.1" async-stream = { workspace = true } -wnfs-namefilter = { workspace = true } +deterministic-bloom = { workspace = true } url = { workspace = true, features = ["serde"] } mime_guess = "^2" diff --git a/rust/noosphere-gateway/src/worker/syndication.rs b/rust/noosphere-gateway/src/worker/syndication.rs index d5810e356..82b6e3fdd 100644 --- a/rust/noosphere-gateway/src/worker/syndication.rs +++ b/rust/noosphere-gateway/src/worker/syndication.rs @@ -21,8 +21,8 @@ use tokio::{ use tokio_stream::StreamExt; use url::Url; +use deterministic_bloom::const_size::BloomFilter; use iroh_car::{CarHeader, CarWriter}; -use wnfs_namefilter::BloomFilter; /// A [SyndicationJob] is a request to syndicate the blocks of a _counterpart_ /// sphere to the broader IPFS network. @@ -198,7 +198,7 @@ where // TODO(#176): We need to build-up a list of blocks that aren't // able to be loaded so that we can be resilient to incomplete // data when syndicating to IPFS - syndicated_blocks.add(&cid.to_bytes()); + syndicated_blocks.insert(&cid.to_bytes()); let block = db.require_block(&cid).await?;