From 3c8aed1039c85086e3f6444f74eec2fe44e6c7a5 Mon Sep 17 00:00:00 2001 From: Justin Abrahms Date: Fri, 12 Jan 2024 14:29:00 -0800 Subject: [PATCH 1/5] feat!: Add axum tooling to provide w3c tracecontexts in response payloads --- Cargo.lock | 140 +++++++++++++++++- Cargo.toml | 1 + rust/noosphere-gateway/Cargo.toml | 1 + rust/noosphere-gateway/src/gateway.rs | 3 + rust/noosphere-ns/Cargo.toml | 1 + .../noosphere-ns/src/server/implementation.rs | 3 + 6 files changed, 144 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0a66d580..9203795a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,6 +490,24 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "axum-tracing-opentelemetry" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96874d8776ed9834dbb02c234a51f4f439529cd833bc3ccc4bfbbe6d1821d0d2" +dependencies = [ + "axum", + "futures-core", + "futures-util", + "http", + "opentelemetry", + "pin-project-lite", + "tower", + "tracing", + "tracing-opentelemetry", + "tracing-opentelemetry-instrumentation-sdk", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -1007,6 +1025,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.15" @@ -1022,12 +1049,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -3617,6 +3641,7 @@ dependencies = [ "async-stream", "async-trait", "axum", + "axum-tracing-opentelemetry", "bytes", "cid", "iroh-car", @@ -3714,6 +3739,7 @@ dependencies = [ "anyhow", "async-trait", "axum", + "axum-tracing-opentelemetry", "cid", "clap", "futures", @@ -3912,12 +3938,69 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +dependencies = [ + "futures-core", + "futures-sink", + "indexmap 2.0.0", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry-http" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f51189ce8be654f9b5f7e70e49967ed894e84a06fc35c6c042e64ac1fc5399e" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "ordered-float", + "percent-encoding", + "rand", + "thiserror", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +dependencies = [ + "num-traits", +] + [[package]] name = "overload" version = "0.1.1" @@ -5771,6 +5854,37 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-opentelemetry-instrumentation-sdk" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b16403065e0fb78b4708ed0e7245024a5e2eaeeed043dcadba4f32af327a397" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-http", + "tracing", + "tracing-opentelemetry", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -6062,6 +6176,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -6307,6 +6427,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki" version = "0.21.4" diff --git a/Cargo.toml b/Cargo.toml index 4e14e7769..3af955692 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ anyhow = { version = "1" } async-recursion = { version = "1" } async-stream = { version = "0.3" } axum = { version = "^0.6.18" } +axum-tracing-opentelemetry = { version = "0.15.0" } base64 = { version = "^0.21" } byteorder = { version = "~1.4" } # keep in sync with pinned libipld-* crates bytes = { version = "^1" } diff --git a/rust/noosphere-gateway/Cargo.toml b/rust/noosphere-gateway/Cargo.toml index 8f32f4059..02c656215 100644 --- a/rust/noosphere-gateway/Cargo.toml +++ b/rust/noosphere-gateway/Cargo.toml @@ -19,6 +19,7 @@ readme = "README.md" test-kubo = [] [dependencies] +axum-tracing-opentelemetry.workspace = true tracing = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] diff --git a/rust/noosphere-gateway/src/gateway.rs b/rust/noosphere-gateway/src/gateway.rs index d37c5f70f..0716ad100 100644 --- a/rust/noosphere-gateway/src/gateway.rs +++ b/rust/noosphere-gateway/src/gateway.rs @@ -3,6 +3,7 @@ use axum::extract::DefaultBodyLimit; use axum::http::{HeaderValue, Method}; use axum::routing::{get, put}; use axum::{Extension, Router, Server}; +use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; use noosphere_core::api::{v0alpha1, v0alpha2}; use noosphere_core::context::HasMutableSphereContext; use noosphere_ipfs::KuboClient; @@ -114,6 +115,8 @@ impl Gateway { .layer(DefaultBodyLimit::max(DEFAULT_BODY_LENGTH_LIMIT)) .layer(cors) .layer(TraceLayer::new_for_http()) + .layer(OtelInResponseLayer::default()) // include trace context in response + .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request .with_state(Arc::new(manager)); Ok(Self { diff --git a/rust/noosphere-ns/Cargo.toml b/rust/noosphere-ns/Cargo.toml index 0f7ec6d7d..60e08c04a 100644 --- a/rust/noosphere-ns/Cargo.toml +++ b/rust/noosphere-ns/Cargo.toml @@ -22,6 +22,7 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [dependencies] +axum-tracing-opentelemetry.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] anyhow = { workspace = true } diff --git a/rust/noosphere-ns/src/server/implementation.rs b/rust/noosphere-ns/src/server/implementation.rs index a14d67caa..f3ccf0473 100644 --- a/rust/noosphere-ns/src/server/implementation.rs +++ b/rust/noosphere-ns/src/server/implementation.rs @@ -3,6 +3,7 @@ use crate::{DhtClient, NameSystem}; use anyhow::Result; use axum::routing::{delete, get, post}; use axum::{Router, Server}; +use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; use std::net::TcpListener; use std::sync::Arc; use tower_http::trace::TraceLayer; @@ -31,6 +32,8 @@ pub async fn start_name_system_api_server( .route(&Route::PostRecord.to_string(), post(handlers::post_record)) .route(&Route::Bootstrap.to_string(), post(handlers::bootstrap)) .with_state(handlers::RouterState { ns, peer_id }) + .layer(OtelInResponseLayer::default()) // include trace context in response + .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request .layer(TraceLayer::new_for_http()); Server::from_tcp(listener)? From c05ab95212bcb97b021966a583ed27cc64a84242 Mon Sep 17 00:00:00 2001 From: Justin Abrahms Date: Fri, 12 Jan 2024 14:54:38 -0800 Subject: [PATCH 2/5] Fix wasm errors --- rust/noosphere-gateway/Cargo.toml | 2 +- rust/noosphere-ns/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/noosphere-gateway/Cargo.toml b/rust/noosphere-gateway/Cargo.toml index 02c656215..f65fae407 100644 --- a/rust/noosphere-gateway/Cargo.toml +++ b/rust/noosphere-gateway/Cargo.toml @@ -19,7 +19,6 @@ readme = "README.md" test-kubo = [] [dependencies] -axum-tracing-opentelemetry.workspace = true tracing = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] @@ -30,6 +29,7 @@ noosphere-core = { version = "0.18.0", path = "../noosphere-core", features = [" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] anyhow = { workspace = true } axum = { workspace = true, features = ["headers", "macros"] } +axum-tracing-opentelemetry.workspace = true iroh-car = { workspace = true } thiserror = { workspace = true } strum = { workspace = true } diff --git a/rust/noosphere-ns/Cargo.toml b/rust/noosphere-ns/Cargo.toml index 60e08c04a..dce46bf98 100644 --- a/rust/noosphere-ns/Cargo.toml +++ b/rust/noosphere-ns/Cargo.toml @@ -22,7 +22,6 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [dependencies] -axum-tracing-opentelemetry.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] anyhow = { workspace = true } @@ -52,6 +51,7 @@ toml = { version = "~0.8", optional = true } # noosphere_ns::server axum = { workspace = true, features = ["json", "headers", "macros"], optional = true } +axum-tracing-opentelemetry.workspace = true reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls"], optional = true } tower-http = { workspace = true, features = ["trace"], optional = true } url = { version = "^2", features = [ "serde" ], optional = true } From 46ce5b3aae2a95a82d5cb27489f21b500f784f89 Mon Sep 17 00:00:00 2001 From: Justin Abrahms Date: Fri, 12 Jan 2024 15:06:26 -0800 Subject: [PATCH 3/5] Tracelayer needs to come before we start adding otel spans. --- rust/noosphere-gateway/src/gateway.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/noosphere-gateway/src/gateway.rs b/rust/noosphere-gateway/src/gateway.rs index 0716ad100..720bac291 100644 --- a/rust/noosphere-gateway/src/gateway.rs +++ b/rust/noosphere-gateway/src/gateway.rs @@ -114,9 +114,9 @@ impl Gateway { .layer(Extension(cleanup_tx)) .layer(DefaultBodyLimit::max(DEFAULT_BODY_LENGTH_LIMIT)) .layer(cors) - .layer(TraceLayer::new_for_http()) .layer(OtelInResponseLayer::default()) // include trace context in response .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request + .layer(TraceLayer::new_for_http()) .with_state(Arc::new(manager)); Ok(Self { From 14fa31efd3c4a897e3b924d2c677794eca3fc659 Mon Sep 17 00:00:00 2001 From: Justin Abrahms Date: Fri, 12 Jan 2024 15:09:31 -0800 Subject: [PATCH 4/5] Remove default for unit struct --- rust/noosphere-gateway/src/gateway.rs | 2 +- rust/noosphere-ns/src/server/implementation.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/noosphere-gateway/src/gateway.rs b/rust/noosphere-gateway/src/gateway.rs index 720bac291..4709e1222 100644 --- a/rust/noosphere-gateway/src/gateway.rs +++ b/rust/noosphere-gateway/src/gateway.rs @@ -114,7 +114,7 @@ impl Gateway { .layer(Extension(cleanup_tx)) .layer(DefaultBodyLimit::max(DEFAULT_BODY_LENGTH_LIMIT)) .layer(cors) - .layer(OtelInResponseLayer::default()) // include trace context in response + .layer(OtelInResponseLayer) // include trace context in response .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request .layer(TraceLayer::new_for_http()) .with_state(Arc::new(manager)); diff --git a/rust/noosphere-ns/src/server/implementation.rs b/rust/noosphere-ns/src/server/implementation.rs index f3ccf0473..d03e0fc25 100644 --- a/rust/noosphere-ns/src/server/implementation.rs +++ b/rust/noosphere-ns/src/server/implementation.rs @@ -31,10 +31,10 @@ pub async fn start_name_system_api_server( .route(&Route::GetRecord.to_string(), get(handlers::get_record)) .route(&Route::PostRecord.to_string(), post(handlers::post_record)) .route(&Route::Bootstrap.to_string(), post(handlers::bootstrap)) - .with_state(handlers::RouterState { ns, peer_id }) - .layer(OtelInResponseLayer::default()) // include trace context in response + .layer(OtelInResponseLayer) // include trace context in response .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request - .layer(TraceLayer::new_for_http()); + .layer(TraceLayer::new_for_http()) + .with_state(handlers::RouterState { ns, peer_id }); Server::from_tcp(listener)? .serve(app.into_make_service()) From 668756a2983142392cc9d5974004d714864084fa Mon Sep 17 00:00:00 2001 From: Jordan Santell Date: Tue, 16 Jan 2024 10:42:34 -0800 Subject: [PATCH 5/5] chore: Feature gate otel dependencies. --- rust/noosphere-cli/Cargo.toml | 3 +- rust/noosphere-gateway/Cargo.toml | 4 ++- rust/noosphere-gateway/src/gateway.rs | 31 ++++++++++++------- rust/noosphere-ns/Cargo.toml | 5 +-- .../noosphere-ns/src/server/implementation.rs | 16 +++++++--- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/rust/noosphere-cli/Cargo.toml b/rust/noosphere-cli/Cargo.toml index 99c1dc029..ad1592918 100644 --- a/rust/noosphere-cli/Cargo.toml +++ b/rust/noosphere-cli/Cargo.toml @@ -17,9 +17,10 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [features] -default = [] +default = ["observability"] helpers = ["tracing-subscriber", "noosphere-ns"] rocksdb = ["noosphere/rocksdb"] +observability = ["noosphere-gateway/observability"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/rust/noosphere-gateway/Cargo.toml b/rust/noosphere-gateway/Cargo.toml index f65fae407..a9539c58a 100644 --- a/rust/noosphere-gateway/Cargo.toml +++ b/rust/noosphere-gateway/Cargo.toml @@ -16,7 +16,9 @@ homepage = "https://github.com/subconsciousnetwork/noosphere" readme = "README.md" [features] +default = [] test-kubo = [] +observability = ["axum-tracing-opentelemetry"] [dependencies] tracing = { workspace = true } @@ -29,7 +31,7 @@ noosphere-core = { version = "0.18.0", path = "../noosphere-core", features = [" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] anyhow = { workspace = true } axum = { workspace = true, features = ["headers", "macros"] } -axum-tracing-opentelemetry.workspace = true +axum-tracing-opentelemetry = { workspace = true, optional = true } iroh-car = { workspace = true } thiserror = { workspace = true } strum = { workspace = true } diff --git a/rust/noosphere-gateway/src/gateway.rs b/rust/noosphere-gateway/src/gateway.rs index 4709e1222..7d82d444d 100644 --- a/rust/noosphere-gateway/src/gateway.rs +++ b/rust/noosphere-gateway/src/gateway.rs @@ -1,9 +1,16 @@ +use crate::GatewayManager; +use crate::{ + handlers, + worker::{ + start_cleanup, start_ipfs_syndication, start_name_system, NameSystemConfiguration, + NameSystemConnectionType, + }, +}; use anyhow::Result; use axum::extract::DefaultBodyLimit; use axum::http::{HeaderValue, Method}; use axum::routing::{get, put}; use axum::{Extension, Router, Server}; -use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; use noosphere_core::api::{v0alpha1, v0alpha2}; use noosphere_core::context::HasMutableSphereContext; use noosphere_ipfs::KuboClient; @@ -15,14 +22,8 @@ use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; use url::Url; -use crate::GatewayManager; -use crate::{ - handlers, - worker::{ - start_cleanup, start_ipfs_syndication, start_name_system, NameSystemConfiguration, - NameSystemConnectionType, - }, -}; +#[cfg(feature = "observability")] +use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; const DEFAULT_BODY_LENGTH_LIMIT: usize = 100 /* MB */ * 1000 * 1000; @@ -113,9 +114,15 @@ impl Gateway { .layer(Extension(name_system_tx)) .layer(Extension(cleanup_tx)) .layer(DefaultBodyLimit::max(DEFAULT_BODY_LENGTH_LIMIT)) - .layer(cors) - .layer(OtelInResponseLayer) // include trace context in response - .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request + .layer(cors); + + #[cfg(feature = "observability")] + let app = { + app.layer(OtelInResponseLayer) // include trace context in response + .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request + }; + + let app = app .layer(TraceLayer::new_for_http()) .with_state(Arc::new(manager)); diff --git a/rust/noosphere-ns/Cargo.toml b/rust/noosphere-ns/Cargo.toml index dce46bf98..3d91cd415 100644 --- a/rust/noosphere-ns/Cargo.toml +++ b/rust/noosphere-ns/Cargo.toml @@ -51,7 +51,7 @@ toml = { version = "~0.8", optional = true } # noosphere_ns::server axum = { workspace = true, features = ["json", "headers", "macros"], optional = true } -axum-tracing-opentelemetry.workspace = true +axum-tracing-opentelemetry = { workspace = true, optional = true } reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls"], optional = true } tower-http = { workspace = true, features = ["trace"], optional = true } url = { version = "^2", features = [ "serde" ], optional = true } @@ -64,9 +64,10 @@ libipld-cbor = { workspace = true } tempfile = { workspace = true } [features] -default = ["orb-ns", "api-server"] +default = ["orb-ns", "api-server", "observability"] api-server = ["axum", "reqwest", "url", "tower-http"] orb-ns = ["clap", "noosphere", "home", "toml", "noosphere-ipfs"] +observability = ["axum-tracing-opentelemetry"] [[bin]] name = "orb-ns" diff --git a/rust/noosphere-ns/src/server/implementation.rs b/rust/noosphere-ns/src/server/implementation.rs index d03e0fc25..f58f2ce35 100644 --- a/rust/noosphere-ns/src/server/implementation.rs +++ b/rust/noosphere-ns/src/server/implementation.rs @@ -3,11 +3,13 @@ use crate::{DhtClient, NameSystem}; use anyhow::Result; use axum::routing::{delete, get, post}; use axum::{Router, Server}; -use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; use std::net::TcpListener; use std::sync::Arc; use tower_http::trace::TraceLayer; +#[cfg(feature = "observability")] +use axum_tracing_opentelemetry::middleware::{OtelAxumLayer, OtelInResponseLayer}; + pub async fn start_name_system_api_server( ns: Arc, listener: TcpListener, @@ -30,9 +32,15 @@ pub async fn start_name_system_api_server( .route(&Route::Address.to_string(), get(handlers::get_address)) .route(&Route::GetRecord.to_string(), get(handlers::get_record)) .route(&Route::PostRecord.to_string(), post(handlers::post_record)) - .route(&Route::Bootstrap.to_string(), post(handlers::bootstrap)) - .layer(OtelInResponseLayer) // include trace context in response - .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request + .route(&Route::Bootstrap.to_string(), post(handlers::bootstrap)); + + #[cfg(feature = "observability")] + let app = { + app.layer(OtelInResponseLayer) // include trace context in response + .layer(OtelAxumLayer::default()) // initialize otel trace on incoming request + }; + + let app = app .layer(TraceLayer::new_for_http()) .with_state(handlers::RouterState { ns, peer_id });