From e242ae0587acec4463adee66b742c9ad0719790a Mon Sep 17 00:00:00 2001 From: Julian Tescher Date: Tue, 7 Mar 2023 09:34:15 -0800 Subject: [PATCH] New metrics SDK This patch updates the metrics SDK to the latest spec. The following breaking changes are introduced. Metrics API changes: * Move `AttributeSet` to SDK as it's not mentioned in the spec or used in the api * Consolidate `AsyncCounter`, `AsyncUpDownCounter`, and `AsyncGauge` into `AsyncInstrument` trait and add downcasting for observer callbacks. * Add `AsyncInstrumentBuilder` to allow per-instrument callback configuration. * Allow metric `name` and `description` fields to be `Cow<'static, str>` * Warn on metric misconfiguration when using instrument builder `init` rather than returning error * Update `Meter::register_callback` to take a list of async instruments and validate they are registered in the callback through the associated `Observer` * Allow registered callbacks to be unregistered. Metrics SDK changes: * Introduce `Scope` as type alias for `InstrumentationLibrary` * Update `Aggregation` to match aggregation spec * Refactor `BasicController` to spec compliant `ManualReader` * Refactor `PushController` to spec compliant `PeriodicReader` * Update metric data fields to match spec, including exemplars. * Split `MetricsExporter` into `Reader`s and `PushMetricExporter`s * Add `View` implementation * Remove `AtomicNumber`s * Refactor `Processor`s into `Pipeline` Metrics exporter changes: * Update otlp exporter to match new metrics data * Update otlp exporter configuration to allow aggregation and temporality selectors to be optional. * Update prometheus exporter to match new metrics data Example changes: * Update otlp metrics and prometheus examples. * Remove basic example as we should be focusing on the OTLP variants --- Cargo.toml | 1 - examples/basic-otlp/Cargo.toml | 5 +- examples/basic-otlp/src/main.rs | 42 +- examples/basic/Cargo.toml | 13 - examples/basic/README.md | 18 - examples/basic/src/main.rs | 109 --- examples/basic/trace.png | Bin 153097 -> 0 bytes examples/dynatrace/Cargo.toml | 8 +- examples/hyper-prometheus/Cargo.toml | 7 +- examples/hyper-prometheus/src/main.rs | 57 +- opentelemetry-api/src/attributes/encoder.rs | 76 -- opentelemetry-api/src/attributes/mod.rs | 144 --- opentelemetry-api/src/common.rs | 2 +- opentelemetry-api/src/lib.rs | 4 - .../src/metrics/instruments/counter.rs | 44 +- .../src/metrics/instruments/gauge.rs | 58 +- .../src/metrics/instruments/mod.rs | 176 +++- .../metrics/instruments/up_down_counter.rs | 52 +- opentelemetry-api/src/metrics/meter.rs | 139 ++- opentelemetry-api/src/metrics/mod.rs | 111 +-- opentelemetry-api/src/metrics/noop.rs | 46 +- opentelemetry-dynatrace/Cargo.toml | 6 +- opentelemetry-otlp/Cargo.toml | 9 +- opentelemetry-otlp/src/lib.rs | 10 +- opentelemetry-otlp/src/metric.rs | 168 ++-- opentelemetry-otlp/src/span.rs | 18 +- opentelemetry-otlp/src/transform/metrics.rs | 861 ++++------------- opentelemetry-otlp/src/transform/mod.rs | 17 - opentelemetry-otlp/src/transform/resource.rs | 6 +- opentelemetry-otlp/tests/smoke.rs | 6 +- opentelemetry-prometheus/Cargo.toml | 7 +- opentelemetry-prometheus/src/config.rs | 108 +++ opentelemetry-prometheus/src/lib.rs | 884 +++++++++-------- opentelemetry-prometheus/src/sanitize.rs | 67 -- .../data/conflict_help_two_counters_1.txt | 11 + .../data/conflict_help_two_counters_2.txt | 11 + .../data/conflict_help_two_histograms_1.txt | 45 + .../data/conflict_help_two_histograms_2.txt | 45 + .../conflict_help_two_updowncounters_1.txt | 11 + .../conflict_help_two_updowncounters_2.txt | 11 + ...flict_type_counter_and_updowncounter_1.txt | 9 + ...flict_type_counter_and_updowncounter_2.txt | 9 + ...ict_type_histogram_and_updowncounter_1.txt | 9 + ...ict_type_histogram_and_updowncounter_2.txt | 26 + .../tests/data/conflict_unit_two_counters.txt | 11 + .../data/conflict_unit_two_histograms.txt | 45 + .../data/conflict_unit_two_updowncounters.txt | 11 + .../tests/data/counter.txt | 10 + .../tests/data/custom_resource.txt | 9 + .../tests/data/empty_resource.txt | 9 + opentelemetry-prometheus/tests/data/gauge.txt | 9 + .../tests/data/histogram.txt | 21 + .../tests/data/multi_scopes.txt | 13 + .../tests/data/no_conflict_two_counters.txt | 11 + .../tests/data/no_conflict_two_histograms.txt | 45 + .../data/no_conflict_two_updowncounters.txt | 11 + .../tests/data/sanitized_labels.txt | 9 + .../tests/data/sanitized_names.txt | 35 + .../data/without_scope_and_target_info.txt | 3 + .../tests/data/without_scope_info.txt | 6 + .../tests/data/without_target_info.txt | 6 + .../tests/integration_test.rs | 889 ++++++++++++++---- opentelemetry-proto/src/proto.rs | 3 - opentelemetry-proto/src/transform/common.rs | 17 +- opentelemetry-proto/src/transform/metrics.rs | 68 +- opentelemetry-sdk/Cargo.toml | 6 +- .../export/metrics/aggregation/temporality.rs | 92 -- opentelemetry-sdk/src/export/metrics/mod.rs | 355 ------- .../src/export/metrics/stdout.rs | 259 ----- opentelemetry-sdk/src/export/mod.rs | 5 +- opentelemetry-sdk/src/instrumentation.rs | 9 +- opentelemetry-sdk/src/lib.rs | 93 +- opentelemetry-sdk/src/metrics/aggregation.rs | 100 ++ .../src/metrics/aggregators/ddsketch.rs | 877 ----------------- .../src/metrics/aggregators/last_value.rs | 118 --- .../src/metrics/aggregators/mod.rs | 97 -- .../src/metrics/aggregators/sum.rs | 74 -- .../src/metrics/attributes/mod.rs | 3 + .../src/metrics/attributes/set.rs | 143 +++ .../src/metrics/controllers/basic.rs | 470 --------- .../src/metrics/controllers/mod.rs | 4 - .../src/metrics/controllers/push.rs | 199 ---- opentelemetry-sdk/src/metrics/data/mod.rs | 167 ++++ .../src/metrics/data/temporality.rs | 16 + opentelemetry-sdk/src/metrics/exporter.rs | 34 + opentelemetry-sdk/src/metrics/instrument.rs | 375 ++++++++ .../src/metrics/internal/aggregator.rs | 30 + .../src/metrics/internal/filter.rs | 74 ++ .../src/metrics/internal/histogram.rs | 288 ++++++ .../src/metrics/internal/last_value.rs | 59 ++ opentelemetry-sdk/src/metrics/internal/mod.rs | 51 + opentelemetry-sdk/src/metrics/internal/sum.rs | 441 +++++++++ .../src/metrics/manual_reader.rs | 215 +++++ opentelemetry-sdk/src/metrics/meter.rs | 628 +++++++++++++ .../src/metrics/meter_provider.rs | 137 +++ opentelemetry-sdk/src/metrics/mod.rs | 465 ++------- .../src/metrics/periodic_reader.rs | 372 ++++++++ opentelemetry-sdk/src/metrics/pipeline.rs | 728 ++++++++++++++ .../src/metrics/processors/basic.rs | 396 -------- .../src/metrics/processors/mod.rs | 4 - opentelemetry-sdk/src/metrics/reader.rs | 160 ++++ opentelemetry-sdk/src/metrics/registry.rs | 129 --- .../src/metrics/sdk_api/async_instrument.rs | 122 --- .../src/metrics/sdk_api/descriptor.rs | 83 -- .../src/metrics/sdk_api/instrument_kind.rs | 60 -- opentelemetry-sdk/src/metrics/sdk_api/mod.rs | 81 -- .../src/metrics/sdk_api/sync_instrument.rs | 87 -- opentelemetry-sdk/src/metrics/sdk_api/wrap.rs | 319 ------- .../src/metrics/selectors/mod.rs | 2 - .../src/metrics/selectors/simple.rs | 43 - opentelemetry-sdk/src/metrics/view.rs | 176 ++++ opentelemetry-sdk/src/resource/env.rs | 7 +- opentelemetry-sdk/src/resource/mod.rs | 9 - opentelemetry-sdk/src/testing/metric.rs | 38 - opentelemetry-sdk/src/testing/mod.rs | 2 +- opentelemetry-sdk/src/testing/trace.rs | 1 + opentelemetry-sdk/tests/metrics.rs | 203 ---- 117 files changed, 6964 insertions(+), 6649 deletions(-) delete mode 100644 examples/basic/Cargo.toml delete mode 100644 examples/basic/README.md delete mode 100644 examples/basic/src/main.rs delete mode 100644 examples/basic/trace.png delete mode 100644 opentelemetry-api/src/attributes/encoder.rs delete mode 100644 opentelemetry-api/src/attributes/mod.rs create mode 100644 opentelemetry-prometheus/src/config.rs delete mode 100644 opentelemetry-prometheus/src/sanitize.rs create mode 100644 opentelemetry-prometheus/tests/data/conflict_help_two_counters_1.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_help_two_counters_2.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_help_two_histograms_1.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_help_two_histograms_2.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_1.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_2.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_1.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_2.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_1.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_2.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_unit_two_counters.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_unit_two_histograms.txt create mode 100644 opentelemetry-prometheus/tests/data/conflict_unit_two_updowncounters.txt create mode 100644 opentelemetry-prometheus/tests/data/counter.txt create mode 100644 opentelemetry-prometheus/tests/data/custom_resource.txt create mode 100644 opentelemetry-prometheus/tests/data/empty_resource.txt create mode 100644 opentelemetry-prometheus/tests/data/gauge.txt create mode 100644 opentelemetry-prometheus/tests/data/histogram.txt create mode 100644 opentelemetry-prometheus/tests/data/multi_scopes.txt create mode 100644 opentelemetry-prometheus/tests/data/no_conflict_two_counters.txt create mode 100644 opentelemetry-prometheus/tests/data/no_conflict_two_histograms.txt create mode 100644 opentelemetry-prometheus/tests/data/no_conflict_two_updowncounters.txt create mode 100644 opentelemetry-prometheus/tests/data/sanitized_labels.txt create mode 100644 opentelemetry-prometheus/tests/data/sanitized_names.txt create mode 100644 opentelemetry-prometheus/tests/data/without_scope_and_target_info.txt create mode 100644 opentelemetry-prometheus/tests/data/without_scope_info.txt create mode 100644 opentelemetry-prometheus/tests/data/without_target_info.txt delete mode 100644 opentelemetry-sdk/src/export/metrics/aggregation/temporality.rs delete mode 100644 opentelemetry-sdk/src/export/metrics/mod.rs delete mode 100644 opentelemetry-sdk/src/export/metrics/stdout.rs create mode 100644 opentelemetry-sdk/src/metrics/aggregation.rs delete mode 100644 opentelemetry-sdk/src/metrics/aggregators/ddsketch.rs delete mode 100644 opentelemetry-sdk/src/metrics/aggregators/last_value.rs delete mode 100644 opentelemetry-sdk/src/metrics/aggregators/mod.rs delete mode 100644 opentelemetry-sdk/src/metrics/aggregators/sum.rs create mode 100644 opentelemetry-sdk/src/metrics/attributes/mod.rs create mode 100644 opentelemetry-sdk/src/metrics/attributes/set.rs delete mode 100644 opentelemetry-sdk/src/metrics/controllers/basic.rs delete mode 100644 opentelemetry-sdk/src/metrics/controllers/mod.rs delete mode 100644 opentelemetry-sdk/src/metrics/controllers/push.rs create mode 100644 opentelemetry-sdk/src/metrics/data/mod.rs create mode 100644 opentelemetry-sdk/src/metrics/data/temporality.rs create mode 100644 opentelemetry-sdk/src/metrics/exporter.rs create mode 100644 opentelemetry-sdk/src/metrics/instrument.rs create mode 100644 opentelemetry-sdk/src/metrics/internal/aggregator.rs create mode 100644 opentelemetry-sdk/src/metrics/internal/filter.rs create mode 100644 opentelemetry-sdk/src/metrics/internal/histogram.rs create mode 100644 opentelemetry-sdk/src/metrics/internal/last_value.rs create mode 100644 opentelemetry-sdk/src/metrics/internal/mod.rs create mode 100644 opentelemetry-sdk/src/metrics/internal/sum.rs create mode 100644 opentelemetry-sdk/src/metrics/manual_reader.rs create mode 100644 opentelemetry-sdk/src/metrics/meter.rs create mode 100644 opentelemetry-sdk/src/metrics/meter_provider.rs create mode 100644 opentelemetry-sdk/src/metrics/periodic_reader.rs create mode 100644 opentelemetry-sdk/src/metrics/pipeline.rs delete mode 100644 opentelemetry-sdk/src/metrics/processors/basic.rs delete mode 100644 opentelemetry-sdk/src/metrics/processors/mod.rs create mode 100644 opentelemetry-sdk/src/metrics/reader.rs delete mode 100644 opentelemetry-sdk/src/metrics/registry.rs delete mode 100644 opentelemetry-sdk/src/metrics/sdk_api/async_instrument.rs delete mode 100644 opentelemetry-sdk/src/metrics/sdk_api/descriptor.rs delete mode 100644 opentelemetry-sdk/src/metrics/sdk_api/instrument_kind.rs delete mode 100644 opentelemetry-sdk/src/metrics/sdk_api/mod.rs delete mode 100644 opentelemetry-sdk/src/metrics/sdk_api/sync_instrument.rs delete mode 100644 opentelemetry-sdk/src/metrics/sdk_api/wrap.rs delete mode 100644 opentelemetry-sdk/src/metrics/selectors/mod.rs delete mode 100644 opentelemetry-sdk/src/metrics/selectors/simple.rs create mode 100644 opentelemetry-sdk/src/metrics/view.rs delete mode 100644 opentelemetry-sdk/src/testing/metric.rs delete mode 100644 opentelemetry-sdk/tests/metrics.rs diff --git a/Cargo.toml b/Cargo.toml index ed54475e0b..69b91814cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ members = [ "examples/actix-udp", "examples/async", "examples/aws-xray", - "examples/basic", "examples/basic-otlp", "examples/basic-otlp-http", "examples/datadog", diff --git a/examples/basic-otlp/Cargo.toml b/examples/basic-otlp/Cargo.toml index 03cd9f4c3b..979d80f66c 100644 --- a/examples/basic-otlp/Cargo.toml +++ b/examples/basic-otlp/Cargo.toml @@ -6,8 +6,9 @@ publish = false [dependencies] futures-util = { version = "0.3", default-features = false, features = ["std"] } -lazy_static = "1.4" -opentelemetry = { path = "../../opentelemetry", features = ["rt-tokio", "metrics"] } +once_cell = "1.17" +opentelemetry_api = { path = "../../opentelemetry-api", features = ["metrics"] } +opentelemetry_sdk = { path = "../../opentelemetry-sdk", features = ["rt-tokio"] } opentelemetry-otlp = { path = "../../opentelemetry-otlp", features = ["tonic", "metrics"] } opentelemetry-semantic-conventions = { path = "../../opentelemetry-semantic-conventions" } serde_json = "1.0" diff --git a/examples/basic-otlp/src/main.rs b/examples/basic-otlp/src/main.rs index ac50e0b147..3a3077bfd8 100644 --- a/examples/basic-otlp/src/main.rs +++ b/examples/basic-otlp/src/main.rs @@ -1,17 +1,14 @@ -use opentelemetry::global::shutdown_tracer_provider; -use opentelemetry::runtime; -use opentelemetry::sdk::export::metrics::aggregation::cumulative_temporality_selector; -use opentelemetry::sdk::metrics::controllers::BasicController; -use opentelemetry::sdk::metrics::selectors; -use opentelemetry::sdk::Resource; -use opentelemetry::trace::TraceError; -use opentelemetry::{global, sdk::trace as sdktrace}; -use opentelemetry::{ +use once_cell::sync::Lazy; +use opentelemetry_api::global; +use opentelemetry_api::global::shutdown_tracer_provider; +use opentelemetry_api::trace::TraceError; +use opentelemetry_api::{ metrics, trace::{TraceContextExt, Tracer}, Context, Key, KeyValue, }; use opentelemetry_otlp::{ExportConfig, WithExportConfig}; +use opentelemetry_sdk::{metrics::MeterProvider, runtime, trace as sdktrace, Resource}; use std::error::Error; use std::time::Duration; @@ -29,20 +26,16 @@ fn init_tracer() -> Result { "trace-demo", )])), ) - .install_batch(opentelemetry::runtime::Tokio) + .install_batch(runtime::Tokio) } -fn init_metrics() -> metrics::Result { +fn init_metrics() -> metrics::Result { let export_config = ExportConfig { endpoint: "http://localhost:4317".to_string(), ..ExportConfig::default() }; opentelemetry_otlp::new_pipeline() - .metrics( - selectors::simple::inexpensive(), - cumulative_temporality_selector(), - runtime::Tokio, - ) + .metrics(runtime::Tokio) .with_exporter( opentelemetry_otlp::new_exporter() .tonic() @@ -54,14 +47,14 @@ fn init_metrics() -> metrics::Result { const LEMONS_KEY: Key = Key::from_static_str("lemons"); const ANOTHER_KEY: Key = Key::from_static_str("ex.com/another"); -lazy_static::lazy_static! { - static ref COMMON_ATTRIBUTES: [KeyValue; 4] = [ +static COMMON_ATTRIBUTES: Lazy<[KeyValue; 4]> = Lazy::new(|| { + [ LEMONS_KEY.i64(10), KeyValue::new("A", "1"), KeyValue::new("B", "2"), KeyValue::new("C", "3"), - ]; -} + ] +}); #[tokio::main] async fn main() -> Result<(), Box> { @@ -69,7 +62,7 @@ async fn main() -> Result<(), Box> { // matches the containing block, reporting traces and metrics during the whole // execution. let _ = init_tracer()?; - let metrics_controller = init_metrics()?; + let meter_provider = init_metrics()?; let cx = Context::new(); let tracer = global::tracer("ex.com/basic"); @@ -79,7 +72,10 @@ async fn main() -> Result<(), Box> { .f64_observable_gauge("ex.com.one") .with_description("A gauge set to 1.0") .init(); - meter.register_callback(move |cx| gauge.observe(cx, 1.0, COMMON_ATTRIBUTES.as_ref()))?; + + meter.register_callback(&[gauge.as_any()], move |cx, observer| { + observer.observe_f64(cx, &gauge, 1.0, COMMON_ATTRIBUTES.as_ref()) + })?; let histogram = meter.f64_histogram("ex.com.two").init(); histogram.record(&cx, 5.5, COMMON_ATTRIBUTES.as_ref()); @@ -106,7 +102,7 @@ async fn main() -> Result<(), Box> { tokio::time::sleep(Duration::from_secs(60)).await; shutdown_tracer_provider(); - metrics_controller.stop(&cx)?; + meter_provider.shutdown()?; Ok(()) } diff --git a/examples/basic/Cargo.toml b/examples/basic/Cargo.toml deleted file mode 100644 index 0ab51091bf..0000000000 --- a/examples/basic/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "basic" -version = "0.1.0" -edition = "2021" -publish = false - -[dependencies] -futures-util = { version = "0.3", default-features = false, features = ["std"] } -lazy_static = "1.4" -opentelemetry = { path = "../../opentelemetry", features = ["rt-tokio", "metrics"] } -opentelemetry-jaeger = { path = "../../opentelemetry-jaeger", features = ["rt-tokio"] } -thrift = "0.13" -tokio = { version = "1.0", features = ["full"] } diff --git a/examples/basic/README.md b/examples/basic/README.md deleted file mode 100644 index d2a3fe8d66..0000000000 --- a/examples/basic/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Basic OpenTelemetry Example - -This example shows basic span and metric usage, and exports to Jaeger. - -## Usage - -```shell -# Run jaeger in background -$ docker run -d -p6831:6831/udp -p6832:6832/udp -p16686:16686 -p14268:14268 jaegertracing/all-in-one:latest - -# Report spans -$ cargo run - -# View spans (see the image below) -$ firefox http://localhost:16686/ -``` - -![Jaeger UI](trace.png) diff --git a/examples/basic/src/main.rs b/examples/basic/src/main.rs deleted file mode 100644 index 0bff3578c5..0000000000 --- a/examples/basic/src/main.rs +++ /dev/null @@ -1,109 +0,0 @@ -use opentelemetry::global::shutdown_tracer_provider; -use opentelemetry::metrics::MetricsError; -use opentelemetry::sdk::metrics::controllers::BasicController; -use opentelemetry::sdk::metrics::{controllers, processors, selectors}; -use opentelemetry::sdk::trace::Config; -use opentelemetry::sdk::{export, trace as sdktrace, Resource}; -use opentelemetry::trace::TraceError; -use opentelemetry::{ - baggage::BaggageExt, - trace::{TraceContextExt, Tracer}, - Context, Key, KeyValue, -}; -use opentelemetry::{global, runtime}; -use std::error::Error; - -fn init_tracer() -> Result { - opentelemetry_jaeger::new_agent_pipeline() - .with_service_name("trace-demo") - .with_trace_config(Config::default().with_resource(Resource::new(vec![ - KeyValue::new("service.name", "new_service"), - KeyValue::new("exporter", "otlp-jaeger"), - ]))) - .install_batch(runtime::Tokio) -} - -fn init_metrics() -> Result { - let exporter = export::metrics::stdout().build()?; - let pusher = controllers::basic(processors::factory( - selectors::simple::inexpensive(), - exporter.temporality_selector(), - )) - .with_exporter(exporter) - .build(); - - let cx = Context::new(); - pusher.start(&cx, runtime::Tokio)?; - - global::set_meter_provider(pusher.clone()); - - Ok(pusher) -} - -const FOO_KEY: Key = Key::from_static_str("ex.com/foo"); -const BAR_KEY: Key = Key::from_static_str("ex.com/bar"); -const LEMONS_KEY: Key = Key::from_static_str("ex.com/lemons"); -const ANOTHER_KEY: Key = Key::from_static_str("ex.com/another"); - -lazy_static::lazy_static! { - static ref COMMON_ATTRIBUTES: [KeyValue; 4] = [ - LEMONS_KEY.i64(10), - KeyValue::new("A", "1"), - KeyValue::new("B", "2"), - KeyValue::new("C", "3"), - ]; -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - // By binding the result to an unused variable, the lifetime of the variable - // matches the containing block, reporting traces and metrics during the whole - // execution. - let _tracer = init_tracer()?; - let controller = init_metrics()?; - let cx = Context::new(); - - let tracer = global::tracer("ex.com/basic"); - let meter = global::meter("ex.com/basic"); - - let observable_counter = meter - .u64_observable_counter("ex.com.one") - .with_description("An observable counter set to 1.0") - .init(); - - let histogram = meter.f64_histogram("ex.com.three").init(); - - let observable_gauge = meter.f64_observable_gauge("ex.com.two").init(); - - let _baggage = - Context::current_with_baggage(vec![FOO_KEY.string("foo1"), BAR_KEY.string("bar1")]) - .attach(); - - tracer.in_span("operation", |cx| { - let span = cx.span(); - span.add_event( - "Nice operation!".to_string(), - vec![Key::new("bogons").i64(100)], - ); - span.set_attribute(ANOTHER_KEY.string("yes")); - - let _ = meter.register_callback(move |cx| { - observable_counter.observe(cx, 1, &[]); - observable_gauge.observe(cx, 2.0, &[]); - }); - - tracer.in_span("Sub operation...", |cx| { - let span = cx.span(); - span.set_attribute(LEMONS_KEY.string("five")); - - span.add_event("Sub span event", vec![]); - - histogram.record(&cx, 1.3, &[]); - }); - }); - - shutdown_tracer_provider(); // sending remaining spans. - controller.stop(&cx)?; // send remaining metrics. - - Ok(()) -} diff --git a/examples/basic/trace.png b/examples/basic/trace.png deleted file mode 100644 index a8ad91dd8840748ed79d35c33c4d2448f1ddfb6d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 153097 zcmb5W1yqz>7e6|5N=OSsOGt;*kb)qHh=K}&q;w4;4BY|(f`o)Hv|`aU^bpb^T>}i= zGjzv&)c5_q@4xO{_pW=^EY=g}Jm;K!_TFcoy?>jxx;h%yNN1UpDidH z4v`At!_VP+9ydMJV}=s~PN`1W3hyrsQqLgE57T7*GqJ!q3h3ZMi;i`lSX(0Z^h6C# z>d8gW$|;28&o|8wcq|v?L(e6~(OZufY4W5Gq%uKCMUwFs#z49scc4Se-;WLOBf_R&k|wz&-w!o7%}(a$^SzgPzzWye(vwgL1B=h4>y{%Mo@npT+NF9)D+cn z)@zXd;A&a&dE_4BZ~f_SoKIuK7*Fh5)Ck z;edF;Thm5=k_Tv_Jo&8w>^yj*jv` zV3@>I37f8B^OoR)?)8!HKOMW1gqo5QESkF3Pxe-p`rr^-4M}O~mC4nnv#`+iJk12% zXK;D1>;0o>ocEoWtB}?C#qj&?M#pohb|Gm&t zo{Qi3DWywD%iAeDlkzs?KneA5&ohkFc{*sa5Mt!DH63Y)ZmyJ_{>i$2D($4=wcl zWwGCxhwIi-cXBo{?7RX04J7n8fr$SdkdETkbS-RmxtEzuaL)Fm-si}9Sdln*Xpju% zJX5!JDssfECRbQA?Po`ec^|a5J*Vor)srf5EIGP5kZW~*yi;oNlk{YNEdfypox4K{ z>v&)0bA(Y+w3_(!EAS!y+BYuSMm9y?@vAxu%$`cqSxn;M;|Z#Tl%=kV3xP062jGVl$= zpKF#a>wrLB7dym<;s_a}c;hr#&hH$^Sf<{i|Kr{q1=IS6Y<4-Q;sp(_a-dn$iwhN^ zypuJW$6Cih5P9ZXW>J%rxZ2OqWiYG>DU1}k+?#f@Yq>wul(ZSl01)=iutWQHt zJW=FQ1nX<&%Lk&;1&wuzl^Z)R*vxLksi(k`z zHbt5FtD_N0G=^Tew&H(w=!B`!rSRR@+WJY#;N3fJ!@jfu3|A%z_AtRW^x`069wjG3 z+Q{%6Vd=5ZcI`+utjHE%CT+wPSUyej?CFyypGaAe8X3yB$rDm}oEOcce_>_}Xci2?V-uFEqQ2I>B#hyL~DYw)M>TbHnnB%Q-4^du#b}X$!T$L z)H2iB(bg10s!r@%#pNJ8qF*b+A7Wlzw4l448*H7O=_Tl-criZntt4U&PEH3{iS?dF zeEkdYx!}?t?(AJ*Gd`_k`l7Tx>6Yy=Zzr~qbqw0w_@v=A zQiFRCs3_vuMyR$vW!QNZiu2PD>Z`~_U_~8e&42=4}bJSqTK4Nmmk>S znGF>t%O&VC{6JN(Fd4UeR_ODWg$dggjY@b~)lS(FY*z%OqWG-PICTRr=c@Si+T%u- zj9AY?IJ2L~*29JKN#2kE1*HW~rq|?#XWgkhF2c{tzfek?&$i5Ba@%vD`GDf#^UT+Q zHwQZOpaoeQLfw=-uUn=lz;!t|5G*iEQWCxXN9&LMUo6?pu1hG#G@0X*T0V&+Ue4Y< zN=_O@ll@j$OEt0Nzxl<S%HEjKaz;(MMOl(9P#;mPk_C7 z*}pO1YLqA4H1#I82g;MR?S00JcX}T}ATn8<3K@R2li`;qdm6D!d=6%8ZAP7G(k{(d zq}1`9&f^E22>L*-5;=_WU7_Eqn&(%ElHAlwf2t;dJ+EyVBMVWG3d!M%Bd&!DoN(W> zL!rrLf1keDfSANt$k}0Sp=|WtTub;10nkXm5N1Re?({*J_;hC>T#A6uHAri7JjSfM z)&Nh+mlHX!x1gmcMMdR&%qu8I6>Z@JubkeF^>j|a@&|{G7aabm_V9a1YMpa`mh0k1 zgPO*6fG4jXG~BWymc04Q`5=nEY=Gw!9N`)c6^g?8ctl@KIdIzwmRxv`Dc2@54nOsC zu=}U1eYM}eS!@oFr0}X##)S%nBg%0&bZ9A##%rWd$`LxwX`a#W*}}RM{$UXL!<9043bFaA>p{N7!nL9;fwym&zVXA)>AlNa>#5z(u-&v z!Yz;)`&0-7*S^Nv-Tfy%5*Lx{mIsQYEW82~Z{w{0#pwuy0INP_PRvS=ic^R3S|?~q zhCBh?P@jC(O~vIi7~|)qQ8$$v1R=&6oB_t)L1sKIOzRmh4YhcC59vJMG{o2ilt6c_N*EbK^|eKhWf z=xEqNA`8W+eXKO3l#!X9PKt>5Dp_2{-KEisd();AC1(x=Wo(I)tZ$A$!$ zbU6e~Qz~Hp3{pQ2SO4}}dZIheRYuA#i<`6W2S-^^0qxT)LzSiR36;SUv*azjYFiv? z^c(g-NDy#r+ooBrEKPeo!oa}r(Q|Z%MCTx(J3xX)vhG$8avOzVv&PoFnj$Dq~ zE{Hts=ura|xVGbYboN(=skUm=-)U0iLVaqfX6s#Ou_>*hN<;f#5eelc#?=d_tY2Nwvbk!Z|T4=Xba&aE;1}sqm}R^U+J|We_b^YGS&7)h&Hj zmuuWB%Kzo70DN>uj(t0_oh;!Xm|1@&X1{A^s^ayC&1{mi{^`95%f4Bj6shBP$D8Ia zdmSBB?RV8Rg~d%}kL9<7&LvJYviwNo{jv$x|wJ?A&FuO4ESy z-ig|`Ndl+Z6p~wC!k5t$enQwMN9+L_SY~3k3%~xDo!l|Rjp3q(it>NTzCIn>t(P`8 zKXXi>d~aPg;-PkSNV+aQnnqoPv0b+i!7v`PQam?jZSj?LbciCm<#D~~$elR=l{_wu zXi<>WB`}@#CCVsLz{_|Wb*-Sy&1dXWig5pMUo0ifxo8k%gQW!#>;5Gg;JkDqgzEcf z%IQSq`&_HqVWDkmgjX|hap74X$iAFBd71vuB8W#3O-PU&1W7c$H;8mo9Q!&*U4=J_ z-?Lyph6^J_$#Nr9eLT)8pQhj%mY3y-7N5-0YRBMz34g|?QGtiTIo8aEZ#98&=$W*$ z&rES(DaWWCR?p>_k9}QMB+$VHy!9GpsmL1WY+-p={ z?buGW@R2j14Yv!9&xH7)`LCZ_?WqkghN#;1J@z2yQ30z(^wK*%Rr=gn_(Y0sqISPm zD14?RjFMG;RlCoktSvfvK=?@`>f`t!#Z3=qB2<$r`PpN)iKvM_o7u`rO5I;P$*~$2 zJH8`S3AsAk-Eu)V!PMFH0(;p=_F$>(7`aL<;`0v5NK00g1bMv=;?xCaaXXhY^*&?X zF}_iKVfY@zQ&dn_X!}If(z_@9(ve-0j}sKj>sMoaj6}Low2gGL`R$HMxIVTEpNB`W zKe}EF|3|^m=sVi>J{0ZWbo8XRRmM_l5a0Hr(q#&>q?72nYUU zk0)rIAP931WI;;CHijT-cXhS&$#q`)fCS6Hi|R^)i0_LhyH<}I>J z9x_d}7$byg&>+7Gaf$t!-<%rBsTB7CxfZZeLIg!5u7^OPXE{XKZEGt;vYKPGdHk0c z+c%F@e$WoOkmxtTN--L*0u*!2VF4~KzoZu@)DwA5H>dX6B4=ZrJ|yc!1%4!gdM}j* zO8Xj7kyCR-0H=x=V+RV%k63~Fk|4T}cLwVCv+=Lu$I*J&_NmcWR)zI~X#7^U#!s`B zGevRK!U9vDTw_M!aZqoAhEpC*oV7Wi`%ceHzzdWF82#Dj{}%7{s2|%Ja)yD1X6chh zUfOuG5B5yNX2L|$lV9DrFdi$j>DM7az5OU+O?zC937-o=5ICGSmYLl{zdP@Bk1_yErHk0mIb9 zt&A&x<^9DQMkWTl#%F{pOO5jt z2pS%ws1yeJWC-Q`Zbz!XQhMF)6ST#4tdlJVld%7s*Q?yly zA5LtqRRZqmobRWYuDIjpy>II74;VVVAq84{#X3dSZTmFL8Lvc>Gt1|4g`5P2V8!&)7B*>Q^py_zH_ z!-JOjW{Tw>@eZ76v3RhjvSAsqe}p+41Uv2^?wGXKdP;e79=&JmfVRR#BKZSCknZ5Q7uT>fmd36L7YJy0eIB{Um_D}(0yG9aYZQ*rh`c9 zG|&!8l%mC%)n+I7L{s_6SzkcxzJmZ-XNI>h1Y*>P^^I0iMCkqRS%Kf9Fd6WtXauzP z_ljzy1w!-*@AYYre4$E6WXsmV>AXxd+I3yh!-H+(^tPK(2-2OjZ}>CrJbEjn-CbxJ z7=!$6Z0?nez6MvGB-gvye8?eb-hL=4Cb6qz%M;9K%buaE)+At}OhwzuOh`!hF_n>6 zozQ|k<(TL^lE{QFa%J`4ez>MKD@cpAx@uFfYvN4sN-<_%lp-@mQ#*5i)C0N#xbo~gFMQ|T>qWVOa1){I?)JKrX!HL8ZRkzzKU~fL({bl(ke;B1uh4jn-W6a?SrEa+pr5^xdM$91$wA6G8SgYD zGhAHyO!qKjbKwz0P+iOd{;GSjhzO=G8}{3BWqQOSrO&+)`W%uHBEh$;r$J2JIOj`0 zl-qLqx?_Ka!r9oFHMN{z&pjh(91+ac*ZjyPvyT!(_4(S4PPMVCnkHtt7NA2ABtXgU zG;a^Y|9p3)pDl)3<~lp&-Qcwcw{P+~;=_)?L~6uc*)hCmipg=hsN8xy)EpvAl;oGI z3^YFoG9l~=x(NoHeO1S|*~9NAFAcpE`)bUEP^HagX&!8SHB|9MPc!x99m5S{p8I~K zjEG#R<*7TGL6A>+i6@*#3v_&bbi6vi1y-*~5rdRoomj|b$hs|~ zH&!u;a?R)onBo$$|Mc z1PoeK467ck^k-(`Vm!~t^Y zPGm+?#5b)TlZEuQ?f~nWI&{L?zI?nF0y%#9z^Fg#L9AF8-y!aGj1V7lbBLe3eYp8i z_3ft7&k~AbHvsje5@Y@Vt#jT0Y{7uy3n`6!cr5hx&6mS#w?kAd6uo7Q7clpQUb#71 z9!Y}XUtt~m2N5hzl7%m+Zw5j(qqkgThBQ}?CKMn_2%7GAVtIHBTPsTx0frW_z;TUu zkU|;f_+~S8q*{wkjl5HwOE@*kyqIp)l7>AX<`hooi6;}M${~wbxHF&M|8Q0ec*~K) zp*At#+E=p1FecCav+sV;1Jn^|6ZkGcyw;6@(D{mEqfF+!kw^nIuh1i!9JNPP&l@R^ zU)C73f+5Lpbc1lxZxfeTb@5kpVzqHMW1dJ~6MfKh&k2j*oSTWjsfB)E4-S+!m^?4W z4wY$|Cn-Z+?SSrVX2K zU#T!!P54;rvHHGX{bdcV_S~`XkRM3G4NZ~}FG=>JE%*H4F_yhQRgL!!9EgL`Tt7l0 zN?D=d`}Ofa@N8|bheW`~F_P;#Z+@Cl)#S(Js~hNR)$9eV5si+WA=5lDPEJt*Z$uFm!K`*3`@D?AXPh6L|yN-XFXIsE~$eUWpCI0Es&~R z&4r6UqZkHu=5n>r#deQue_P`#>yra>-}-`enzwf6KXP!3T@#QyUyN{T=B#NpOO;3g z!vzC*!QZDoX|y9cgF+TMAg%39aNj20aM-HkzHzY7=H73$kzIgrvph?{`^6SwDWAvR zfuwC>cpMl@j@n6JrF`vsc`o}#yUub1&)YXM1k!n%JyL@+n-~@MO|eycHAovD#@%0D zqi^UP9!X1^o%;U9JBNy{Sdut+!rgCiA&Q^ZayD@d+A4nZotC0F5P0Ki;3rJCyWV=( zn=x4J?5#XhWIkyCHcr1WVE_4MP~o7u_mi;zXF;qd*5W130mJ4uJ8Vo7JwE}e(}e>yutC_D;cMcM2DIrDsS$Fj4wB? z1E3V%1IA4IF6ax?{o~IK9EiAA!|UW1^s;XxMH*a0iN>17mQoYFw_1Wt5m56XS7ok` zA`T4`aTH=9?4Gfg^%-*Rx8BO&rPFNa#IxE%jSX2*kJxf6nKDKAABOJm*~Kapb9&gw zF9UnfWy7k22MDu$t{HJ2bfh5PwsMrBLCE>jc-*z?oKNQ9yNl4jWt!oM{bw1MzTy73 zl7|ZKY4E9T{@`$^SFUt%pb5>{X+1mGU_?O<=eAiIXOlWU*z9k}ynL_eYp}c1ibXs; zzhh2gZJfJs(!N!^3emeEfxr`>k@^M1BV+pU5E5NtfTW@O-p5;al2AN7i!NS%KAgN~ zcui4;ZP=E5Ir=%5c&9Ui<)0Voe@(GR%fI ztg9S{Uo+6!97J^PwDobX!0C7o2mGD|6{j0OQ+#jjll*uP-eme^y>%yEY>s?pBZ zzrLFqsxj~ugw z7zRF~YCoAgXOeJUJ9^@R<6}((^PFwCz!_Lg4R*$LkZA|ExV`f?1xCp#(Dry{;z^2A3v6w7`r-Kvjp3 z{W9_R>1Ip7*F@INa%Qz`P2ZPtYLAVF-DsCq5i@a_uA$TO6My=#Oa1L!JOe(ADraNd zH(Hh}rj#2z8VF8RGIbUamjWWv>3bIIeb0(az`o#&YK3j*1=KKSk?z%xtDkE(sehTV zTWio(f?$jMWp*}68f+aaKqO;$=+}zIiVc3g>#CtCM6pni5+fS6oTYDaDD8|&8D@=* ziLs~_kDd92;cuDF>x%62UMIU?EXE+uGDO>AXSgn)#imSWy`zsBS>UH5RZ0IAOq4h! zEbW(1pSccJiqZy6vAZ!Q;s`1eDA9z!Lw7HiE8&($)aJB%ypw**)osMPI|28#tkTvu3Qiq&jAa~7{dZS=DVs!&lx zZ_D`rn_n>)a?us6%Qd2VzidUf>xXs_-J-E}6I_0tG9evAVcpox4iALvAijuw7*_){ zxrTRA(%?}Af%haErDyOD22^%H4*4!D@x}c|Kp=Gfi>A?P6dm2SY>h}MauHcTwyf=$$PCD_%Dc2=D8|_w>DUpi@EaU6hXq3#En*4d*}_U7!qQbNzV~5 zasHtG)sj{;Ve03Ht+ErdD^<#rZ1ldizg3EP)u*bV^O&DHt6{HFP^?VhuW&jl&QLcI zH0GL_I{s)s%6B_@m^iUdaq$K*uN>9>T9f-qhkfif{dppC$IjTA$PoHe`cylhVA2{W zSIvN`C(d0c|9V#iWEufYF7o|F6)EDxBe5W}OeaqW*W?*m<9;#HSa_Y9I<8Tf-VW`v zKOXjzONeVu^*TnBx4AjN*>cV_oHB`_qX$|4pgbqGE1~I=i<3ltF4Wia`ijDlCgjQB-+v=>CNK`b}_u%2E%|HClcQkL5TrV z#O!<7uMHwRCy_{AGLQ9y1#&eKP`*<)+Uv{vPxbxh^0`<)-r=|YNViyzLQ^9ahkd)g z{&K7XDXa~TGgz26ZbPcjaxmfPVUX$#i-_~+d8|Q6P{@SXQ|p^VW1iPdLhrHUBfi%W zOwZ&`+oL1y8UveOe=L<64w;(XI?^m zeT}uCR&jWSB63~m#mf{9_Qc1x$FxRD`eO*l@t-tb%`>HE93@`CQA#R7Hj;w+LllrF+C z4PQV{>DhXJnBY^Dz}cO%@do01ykuHj*HaheD~rIBHme7oo}MWfhjKj2{1`LolI%q9 zhEdk^*-@YUrt1&i&JMbp_AEH`VYBuWE+qWXVw~(ZzuS2ex;DNm-OY;(Fa4Z8V8(nUDA%!yBUi~V2&1K})F&~@k4rLYQi zt#=B}<7sJWy%-E;thaqci;D`=)90zXS4`%~M=_oeZmQjiq`cA2%=u2S9sUNw0-L0C z5^eX$D1JBoS;#*dp4`*rBD|$g9^mRv$haa|Sz(;`8EWIZRK`f-BdUd<|42#IFNa?gf|<;jyyb+S~wvthYAxN`kW`O(nMh>4G{?!DyNRd zyyptPV$+!iJdSI{ycAC_tG|%jZ)|M5)P4Tq1-x9u?OH`^Pr{*5-$kGe;naNiQ9xnz zrSVZ+QOPR_+}=hfbLMvXqCL6U6DFl9OTjJX%)OagoInw;CAf8rvm^J~sBP^7V@F-v z@kBD?JR42&wZ;sDprY~>wkBRwk&zw02+HI3?Fb;t>Ks^<}GD zr|`w9C`B#^zTbT@B>4uy$qDolT>epXy)%Kn?!`R+>D=m+IskI~T`n{C_VzwI+1S`5 zTE}|gruxLh#PH2d-%^k|mC{3S>Fw12b|c~iwTM#Tj|9Zt zQXAw7qNTRMsU?5%2w?ejN#(%KUpzTr^FaQt-IsR7TLLlXL#Mw*2!Jj!G6RXt!v{6O? zg7WXzOj4!`D9!&%yK^bcbydcH!PB7sqhS`ZTtoD~O^s1vf9tXMz@WGws%GV1mk^5M z;wT?uYxWm{zcT=C@@V=IbNz?%$e&=Ee@6-;hH3u?z#sU`kmo*({x2L;n)X)Ak%|EXzpB7m%P%0F)+2Z59xo!YC;(KZ!#2D4Xy15=qj#AoXk>Z-QxZy3iKCmXXlTpK4dMUkaF);eHA6aEId8}Ro#?A>QE6~qNGqTEkobnpQFsFuex&$%NKMVa0cmpFJ_F8ECdXBw zJ-Fnhj|_W%S2#;0)Sq%4w@f$wq-eBjBNn_KBJiN{D9ypkLHgfxL@#lIK+Rx@xtJA&If}rCe|cRQ;4Ywk zPbdWFfMQPPhzx+G{>}0T46i?Xrg=MNLB?PDMIb|Nxd&dQej2-s%y5pdH%<_HjI&O# zoaD%4JUYi#m~=i9J4bzk;yEq&n`IG$Z&Drrbu;*PkI1S}$i~6x*eozm zWaIAkHr3KAFPF|S97TIkM#N`;)Bigs^jiqP=`*8$6aI5h?&lY$3g zM~MM*mna##vE2$^nB3cQ+xG^Bg_X+er@XY3{Gdeuq^I<2f;JDvQpvl$-+1%;nmTTZm3Z)B8|VgYN;Rw_*_tru7!rQ0Y& z;baYlEg)d2&Xe%gqgDu|f(U^U0eLjPRV<)fcu*%LEA?xyZ~U|n(|P!iQaH1}-BUiM z%26tf`QW7vVO;GM_L>AZRmtm!`#o1hK{2rKuh z+W7idT{W;>PtWdueER)hx6B2zv-1dn$BsIwvE?=V^q3msduLbY3R`R-IDdI*@_RtX zBXc?yGAul?RTmDq1AL(4mKm>l?Orw4W0&hH`)$nBD%*GJX!P}|?*1f=a9MtsHY#4qp5fD9|GmZ2UPtT&v#w0wl>!<{ zbbHff-LAV;JTC}q6vjC?e!bk2il43;Ks^RH2N$LciKzdXsowVy{4HXXYq{e@DZ-MP zF(+LovWd6FFe6yNi)Hc8{AD4UC6Wn70n_$NY18+ zioS8XOWyCad`{M+`qfXznYszV7d6A$X;MPpGO4-LQvm8~WI9b%zN_jb^IAxf8!xqr zN|koWNEJ25ZJ%S|8uZy1Er?Uy+gs`X{6tK&=`9gQAA8(_E-!#Sk^26us11j4n34t0 zo3I#&>PVMa_eNr-fpmqFv?h_G_oo6ZD08&mTr)uilMmMQd`~))T2A6t!ei}aFDY;o z7BBLu^(_rGLfUSS8}s0G0T5}p?Q^)9-S3pr6}}Q$>Kl#)#AW2i^NOy`*~ZK*A;{-1 zv2+4YQR*ow1>Jxu+%_P}uV z;&Wr++y3e!9>fzhudL}egc#ACe*BSNHWEoi zxTx$**?>#m$qUh#MdR=IFvj+mH6EuXnxwB?;~MB!Z$|Ba&nm6w_r{7$$^KCkq?9Q0 zf)@3O9VSKG^spWr$TgI!S+v|(igRtP%ZvQNvIHb}+1@#i1?Hz@(pKwS3;4-u6`nI3 zM7S#(g)t=KV_urB=%9#3Ytx9YJ_|iOmEetZXL-D#?ss?azi(PEays_F5U^@jO zIh|A-JVHi+bYuwOwcV|54M_<;%RmCj#MAUJ3UH*}@aMh`PL_(p^T!jx1Z1z?=mPL$ z-s1b;EOi59dT!D7DHPrwIiX+@5A`v`M})Lcv-53TAW8a_xnSH`zk`AJDGBqbE!F*I z;{K*VBB;_!xsLSUP#f>g_?vbcAr4tWc@L_U&+v8jZXh2)4t5qaP7}(<)FD4hKG5sz zV?r)T(43RnoY%|A;^4 ze#TZQPri1kW9x_aNN}5n7xD&A#JNWzDF(DM<=IriY{hZI1SxdCPS9z?uRR9L^Tt49 zjl_`z6-VouKa~xa3WjJCgQz~rjm5CYK0bDBk7gR$L?FpI0wHP-Y}b|BsQMn-82J&2 zU#;xbwZ*Egzky9EI`OQO9Gh!xjr#%Fa6B>{lk-2La`yy{U>H;h#knBiriy-C9hxG6 zNI6FS2s{X(MBveavJ&Y`TEx1Pxi|bi`94QRZwGr^8NTD=L_V`4v~@qife!Kv#4bDe zkej*IEfK&XcoGg$l==G93Y$IH{2SBQP)!DjwuBf^yZ&t0Ihta?_gI+PuaqUhkB)As zljvh{gyt9XQ@g-ufan=s%s3rlIpT5lk?#kzF$z`@zIr@}7TJ?7+&!w1$h`tsp`!pZ zD*J&K77`_NzTD`05+6{%;&Z%h{-)-`9U|C0pPy-?z0nQDyeK=a0)SY57KRiL)gzm}2p%+73DHa$mmWc%82-Wma5Wu7y$Em_qK)5nT%}6jh4#h78dZ8_AV3vi{#H8i9nraqM_9jbXyg7Uh=!R$k&UM@|gbp|}7R1gUub z;1_`DQ9F$TvaS}C1#a%fFx$1eT;Y&V!Buit(d$@50% zyE}#NUHx|e;rv_fRS&OM;228XKN(J1Nm(I&GKgux!nz4z-&|Dvq2c-l^pl;Ac@YAS zlSNDk>!4=QNF+GN>?DvnLtwUb(C0FG&612n=Aev`%%D%vZZGdq^jiW;7g_oUpLGD} z(6Db}6#{8UG*Mf|RrX*HQw^t(wW^lVBxOX|-F7cT0(rCmrSn9Q;L$GEbHgt|UeC+0 zqQcPu;j(Fut@z(07MATH!M;+8;gZGgDvnn`+&fMN_1Nd4oX$TWt_DF6S@T#xHBERg znT}c#U|v%oT248N-xc^^oIXn${+Q6hh1Dp|$7F$tP+xD%e5IHq0oW=vbu>u(Wu)Uu zTw@>fv0=D9wYqo3d~PJ-eX#2Ls;RMkMP5tUdLHG(EujQTs*pa-3oV9wo8 zKmd(s-CxLm*EYUS022k_>3i2wHqHt$qe!x^Rpj;yzKXYRiu%> zt_(X|dt1{gv6lFq<~n0w#7>T0#)*a|ngSiCN67xpZ7dI=LedOjt-AY@3sekb5N7u%JrjG^bL-m{CfY{k<3E~=0s&~ zHlsn`^jqzEx(l3!^uie((-0it+Vy#G7r)9*e-3SmgD@-fZrf7;1qgQvZb;bZn@`4))7s}!EG6f_@{9-nA z`-&G~`D1HhrcKEKcS;xEZ8kA=(Any(S@IMKB3Ea<@r@eh;aEndo8*J1HWsJHenv)) zaw%j7u9;)>nS+3p6p*NNd91nvdo;MKUQz^t;Z7P}Lm4v9Otf`Y+$c|o9*7v%2aL^u zZy~dL^XX(nyTN>f)p)6Rl^b=-p(Cy%kDmO>1Tva8Ke^f0pUZx)aR)MZZymR6z6$fw z@usHoo-!ypG{?xj_Cxh}tbowU4A~3lrjXu)7rEg3-@aqKw61mm_zHzQ_|^e~m$ea; zXkrTD(r;{k#7Vc0Au>?Tw<1AUHuh=e>)8xSweYuGp6~VVqoBKPccwg?7wO@$QwK;{ ztRMI$1^XjfHUHfo2RYfR5kIbBpjz%lbATP)2t*;z&j0{Hg9BEM5!e$%0X;>1o2N=_W5`>R_MHosHm0Yk}JaJ8&`R4P4l$i)tma7IwKI z;Oy|)7rx=5jR7CwBU01=YkC^3Zs3i$bhT44vns()`F8j6(sP6-=ha@8S}@}!ycuMT zy6_UwCv=~|x~<<*Ff4Jo@Lo5jndO|N>5rLbUPwKdl0*zCNnfB0$DSTf>8?uS!5kAR z$DDUvA3vvhe0*isje`lj^M_xcHe3f!f*GO>Z)`wIOPGhz z;7Zl0vXM&+uvaO^%Jj?h`{BbPwxYM(p8cY@(kjg#tpGaw0YYv^WGnWXiqX7j=>u>2jy%YQL58DOOoE2J zoZqP=arf`11wc}5x~aq^jbhkMMYp4y zMV3`BPz`=(?KPY0ntie(vJ>|zt6c!)d;iQ~!u!}@P|k61#Ai*5)Mqo?OzyErG{Y1m$Hn5M@Q%6hsH4zjX9EdOqh$4_OeEUgS+|o5ZZC z7G0c}c?|wE$Uic-leLv?bBGb`8~mL6+$M{I4Qs6cpZbhJRZB?!j6M5BKkqf%X|E6whxkp zU#tg7795%pO0#EI2oYO9fVS68mCef2dlqx_6|hE_i&o*Ia4uF`D=;q@I?yk30oH1$ zhb4TpTN^IQRYa8eGw-JFV}eertQpEDL;Wy#h+MdABYx?A#UG9Ls zSU}uAeh;`jv7yzULuN9swaYSmDs$E%7!}Ck!xm9P+slmd4ql~KyHmWm3_(*+4K&t2 zzDDU303c>pomP%Ou)o|wfY4K)5^KAKWU*W=GHpCnpxa>BC|R+21j9wW-HYBsDM5MP znhuBZNKt)?Xu~@%mKU+>1F;F-8>2_YZ)$i&ic3|RLstNo zx&pH6qOW~?Gq(QVlaAXtF0N){+1e8FJ_QCirbq@!IKt}}QpQcna(Bias#q3235IDC zesp6wc-%N6CE>)iba4hilka^LoSi`_rX}$z^){b9RX_9sjGW$>bQf&(rh?14&YH2x zL%pMjS)KYZB3M#>*JJ>+{o1&Otp=VSkOU`8dPfKeKiTdh{nh6m2+7e~rLEWp4!@(7 zs=z-a$WSnv8rAqle_G)FqxtI$yX^O=3>nuPV(i$vz9EKoFLxug^}^QxFIvOh& z0FB1fe4dk`R1&LVazo_r1XN{+Y*|Smau*;RP89^FIP$1ifC*UZV9!9Xr;e%%fzSB; z!u@NCw+QsV(e)#x&@p~_w7PDQ_v8uiIU{?5%byV*tS1*{RHQ_>l1HsLrUp9shwwNyVO^j2mX%eCZm? zJ311DHd&U@N5cC?|J`mnwIY)&+x%}|e`u)9ek%v_@@pmM|3%k(KsB{(-@~Ez08*rd zF4C*iAib%80!mS;G?5lS2uKH|6QoF!CZH540t!KTkxppR1BBi~?*iZP-uv$Fy)ph@ z1|uOEIs4@7wf8=2&o$>-PYg5cGgfCqpr1^`(R$7|Wp_NSDPYT zBvg5xeRkGK&el#OK;{Y~t9~~wZ)aHKKf9^UokLNP@_8a|;2Q$+nZ;mAexj%thy1sl>K5>Wz~8(`o+itp!>d~*OJ&+a+)#7EWUv@A$$UA zOm_KVZ8U=&{HXf#hg@8&*PZn8$C~*jMn%z5`KtL`oIlj`)as23=c`vUWeUyjD~6>5 zeRj08#T;umjE)0Kln!D~Fb#FM$SPgt4IEppov*^$KUC9fD{0YtIxr!pszR-r)JVN* zV|&c+28-4FpqCW#(f+SyH)F_tp6A@$p(P|h`OPu+2J2*5bdUrMzNv?&m=3Ta7hlsh zeJPOpkja1O8f9uj7_$=$Z%PyXILI0dZNAy8QywIp5(I#@DsuO0Pf zanU4i|5_<9x`xB4_r;2MF8Uez_YHBRmNu)XKSjvnVB38u18vc$O{2B06=ARrg1^r!g`J3a8OPm%Wzs`s*mlsT|YL zWSVJDW`xGUxJ4iZ+X1kGjQX1bL^)zKPLj43k*mYSyMR)<3R?!3S|F9hiLBh~!xx^4 zyeoA1iDR!DS7eH1hD{F)>lg^&uPRsNM-Y>tl1491e0OKfaL^_nDLh*sCr(Paa{YD5 zn}B2~N7Ayr`5Bvj_B~(8U8zUBV-m!3YoWh5547L9mwsMz?^DJ>M{Tz#LFhW~$Vi@M z1v&JUtt+b&e*C`eSNG=CF%<8EbuG#Oqbrggk7E#$^@HG-4ThO?8wV+&&oE#Y3bp$x zX=7*j^kE=}E8L-0YC0MqXzTuQHB~->;G{voli((uR{;|;F{)ysoD{p|KIfL@mW6}G zM^?qAf5nH6enw?0S`b02iQsCehfOtQ;r1*n2^bpd-%Lnr-07vWNIQc>Eh&dgw7s25 z;8UPd)E~r?Pn=9aC;HM?)2P>KnzZphQj3Q+*k-eT0F>-S{Ru ztsIQ%A+~CPiC&KnbGulbP&fa6NN%M)lZ$s<%#Zk|5*fA?@eat9Ln+dv17A<$c?xM(Wn zdOK@JHGxT1bbPLW`aw~#_9Q_iUo*sO8%H!!$eFZ}k07+t`#G8?lmIeRMaE z?`zlhRE`sFXMqbfs}Caj4oA(I+dCh+lO-9VW~77$M#7h+oa1L5bLbUwZHtJBbK(Yj zq;6nP=@_tP8sT^QT|`*l1Twl+j}3!5-bM8cqRughSDIwYjI^FVHCw~AnF_)@UZI13 z1DvhQpHV!-ZPx?ZefS16dHwB5gOWo^Tp|%)bG*sgEoty3P21JT+ikc4JI{_=%5!A= z(>4Hb(ca(Ug{l4PmA+RG=d#P>WM#X5KAYNNq#!&o6Hv-Jk8&k}g?9)t6Fs1c4beh` zKpj8R%sn7R2j{3mFrYba8#j4;?CNY)Hi?A0wKIf(duW{ zeclt~-W2>A4nj(NC8ryShJaF6si??K7Nn|K0=njFl{V(@XfQ4ux2zK6CTD8vlVkq?qq`vPP)Js^x8u;c%)iNo z2WLnTv$Hl&_PN~(Fs@B0=47-$m@1~IZND0I=0a3h)z)8YD%B<1^^GN zNf-ZT5=c^0DX@_f*3)DsG_Lg>Y9S4VD$&R03iDmA1i35b%+3J6J)eBE1+syRCdIxZ z3^J$xjMl*o&<=)bv)X=)UtF{ik_&lYZ{_SQL|(}HRa>&dn58KAA=d6u}wK3BS}U)&$|A%v$6QV4*tj}$(% zK%g3h5tX<9iT4LXeLb!iF<}+=N!%@G`#Wv5?N%9fh|xeW1ZMAxK?HL;5|n%r#nK>Y zPQDJNcQngB2!blzx$D!%Vvmaz?PLLzeo{|cAWF9^(}oW~ZHFOH;}VB}z_I$Lo&cu| zB!|+W7cNd!BoBff9P|q=_-@J(E}y>QZ2ZX)NHyTFEMFBI<*D*u$F%un$_F9{7(3yy zjGz|gL3$*o*(75i!BEF$DBj1+Jq(!Jrom+DVnBJq6#koUzO=IfHQlfgyr z(-#pYU5swb2!%lNdIl9KvD!*vTXvQOgg(8PSYM--%wGgXfzA0FZ$QBbn@&UZ;|nMt zTJj*d#C`A5hViSOdzO?qdygLl8t28LF)E$f=Y6kY$y@F{K%?_55r~L!z=*mg5F=w_ z>{@@qV*qSLf_B5{Efk~|PsVSa482LdRS~ot!_De(?3H!3K!6XQ!ZuuqG$y_ac@yy^_to z>l|=7L^GrXir1VOzl%_Y-lPKvvEIKH z@W{T-ziNApJI29F2H+Mk!msspUt@yJH(9>C1tM6&6xhLFV?e{;L5BnHI{DrLkyB#^ z!8hjmfjE~}?DO0DqohMWqUs*zaZT}Vyx(dlGpnNq!Ozh={?}J`;UY>&ILiU5N7!== zs!*I8X78#=vQ3g~#vpIU9?uJqze(6{G-wLDwCD4U9|eh%Z@3SLIXqbeqKDm*sBeOJ zXG?xx%XkJ}V!&Umv1#2k={x&dlHZCFfR=y^80_B`Yk#0rxBdasZy*={gKefFW)=b?J0_Tjgqt zhg8Annt^EY-?!W__@L&`bg<>o0#Q?7BL7v@J*zmtj{&=EY@Y(^s|Ow~`8t%y#cbz( zoyqzdqUltkLt8LXiV)j3AWgI0eb<Hx>jMOn;S zP{{89>=Eh;TSJbdj!e`L&J7M@#)YEV)h}sr;8bKAwa9lBRU;&vH=Xg(9}Q?g?^-FI zSIARR(Q{M|gnmyk1f(yYlWtPO!l|U+KCRJGpOMZ%yB5xx0gA{cVffgLNGWdojdtLQ z&3)j;l4R0;&{7cCmajJKnBz3|+mCp0D#6+gimQo=VX|(^PlZ6xN-QCxh9tr8#7;Zp zK|*t;11$W!8v$TerG!WS0n;(MFFgx7N9h*nI%+S*yq5SMU=XIZ&H$ZTVOS)iim^NT z)$XbrN8@cYItfI*VsHFl@5GYz9!A$^@%AG%(Ng`cKbga#gP^ReEm2f`iW8J2Q&omD0)SP7a}=A z!O(n<3RI5ubS{8Z(h9|qKN`9ZwhfU&DjR3kS2y#mRc`&3q1-;3hMLx+QMYb`}$i%%) z`r&-+^EUt=40!TEZ8A0+Oy_UCVu=Fqv48FxRYI^%RZc0_N^WFm{U{h0+e?65dVAsN zI#byB^2XvK4=h|V-N~DDI$%pv!PbSJ8 z$JI0G&9ZE-!DkZ}f$TUC1 zg=+*u?a7SqnEquDiXCr&wSZz9s^$MoN9KitqohWE*MT9bNj>xi)!zMeHM}i>-dXrQ zt#Bq6o1t0P9}U298X(({nMqIpB*b7{DdFgPWwD&w&I!%Z4M0B~3XN2dM4@zs5{BmI z=UsxL=rf65ZD~tvkP&)V!H8(SD7$5IrtSeT!d!CR6Xqf~003zueGFtp^C9(cU*JSp z-H4FM&x1?fB!VyGl8@!Q9|j550-c_dYRda8Hxh_^V6E9xGdek6D5VCMe!}s*@17-2 z2V$yBe#;GvY&t;v^^mpiW5(jk8DH34aJi&N9%!6r*5vA~zf4mN!lw3?jEorI zF`j;KW412m&Z-(zwTvG?^*%{%hSc_)$2CaU^*@hn-F2lQ<51+t1QHlT-=U_)7ki5@ z6jPYlk+1XNDK5?>oyzMENwp`zzLqX_voo2KTQ!UajWDXJsyQqQ+lTG*S;+tcfvsiH zS^j=8;2zb7^Zr|1-3JwJLij8E{HGDY6});n&lUA=#)d!-clhdCG8iw+%+1ZiBXSZx zyG8BE!HDDahOM)*vRv*wp|}1`>xDBrKe_aKLZg_s`jc%T<&r*Nxt)n82iFoQx!@GcNwhYR5z&NvVh+G9(n{|hKYz1AP~pL4!X{!Ajc$S zu?g7(aMNtWcu6bgL%!g^Pn&s@{ZDu2m$U@u8vPBL^;|JvwzlFc9?Ft{-=`dbp_HTV z+mCvPMGQP)u$;2kE2B{8wqEAxF3X0s;foWI2Id?=02)jgo z?$W8{LqdUT2bMJ=K&6E9;!%!PU-ws`7}+>w&|?1w>06>4cfrvtaavJnwd<(`h$27Q`RS)5NHx5NS)~`1DVQ56ZI2$we!z5&yrup zNc{$&w4u%05ZYA%?Z zs`qOu6v8NVvP3+uWmf5?{E04IAV}4Vm(GEX-#+?MVs|rUCtL*^n2O3enjwZyh(Wgd zK59+nYMFKyT_j8t*Sy39gr?62Ik7)^O}T>DOkC@;9oboq?MEMo`KQ>iV&X z40eKmki!G$k8 z#gXSm(i}WdQf}tcGmA?$*Oc0kwIp^-s@v--;m39-30mRBUSZ9!kg zVvJ4c+-_?Zy24iHh(d0r%mbsA3?OqUL6yJr+uzRI(8quwpX7vxo+0O6=CaC(_qwebO*4CqZV5~I+UYm~+t@q_GzAd~3puXFH4tBsj+kPNxW}`R~ARqda z0GrT4-~P^|skmyD>}O}2)O&=)8U>ldp0u)G=wSW6 z&ZI~@?~M2JK4rnVsgEm4rh;pauHTz|)9Ek;3=aGe!qYi-MG-cBG<+4fc59OxTtiF8 z(iZmD<0;dCl>=envdu7!0zD7l&e8NmR%^u`H-OmpZ2wk3*ayDo)KwedZj5f=bMu&5 zohfpU21h|Dg8{b2_6S@fdDq`y!H^7&sXNiY+?oyVlRj;H5^>H+N52~UFv#N^kOn(R zAr~K;M7KiUoQfl>IP3AgjYQfc;cNj=kj$9dQlfD2Y;eh2t|zQ;M@KT!^f)QQ5>;)y zAa1aWgN*7&RUCP>*E~L@R8{ZtWyCZgqv7G|ghLb98?&kB&?>j)A%6kb+ z@(KgQOA_oqCnp)%>fNq^ttJbj8tumzTvHkFNk{}wzRf&K7()Fp)Vu|fL#j_t5zNn$ zLTj~jknc}*&*rNFsB(}LqdR)=)ANtQlUHepCG*fvxg?Jsc6)>7{^jXuYN0}|f;xW< zFsTbJg%=>=zfdHZQ8@dLnvS+z&!lW%no5v+-=ZFqd%x(th2#!h33}|3gyY?9s4&EO zzv&D>IyR~ISgAld?_~^+p})OM4cyBP?WKG7!Tp4vfpjsSUO%TYPsay6ettm zHM3KamouV3bRA1bTOaqKzgFtrrzB{S=rw4J^jTBC-R4cS9@)OQRsraMT=w@K$0L_M z7}@}hLD-s%nc>2hdP@Smi9k3%1~91mtyUv%U){l{U_s~NodLUxB#WMi;&}C&qy4=(arqbwxT(+z@}76f8Fo{4+FM#&Vg|5qJ62_>@LUv4E24IB55_pC+h9Mo?L-6u)VA6 z&N+||f>r`)ZIbLc-VEWXxmNMModYT8sr(#8ci{ zNMDmhPu0!>Pq$fTePbi4@V3WB9WI(W{vDHZVBj?qkZzVeqe_i1?Q)AojGz?A&m-jq zgNdG)<3ZIPr#yv)|8P|h#?$KJFrx3NL=q+q+sN&MJ$Zr{oYD3u z2etb02N%-eHn;Li6^xwomuNX_j_S{UVZh@grz3~HLHCoFyVZfaGw}Do?_|c8z122dL#8bW=YR>y;fPu3oB-;`@5&A99E@&fMnIbVTjEz*Z36bejx1hPtuYZ-LSgs%AJ zUH}HS>x4pyiR#@05TC|cNDMcX5Tln=Q#vSgW+sF>SUH5Pe?0skfPh*u=T;9Pq7xYQ z>^BQfl6_UqIkM)#Es)aQ-d?SF17~1s6s2`zGXyF(1LiAuI-iCd|BeB(6J1ga-gi9w za5UwAeJPB$P0*bAbNU@1@qJ=I@lECY7T}NEuCw=H;)O(pRVyg_N!wi{@(X>#@eZ#a zvG%`tr003Ly1ELejmdVXB0C2Wf3E!e57Axc7V)B}Uv|i`xpGMCb7}(;{t)V1odhys zgjEzy*rvKV&V+sD&SVFCJOLZ*P;_!!*2}ueSmwLt167IPyD1E$aLR> zrtf_Uxv_YH#R|VROmeTyIOvT+V!*sk1~B`Q<-0B)A7T6#0 zoNVpE&&86e&CV0Y-O4ej*9{8e7%;Kq&_ie_o2#p9Fk_)ojyxw4uj95=cHg$R7m1Wx zV?;ZH?o5+E@Ssk$_JyXTFL=eL?x&ehNz`TUa2BZ(HULW=&U{u$gvay2$sBH<| z+R~+zXaJk~@9vacMD$}&Bzw{4qc7>~s!Vc%+UByevx7O;IP}Q11C&wj5;Zk7A4gxl z9UnJ}8s0Km5ocK3lc}Y=uFTLp?2|zU-eDq8;d(>j;^Hzvc+nC=iLZaPRBP}qZ{l7> zhRF-wjq#V?cTZ2d1es}XCO%)9v~&=Q@@p^`5hWw=A9- zan?@aX-iFGdjO0_NyTE{ZzlnZsX)0EjF22kO7P78{=3Ts*^~uU@Z~$)YnK>4v$3eA zNCuGb?x!Ef2HzTznI-YhpGj6&@w9YWerqy8Xq`Psc``5GSJ8DgZ+a2zZ(}iOAL4fL z#)5cd_Y;%k`mnv__ln&~f!4$MHif3+P@%L{A_L!nZCnm8fH zY7GyCLO=AcY&@L$`!oRniHt7Xik&p)?AqehBC2qgjJ!Hj*^2bZI$b@`*I1dOkxVB*?xEjDn4E#ziP78PE+wNv5{^eJ1Blm^VB{4#N3UqMVKMJ82 z{+iC_f-`F7fi~lXhYyB?)9eyDe@juc%LkSwT)<4pjzaOWiUeXG00FBT;4-9|r z|EcJ3HZuPCG6v{tDG;6*dHc8P9{0Ux+;6p%p0POnbr8nU0s7xf?*4ZZc$KUz|8+K1 ztjqtLZ2>{@?@uAj-}d`o9iT5DU?pi5C0AG8Zpt_PA%2WcVxrQr#)`%*R_yvGV)7Tj z!d;{spEgJQhOA%qKLSqV;~23iO3VKgd;x(9>3w(XO9}pSp|nvRx))skd8hP!E#N=@ zgHD5uO#XfYC?z}9fF34q-`%MOKT^Ia-mbK&`2SN62q`}IZ^0nY&3`ZJkHqYshxhw` zPfs9}xO7PjeBNB?_~#THRKO|TGP@|GVSyNnizdCq41Q+ej^iWGm9rJ%;S z%Mtl!yPBckFBbs-nwn-fb4`H{QCW9CY|S=6C##*G4Udnf0D2SI^_`uEU{?9mdVuZu zblS8j;BGB!zSY9-{IJN-2gvNVXL07ci)qPr1cE!)n^ORKaqVDkQSTjZTI+rbAWpb+ zZl4})8aWN|FDBl8csdQ;2BNcCBln?q?$~@fS_A;*(nG)>wKpveHdEOi^P6q-pWbMz zahX||g9IvUuK-&}p+>(m{T0Uisw`@$Z-xw8}+20DOC|Uzgx$ zPx2Z)Ia0T<%&>>(HkCVmur#Buv}iFknEnCe+qM&k17b&HFBJhffMe{?vuY=Y*hxU; zqT{ta^*%$!EyUCPaM2)qE;_d+{IB{8f1c2XF$hGm?(f6(5u2GGo*Z8%X-oGeZKl6k zseX(FWKfB<4w2S70QqdUyeBwxVktPb2_Y+T8z^2AW>BCzS`3~uw=1H%s zSem9gGW@r7Gfu++;wQZvue9@2dxQ19xOE4)6lLgsn+DKkRi@{6K&fTp?JZM-ND(wy z(5?f@w`5e*HpTKj2Xd>TD*MwVqx*+QxAYrkoalv~s>wEgYtom0eHdZyEe7a*wsvSe zfY_5%n!L|`s{39bfFMcwv(vKB#>9qIL!dwoFm5xZj`CrTd4N3V)4_0zor$KpEpm4} z(_tQ{isN(Ex-uataM!sOg?e%xa`M`0u_Ly-xf#?nW7GL9O@tB83}I(rKK0nYSO0P% z%79@g+|h6Tger27ZU3Cl{2oR6qo zwQ&B8(UR~HRY|Zs-kx54Rd#fItRaLpsdcwK_i@TyIeLgMh)yA5{=FS=>w)Zx#C(wu z5lJ(YLpz(#It)Zr#QWfz6-!%f2@v(bt$S=IZDg{ggQ@gUrR|icZ@Fg!y#;8~3{zbI z1>Q$o#oWaSTsjtY=TC|-OweVFxJ9%0N_;i38eBM>#7pFU^eK^V-A1KxDGg`R8ER&!o^Fj)g$_ec4r@^iMY+@QcdoXIQ)8KTL$gCQ6gQi_NC*BS*-M z6Z0=fH;2N=;OH%MEsd4y<+R07D`ZSd1C~v+>6@E&54l)YCr1L0C>%kh_M0+Yu9$#1 zcGQQaAN;%KG_9Wjn10vz0E;9hM5pc0E@hMT4TTI%9=fYVm=HnDUQ!FWe137fy?#OE z(9Ve#vvB{x&f*)>3Qo2$kmyG@g$_%ll0A2?x%`ng_?nYpq5@ ze8S!4q=lYq=(n* zeSfLV@K;Na^CH)$66N=i2HR8B4y_QxyZIes3=%r#tV53#&HB7b-@!g+L+t4kj~kds zNdgy)AYO+e{-{SiP0Xh7RX8_h)PG>93F@#`e$1{wZyK#vR-wMBJ0@jXR}G(tm1#Bw zCo6aP>K#h`T{;U`AlhADi;ILo3;PKvEj-j+*J$6U>$+l=BV~tid9ks8WH*%E)#Hoq zc4VwOz8vBHETH9Q9*szX2?LtJZxAA`p}SkYZpED5&1qqH9amG796_!VuJ+$D07)Lz zd`Ey36<lYZ%f$;`du|@=pE^6ex=Xbmyoy>~mhh}dP(&RBRU9nB zXt(M2<;8BpHo-akA}aJKj}7ikEm6*BZ!)zrp^P11ibzx2^^n4;bYX#ouyDWH+~YwD zS<>Ja?x-%={HnLo#}!XiRaIpk@3%`w{K$y+)Pl;$bXA$QX3LSn{UA?eqMe8Yob%RO zGe7GcWrX&dc5=AKu;{{9Z*Hpvh^^lgG`6(REbPi?GhSh;<3<;GIWu4SDpSPOTRVN) zaqEj-XO5-En*MmjC&I+(kuUW89jpK|6C=5uF6mgAyDNHZG|tnyO?ve_Z-oqA$d*kU zs1|BUcght^&4njUY;i{_A5>r+y>X8v7!iFW=x<-Js?~s z7XGd^0@RamHN~K*IkZQGw$G`@Y0b2}V3di%g}zcnh9>5$37{$w#2X6Vu1=ZaSC^nI zZEvA6T)i5E%~mhTlp-D{tlBJ}@|F8hkdse16rCIzg$dj99_P+Yf=#6|oWsoxe?im) zjwR2PegxJ@J>YYd-X3a3Cehf$O!B3 z0%T=mQ2rG4;g`=CTE)zuMh00-i?yT-kUl)6S-CTJRUkP_{?T>(8Iq>4n!-4c=w#kyGP%0N)R;%e`hMDX z+3m2ynWy2>7RW`m;5U-?L+3=k;W3#pl!tpRB=9a~XFl&3%?#^kFAWWiVaHstg)_ii zh%=f8SGRxeQ~&6B^J^LL4J?d1`JNH29`SsAZOV!89C6yxTvk(0Qd9AFiUWMy(*-Z8 zD|6aQz05y#(5O^?Ki_e3&+iHrm6O&`A+FGP9!kY7ga3R0ECV94>-ViLEx~#;whzO0-k`fz8DgG zZ{giY94O7;F7*91xy_^9DhKFEd#!;RcV`_NF5lT=y$?%0zQ{Y~F6Su6`9}i1JByuh zXb~^%gr7JCH@3S5dJ&3Oka6|KDP19F`xHovve)w`#vcq8*og@tPPfh)SRm)y6R^&^aT zFBUWGO_lUXc2TQ5*j<)*v-|B3+}C9T5@DL?WpSY5&5GPkMtPMTBIIj;2L74d8+QBV zF54HqGhEn`&@bO^Md(*t4UUhTvsBP6uXXy(A+G=c+@@ddVO>Trj(_X7{%hwW)}$sD`%DA6J}a^w>p;~p9sw(nuvwLX%C^}~hvCnBVsGWF>=^(Z z`Yo<;&l|Tks-0)9%m=KnbWKplH`@`X8dm1&7}+v0cy7rmqe%|0#4t&~!}1Y75k2Yg zHp0~gyCe2|V-3BV(5&wt0sacM?;>;0JHnK44)5x7KBaMIs13$bxC0^uQq$;J-MO6= zZElb71+$#5M0<9(Lsm3x9v5wRgRF+Vj(G9DIKNu*w1|MR==HLA&V}nLg=Y`LJE!S9 z7r>VZI}ek)RF~t)wsRroq22jC7cj_0s29SCY_@-{&ic4GFAgfKLNaSD7I$;#UE$r-EIeT&eNDY6Zb2po1l=I3g%tyXPBpzj(k&HR|2^Q zc>9VPQZgYPmzjdZ;HlCnwc6s}#OyyWiTA(!zEL0(rr8cAR&69Cd2W0S1UIkwP5@2t zlVjG72)u1DsDoOUhjQK<{s=1-0sXRtqwM0AGlE}K>UA~|4xMkh5KP6jD za^{hFwr2fV;|OxPWG2Fw@}Rf$KAMAgyAnUbZc=nG%_dM8PStF!pvt*xN|GB4xAK@_ z@3JR4(p_OXUt?*9gxJ=~FMd=(yZI9ZSF!H88JnJnnZ$F>1=H0~`*)=;Zgx4W8jE6m z#v`F?LoB;CI`S)pR7+IL`{xgud5J1mVPmHXDry~+^*zI^ws1M^}pjtOR`qs9QG+CwXUj!k!O(QW%dOR_SYpk3`7EsVnhJR5ickwbN3Q;4paJA~grC-uJb8`VK zXClzwPX^GnL-Gf#p;GI&huUTh76OB4KMc zw0j4Z72GXdi1-iO*koQ#a?}^c;brZ!H>sLNXEr}MBXA=!HvDB>b7!Ec4dK{jQa4ot z8tO=M3wgKLoDQkwuc2UhxrLB+>qI&%fT5sT%?_|zxHe5KIeyNJr9ewh<`z+xVvduJ z0pr4p*vA(NsP@$E_ipdA`>)MXW|s5V*FP=0->X$8B#I*ebc=j0Z466nJhhW`6+E$fhbk8derBlSm=XIe^OXjavt(f#Tf1O?%?J zwpT~bU-7IBp1ce2wy2Q~=}eB5aDQacp1YkV+U&exI2(Pn>u;h6N95(|ptxJ)e(rE$ zldQ}#=Q6bwxSqSQSiMj>tQF2xo2f3HoV)OlvCzye%c6AfQb$|qRK{w57$WNvstNTJ zoIbp|Qr?jeDScR32EUsX8uN9h;vg<_0GvBLYZ;?}zW{brVO345SD9N1P(yXG+}%*H z3HX3qbPJkA^m+57sUHhZK{ns48kXF>y>YrN7VQ64xTl)++(SL*O|v%}qNi1%$|>H_ z=CK-zC!A#jUNAxTWC3wxdptRawYG6FBirS$WPX>)-;w1npL2Y5yPexmVscvX z+kDgY^c+4E3fGqUztNO8guIM_f>nHO(|R^=%{aWb*QLu4nT%X)E`+FWwf6_RjDP2% zE=ZVjyE%KyFyJ+w}&GixFnj{-EF%T(?#3T{N9rxZ9T0c7kjO@(~U+!R;}E*x1b`t(%MR zQmnWM*V^M(Q+`bor1sZx^)Q#@0wQiNJ+D&W6|M1Y-tQ04Vry|`3t<23UTUqNdKIcN z6*Na)>gL57kTknF=+V5RELTW3Q!R_TghRx4)RA4~tqCu-2q^wfPr-r72q-vd#KK2@ z`TOfTc7=)4D}q`F*2XLCkaLP>+)ZqY6O*Se5<`C{#CkPE3#Rw1UA1nP$sW#gxX-`L z?D{b#M{80V0kzp_Ayn>=aHrUW9OKK`KMq?}n>(-3rsw$5*JoFoRcl(~ym@@`8hkNr z>9s^Ra3nA4*4Y`b+$=h9kS^J~AL+j#yW=EZD_kiNA15or=~5Pox?pdcS7pb-`gD>% zNbISxO|L{)$Efo{ZoqjpeVTk@2ol5S7t2c)zohyX?1AB#$i*{?+R&XR(`hjzM+22R zFTK<=k)gVx6+iKrseP&=pj7XlXfQ|KQRLrgQp8GPKXLg8R|j{P^K71_K3HU#Q>UC2 zoBW?eh3zjQZ+S2>bfi<4qrlDZ<%{?6Z94Wfs#6T3%UuKzv0Iww=*ehn1?b7>)h*6w8Q^8@?)6|_Pt3)j}tdMtXNxik>HgL@Ea<&2M*q60Uhb4HvIy zxx`I%eV4<@{I)`3JgvrUDac**?c+TDKP2aKh57lSn8&($+fTJ#umJK1-0KSchZXlpzmRG`X@N|GNzn zOPg+m6G+a=j znp<<7{KD&8t`A{N<0~%SkqQdQvC2%p~O-$Ck7P=DUy|gDNW@5mW8tu{fq9 z=B*U^#9s~?Zu!jCMM4QUHJM!B`5J06=H0Y!ha?VFHqOhQe}0;vSS;5Vb?Q<{5eaz( zEOV&O1e4as6bmJy2aR8*>F5>^+cxJST1{b9qBzHQP1+kCvsUX3$CEL2WSqA|DXB(j*huyuHd28V?&lc9aAGs7XV|x{r=Hb>DpdVooCt;( zOt)vM`*bs@No8uwOpi99AGE(&rhZal{}eBIDK2`$yG^PCU}%4;OS-#Mb^|_kFm0)m zVph3US8S$t8+)26^Qur9%aA1DrU}fqlrx2r=(i^#alSPBr_!&D=Xlk(0&m{+F0lFA zQT)eW4BWdJXo>@P|NTAjq*IKDApY)h-L&c>T2f_Ci?#Lrp*V3**}Q{LxASm!^D z!$w4wCGVqh`6MVYs4PxY0P_B)cdUZ6dBbaUrK8VF?j*INk-&fJjdZ$b_jy!4Sqw$P)2apG2#xX_Y&2@d}Lf#j4Z zbl>`L>p}^w3}80Ch6`YvkxZYq*DY0?#sW!^P|`k~DoJk&SO>LDUbBa6q3?$u?_bx` zED}w|;ad5`Jk=B%SXYd2v*~oZ?0Wu_sZy2nDa&%v(X)0)^IoVajg!T@?*zDA|MB@| zv)icHO9fcEe(IV-;ER?G7yjUJGwb(|UnbfdjF!@>t#_sfM&Y|p-yeP;6- zr*DMh3cl$#h=6)Rnh0NuzDk$aw#<99qCf>t&8wR`dx!pHl6X4Stqc_&Lt6 zU9YEAPKw`F>DuYz@U}o@`)8&e$@A@D+lpmgOOusXRSH)0Hy!8fBxvr$o7E=|v6JYy7h7|}uV66Qpw^aQI zJ5&`0)Vx%-rk#VKXXt9VYt;GI=~1UWkhA%?b_*n=4mIl`9OPYAucIgUAbJO+$%Ga| zCaRkC?HzsC7aT$XfyhN{h-wfSbhmRzYL<~Dc*?Fi@+`ZOX!Z_r(Fty%7hp6CgIHEjf{yuVUuDQoGd3lX?O41#VU06<_ik1HV(QxLa4R$ZKNkASP-T?p3hQC8 zL_tVr8j9b2xPP&)ZO=KJ`PseGtGed`)I|7RONWD#tlKj2m5Z0cGC$p61?B;SaM~@Q zP$)Q^BvFm^m za$0}bH)NNyOZNwT?bRK7qJc|5A*?Se`cG(*R->#M6v-%^0m%mmFe+I<6YQQ~v^X_H*cdYgjt1(i|F!L`AT zE0%UEjd;o17KV#W>IHZ@sP2}`*T?bx{%8B@7~LN~Zybq+Qekn>IQ7OV_w)ew`FKp9 z?(~h)ZQV`5N2>?k=B~l5-&v?4zKEaowOW1IZ!oT9F>>h)g-&qPz(XVy5Y^Rv$t_X( zM4yHLUDtOy)JF_r^WqUK95f1;l%$ek=*o}0CEmmiGL`j_sb|Cl@!Ls>KgOWSxzW!R z_st$di0%mpOEowo>j3KXg5tr6>1{>Df(24SSJ-S}nk_bWpIrrf>%|NCK{PccKXTLG zpA3!jC;-T^S9uqhp33SsLGZONO0QLi5TYU<``zkqLCrRL$Hi+(QB5=9Ra*?R`YD#B z0V&)VM8FPl@g6$thuYz#SQ%607haX@3Ev*kM(a@MrnU2J>SN<&|dH+m3$h-fUdSmBtfn`AT81p^z$>lyincyYZNb$@OsV8o^SC`E0 z@)JEU`^#nHjnkNcPnM29LI!MY|IED7)bY+cnSDS50kcaABF94lGYg$z>(4>A%>iTTaHxk zR-xiTF>dVx8oQSAaKeFiGuGdq&{+>}U5V|!TnoQAFwetMTv-xXKbjg}opE7AyX7#8 zg#kI-`vr(ocQ0!(2)xI$D#O9!1%&WZUos8C!-yTt-9lEh+-!9KP)cX2Ku;>BSHX2k z7f4#UvAAa4$LQpNqd|VTTl?`%bTIT={Hd(%ya!OH8|w?&GblWr2>X@S!TWLiN9QMqu6j(O+6w zU)hU^gouIlecsA}v6CA%3N=;TI0P*ByOSd`!-LWb5FDno-+)&HV*Ev8`+W7H zOu{X{8_rne*~y0^789_3&>f{zJ5Z*h% zq2DW6)*StIloCR1>jwZSWipxH$`(sl0xA;>xaqXB=frlLl5bk zV~e+hJ+zdl#(p+KwlH54s~0C(k{biQU=Zhp8|EI-gKQ2Dn#b?WykDVv0t;V_9$$hG zgzDaut5syI-{WMR_M7Z(y%?Wvkg(E=6PWOoCNn+yGZwV}cPzL7E)bv@A6>_nfFY?K zu_EpeRT0D76i{r?Pp4Uqn8#6;ahd8~Z{|xlKtnP16T ztt8>XX&$Ck9X9^r5wa$g5Q{r=>LAR_gupDWFC!Uh2RF_eVa5~t80Kcdk^7{Q8uL*B z&A6R+_garj@zdBjlAfbZGmnoVPpedwd9_8b4BLu8)|Lq!>*<>9k=zWd~7r_!k!)se~EpSP zB9>||ViRb-c&G*;>A%zTBr5X>xy_Pg$IXLalrPoJ_GnW}Uvm5WYD?_Wbd=L;oi&p+ zpEgoVrLC1`e&u2mzZ5QeD}B*X&5HzE6mn#1#JTD+AP=|9JIs4gi`LDl+%f&igRW}o zhuhDEj`Q6t^W8Vr|C&W>>Bw&a7%kx_Np$man6ta zghKr)V{c@^jb(svKMR|sm5WE|OYI7nPt@R2YI|`w;qnMf*qGiva`xe$$L@QKa`_=E z4aV&dQ~D7xqm}R&`{^@JQ7&ezL49yIKT{WY=`OqU&SUR>%6OM!;0VzjA6*vo7b;Cp z$Jln=%t<{o-<6%G^MgH28^he$3h&Xm!#K1_ihZtWk5=;1Txq^v#%+Ik-wEwI)=u?w zN4fbOgr;x6VS4I?eFrJB=GeC?rJP+0mg355^WAAxoe4jt4L>t%KTrA?aTD5P*$?UPR2?~f3-?M7ljSym)*Y zCc#v3OCnS}X&jnDA=FOySVz4&ej*0-)%=TTm+#Kf;??+?%Dv*+mUQR61wMz~1@a2V z#pl9K$B)0d?GS`BUhUGtVKIJPj{L;ef?CxrW?cpLj0+nKN<}+1ln!MH^Dl1PlR7?q zny%|r(-|8IpffRSF|!NUnZ=UktDn5u7~=~QsA>B&|8@177-jmWDLdBbt{xn19KLF^ zC-y6qa!7o|I?YMU(+f8V!>>f#B4u@D_j6wf`P%K6l8GU=aQ7uqj75#0FrB`vMZD|g zc28U^HY*jG%-SohG+ph&@(`2cvs#yqKNXios$g}Py(IyM zVH!h>ycca}1DUSPp^7)o^6{A5P}*nItFEBf+zvl3odN6%bKg%sB8Ym-(_NcPs@212 zDd+v>)Fp+)T(!g2_AEBsvR>A`1^j3Hr(BvQdSt7gR`o?`PJ6hwgvlg!-(|1YayL^* z@e=}^iAo*ROGnJ8jCE!(*ll#4`)IxrGErY^;Ft1~Tu{2rb6!eQzL(>(qSZ-KNn9aA zP`Nx7L7{CnSraODuO!WzXF&<|0)Hzg1CeY2t>Wr&(gy-Vld72e4Qu6R9HTr9eXFFm z4nC87m-Z`vp*>B^K86kU(+Z#IZ^8XxnUHB^OfLu(s zxX^mfDAgs;oyb2Q`MUsVJE|^A?rmprG07M|MNTP>oEK*CX(vhs4Z(&A)x)2uoaxs_ z`6xC0_?D4{nvkUE6*KHn(U}%trw{$HJokHCGTC`=XhSZEuTX?4_zdeL8_OhC?xtXWeQQnRCIYgLyzdBc=G-h)m zGQjY(I}>60GvWZXz16DA>lA;$kYa&TKyO25NNJs7t5{0N_-pB-4(5fvgYAMRm!zs7 z(fvQ1k;>E6;P(YkGs(+mXEkR;W5uzghHh^|{CIFn*~8bOZ^Mpm{WGR%9YtS*gZ9gj z)SxaiS9kaGRZRd;qx-$APWW#%vVlV9ubNZ8+tB=Jmg@2WfTgL!m0F1E#00r-fCGSo z>F7Zf6c=29LRt2*3sJwJZ9+ntQ*S0}jqxFd)R@^5J_#e*^d^B7ekO?0;w8icb((c3 zB1M-A^E;k;BOVQ|rpOS8XI5p^^1COMqkF=quT{GprerMP2SodAEF*A5e&e=)KkM%I z3bx4pJvj;hX8zuyQx{e?|L+&lIKLA$_WcH05yNb4&?X_M!>F&mKPrZd4DqFUVfL>B zrHz%7GpFzatiLY%!Y@Mqs$Mt$=S3@azO@|MxXub?5o)4>XUjW?0>!d@XrZc zU(wcCkAGEcv2qXNZ!&NuSmftU>ib0M+mI3;K^$}vOey~?Q5tITx70*Le3q(VF51;>l_Y=DttNJ{lk?OyYW8f?#Pl#DT4CK>P+C_fIFaDhpY^ z#r^B=y}!qa0Sd)@-`L~aVS5)UwgCF>#_xdz&jktkK_8WAbaJ)DT5YAxqK-eUN9K{tWz?{YtJZ% z@Asb;eSLk`>w_bk$M$Y+g8cOp)@%jlef|CM{WuVJoSRQ4(50oi=?4xoF15lZ&?f~* z4^ov1FdUtXX6yluL^2wzjP+vUhmq5VQ#v)TkzpS z-SO<2>T0Gw?PvWjYiE8qwX*;0+es{A!d$wx%vxz~W13rfn<9TZQanuJ3>`m+Rw{*b zxjm*o>qm3sI&5R6*R8*dU&iC{X3Z(-=%jh7@Ws}!xBKV?IPG&lso&x;K0Wj*P3gr*UF(@?M1A z^XH;Ge0&Vg%nc1`oG7}vCimfCO!Jn>DJU{St)uocLMtjNTD3BfkJi>KxMqQ)<&`-- z{J_}d!#fqXwbuY^a1s*}>-c!>;YUVEoW!*{^hJ4i2(DItCG&ikvb?JH}tg*3ndO9wxn%H~^KuTHw zAlK1zBKUczhta92sZ!-VeSJw55eEm3-`3nUDk+NX*7{#KwY0SG*>2LwNH%}(erT5X zq4nYIzM1Li@fwptabH{+jiVD#0Q$a5=Un!F3TiV5!acwFR8x~!e`;VL#_bXiBjpF@ z?iw?hsdc>r>jSoP5BM3Zkg;q|8-m$~m*7OGJRV_BPtU`);8m=C7| zhw~|Q&3pverI)?E@)2ciZEZb4d??>}vFuG7E~r?F;YgPMK3+<$X1=MrX`$&kiWlCy z&-#VDyu5c2kHG3*beMs@x$%*z>=S2aXW@dvLc(KD^XZopmzSRMQ(^Fv<-=%dVUWzY zp+SId#uf65&7D)(@`0>XlatSozV630Qeg5;ag?(@M2e3-wjc}JnLo*zJvx=Kz^$XT zvO=iq;~U51>lhF$D&X6Uv7x&93TJ`T+Mw2Vv3uyKtANd^duAk7Ha3bnI+T&W81J(# z+|%3pUSxQ?W@37JD}B2lRl_pcH|3_=gQ>Qm!Dl^uyLHP1O5bG6Va6ITeZpKGzGz7I zjV&ygOP=VeqM>28EnJUYCmrYRIeRoHkw#Vg^vwERgTwKNoK3|e4)hK&{E3xBDmtsJ zjg4;EcOLPfcrh+@y&ll$=;J_X-yqs~MbWdy^|f6b&{$f=gP#^@ZAV8(PDo&THgR-r zcBRMl(7v9;S%3I+OZX@mMPkZfR&y~$$|o8=m`dGv=v{}~F9A@1cNEcb6A?LZ6kv}y zJ~_dsZ=DOw$xBP49OkBumH~H_B5bk4{1W3vi=l<0H4(5lBz# zQ{CQ};H&;&b@7()fTE?5k$;}RPs)ev`$($!KVy!puaP`K*xTpsI5lWHu70gSYI`Jk ztK;(LpxO%0Cn`P&5mQq-Z?m6fNGsm;@A!PRv9n`_gei`HMJxyBK(es5S5o{KQ?k)W zcq8}-pT7ig);!N-)76)0!Y(=0&{asZMBh#HJ>WA>^kSv0d;k7D0Kdr3A5=3bm;TK} z!I;mohv$Am3$q7r?3lkWdy0He8L(U!&6dZ&xV-xudx|=MQn=;8;2a&C$|jUpId|3F zL|^&dX~%r#UFO%hUqyrFDsW%{guTi&Y*?`0 zi40VMIXg>#&|TWa0GPB49z1t}m+GLTrEsE)&Zm%`Ge0(*oX@QupWq_DE;trg=`B^ckZ}FNvTkEsIWH9A6=g&i zZGx(EZbUuUZ8(y&x_m7QrqOx#TRc5Hbky?;xa^bVUOu9R9qBM;-#cVyXXgx5eNdxI zwXdBb=V!&PNW(QxrzGiKj;}0Lv(`RG1!l;+D#5a9viYoWrH=QH(srG7we?DP`rTjL zD{APGnHwKBC_ug+*d_aZrG1VLB*!|mkVr9SuauD?A#1`9?>Jp4V>vr9u>AAk>?S|?xj)>)G4a*UE{SI@-yJ@jhDo7p!Mex z$oyaYL_U&kCetKd=iW7CKWb}hvppoa-_?3=OeVsm7xf#kGuq!jC_jtKD(>70oAo!~ zs0ay>6CPO~#N@U|Q)6*Q9?k=Cdm4FkpFh8`b94mQW)RB&_{U$$@*qTB*>VC`G|21T zN_{V|cjjN;%Yv@B+`A8EXJ1EiCYAhyyc1724u1Y@ zXBVXc){p5Mgxy8vb^i$X+GN&MRy8 z%<%S(4$(d*2ZzDmiNqGxOg+4%pahwhC=#`%VFjqt(af=Gc*JjA&_?NIPE2SEeSon7r%U zizwANPh=cGOO1cb^-^PbLbof#uJRdYs;`r@ubO_@{s|zH9M;J8V5_S{z}|E*z5VLm z9>)F?R%qrKeCLdk%v?^3-p1kL7+7Es=9J+0nT0C4{KE5Q3 zF0`iptOo66KcUb%Y|rtK>}lwF>gXwZG-wp&E6Vk&qsPdm%&oLaq!?>B?UMd>h_OG& ziJ6*uT*Y*xIZ%u2NIpPKv%Zbpx}*%Tw!6FnpT??4&K^85OPL_B92jomo9eb{xabu9 zOp{hGUWBEmA18+Df#Ke}Zy}e{JuY^7qHX2vef_Oot&i&d(dp^O-x5gZieqV$Xz(sH zH1z2h#;rvZysU3uiPOGcQtn}Tr-I=FP~F;s13psU4hv@%&G2NJ!bm$ zHUd2pOq|<&900|@(qohZKB_;H@A;7~${jXw@mp9ZckgDs_h%Sn|1c3Fj&g@1g`nm35Avnw@^C=Y$0Ju*bi(7uF&!yIe zDt|M7kCh97HJ@TsaB6^UY3S-J0))bp$L;z}d#cHUvx6OVrMTqFMVh+vlm9jx{EUjb`~F#Tc_o`)|H5h>yI}J4oJ6+*HbTuAMpSaGKmHMb0}C$z8X`Y?n#`>;cNBg_(b#^{ITtK#QOc_4ei6OtMgT^F`}u!gw? zA(0Lf-n2TGwA!^T!-4647A6kKD5GUh0{(u9L+G@CL;9a9$!yK*?>%H*6tjP|{6hTi z16RN=8xqt^rZBUbh(^}rK*DQe6;!!+KPXSt+EOz?Qh`qxotm16p5Y#FO#gGy@2!39 zZz+4m^7lc`=Yz$le=trVMcA-sR)-}y2?t|`7f6um$gzuZ>DJ2+4H%I5G+LuaJ~$9x z8&t@=gzQYu`g2pbtMXEe13+E^?$YCCaBNBlq@4x=0SKoCZxJAJ(47cQ@XXhtQvewb zPdEXb>^b>b12mEveflXb_V2&2mcl zNujIz^Cr94N)S{(>ad@4-6O{L{q-Mj(Xisq1*A_qkrUC%KPacK-E0V;moMv;i=!v} zIrrQY0rD=VvMR3N3Kb%t5?6@?DFHPnoUPcK_pJkhb6}GOLmjtC+DL-RZwSUU;!Urqot;a{Y;uWo56q98+n1aLs7 ze2lQC>D30P2`Lx*A-;eW^B|Q4Hw1zfCB(h-mpt%?aPkO%f`#Z>S_)99gB-jm;HCm}V|z&nuY&?2Mve#i25ulgRtzqx4?BQNLxWps zAdTQg?5*3@bSV}@3>~TTiZM#!SQG`zr3sn~a7woql<+|Re&>g;(ZTb3R1c2=()`X3 zZs081gO>v;<^Zc_@QRUIQhnf#?Y~+Bzxmagg}$C&;t>OS48MW*SmaOVg8p>Q;NMFB zz3>Ly>sziHw0v{*2iflSLnB)M)M@`;b>ajG{a^W1C;CNSxTL{r3lf6Y1FHW=^hkEB zxj~~_X~3v~w|)SO8hsqtz5niZV?HZHU~8u77K9p#)O4*N_x`LqF+#!WiLte zZ{H=b&W)G$#-Dmd{yQzA3tmt>AB;kp1cn zoQ|$QJWt1|L}JG zY2>ePAHEUgq5z$4Cf{peuSs;W%qzR zMuLCD}1gg>MBojH&VrC=fX|09HKpkE|l*uY-^RVenW z|9{3d1cJgsIsgj|3W|_yEU^MnOMYoMpK(9#<8U^bekig5 z$O6|x5rDmLTM8rz{wH>XkUV(wTkp63*2eQEKwpBwe%sUkZ@yY9@angY16_b?LU?}A z-$s!4m*HtoP3GKbG+vFefk2F)A={#y5|-Dw8D-&^4iX6eX#j+$Ev^2k65POTR6(#1 z%=~{my!k&0fAfFYl>b%aDJbq=Q-|oULRAQmSANu0H?1nFO$3?~D3b5=5g?t%Evf&v zl4JYTo&UB!zapLhn+M-)g4-;3&Kcf+=<#>1s7F_W#XZ`b?-f1~NFV;rhqY{>Ow3 z%sv0wzyE0v;e+kpEx8!y^n}@eGj`;EnD<|McyNgVjRw*x;IcgrtbCB)_%#NjKyjb) z?JFsNon{g(`{_^jz*OYVoC#0}uH74=h2-IrmS5T(z6A&GHhkU#A8qg#aJ%5Y)Yk)l z_RBcVgEqlQ0Z$A!9`G0NnHx$+>W;TCh?E77p8w~!Su3(b@9+n1GcKp4kK!%R0?Gs< z*BBTIigCF9O*Cs^_(eWf9AFy$4jcBxgJnAOpPfUJU1yjR72wgI3nbt=bo=`k1g-6v z)#X2lOS+=Lp#6RAp>cZsy@#AiJpFzOLgmaG?M6jHe8ovZN^hFUi@2ok7C1_WfzDA># z%m?{DpFMMiT>@&hki&C<-wYxIPWT`}ND3kNZ1`Oi^8YA`8~=AvzUg{G9}uW-`!R+1 z6nnfw{Zy!6;MHM*c<}9F*AtN;=PXntKkp-qgHGu2oH<*ziJf$ zZ2tESzzSzQ_UV8EcY+MC!hbI@AVuGI+5vwBdJz60Wwpi(g(zZU$1KGZ1R5X)&ABnk zMEODc*b)CR5`BJrfzN+X<=J~G2!!L`vbz?k!#V&b{{7h=T$B9jLjw{d1qD#t7ygL@ z$n=Xfa1J~l>1qA%Tx~!P{d=y;t(kt7mn!5P4maVVjoqW8%`!wGof23?NuAMwIh{s) zkH(p3wa;*wRd-}8`OK7iBA(weUVL)8;X%y*KD+-z?bzDrB%zP_$M@JR{}$5@TVFmQBrwa}>v zU});NJ7Y`v*8RFZ4H<|jycuHsh>(HLAXHaHVHwz5HhKB->5CUHa=v_dASDX6U|1L# z8B1ATh@`)+s^YV>wf$KMHcAh?)6&xwXk$4jmy(l7v5pdKb|f;X!9K~*;9%w<>G$v7 zyTCTGuDh#s8s)9&b#QuW;bklQ{DF{A!y6%T zf~lDq=j*0xwe^?Jl3u??0^9BF#X5R=SGsmsYJ$&eL(pJb(ZpAUsUI12OiTX{1E|CD zz)6BatPh$K%)%A;11v*pKOZ5(?!5|f*x1}ucu)m4Bs5k<9z2MfoSb}~YObcHc5r#Q zbs13qbaQhveKS`xmbO~bJ~-pn@k_31Cm3w?`Pc2`<+d~MNg#)nXs72|BY`4VMZh)n&W zr{C|V6Iz4!eT6aHFKK}P=ibX`Fl#KZUu}Y^utv2X-<8wT-+wVh1XdAK>e)`03^3Qa z2sh{Zz&fmjLS8@s($)KBS4x80C}f8lgwJ0-q{G6CjELxLZEbz6xg<}I*C$#tiM<2% zNpYuXs__F0Q_*+E3v&;fD83dC4-XH0|9)3dQPB?^QUIHI1G6iahv0khjaV;&a znELv9QFo0ePolx9>^f1qk5}dQg#|-PJ3E~A(GxT@w6?=K_6E(*hQ-Aoi5DL!6WdDZqWg#3-hP-tD=;51{|ZAI*+vD5 zR#Dr8kn1vDFRt%oBWxbshPT|hUrtOw1+h4@5c*vT;2;= z#Q*GnC&*pVGcqz_aJbTIBX)kh*r5kS7T*^S$8llUoCY2yB{oxhb9}4Kwx#I86+1mW zkMTX)Caw3E?xRbD#rFHPma*x5Au!T@hoRI3f}62yHo&oR34D1AK>%#AMdki94yDf3{7fKyq?&%8X7wY?9rFNm;W3%H*cpqcRh|Gc0i-k46U>pJJ4*UI9`1tu(EP_5UY9Xd<7gj zXzOS8r$%py`|QmUjX!?S zFPXZY9lRHfI^2lPPB{F$`pTI&jacM?>(9x&BUYWN$IRtlaA9G)__(kjJjH~23Ri-B zQG%))-m$nkP9Vx!69_A(%>9N+nax`X7g-!CV_rTk+`2>i#2>_WYo7EpQDN| zzgBNO0>sBNCMLs$DHDmh?io;cdRQB=dQYQ>LkXz1D5<7voO0(Z{HTJrTg;nb2O^a8 z>1W5K3L38~%W0@}Bz}v8N-q!z<+dYEn$hethG6TZ{%@WnL*`BLd#j=zKUtl*QMgBU zyXkl8ye{}+=s0c@I>&5cJ4aCpKdV(+94*ud03clwwA9uA1bT_uD}nmnHpNy4jZ04d7@b*c?P!Ani5$wwdj|hU3ZRC z2)RTM(;#zttY`1cx9u;r->`gtE#iG?(eYrVR;H=zvwSzN{!WqQw82tfYMyP+DGApo z6+Wt0NE|dCzfaJ(ySlRTN*0oO>lNp|O(u^zub^2!$w{B=J7M;m+(q)kG?p}~qSV#U zRlm|qY=rQ~x5D_gbVm-osz9(7pZgW*$9jd$cwP$1sL3)6Cg+O)h}VaR7>3p%4p36%HW+$%=7S9CMm}cI2 zA!KRwF_9D1%Qrx#Bf@((=88s;cx@A$v;_95*zsUr*!ArdZ$6+lKKX~$1XQgj`{6@| zUTZ)gpdNWdZ8mhRpVV^0sGf*J9~eG$qV`0|NAp_+&QyjrQ(pa3Uj!hY;hHg56D0Ff z&s|$nfrI0XGaZP|g4G%VLT>Spx`uG0zIoPiuxK|~(KV^m2r0>sI~xf(D@zygx&x22 zSW#**n8kPEyMMGy^MoYRTz@JtC+n_4($iV84lshPv8_bCMkv6RH;#>DhMXK1)b=_> zzybJV+KY7Z+&}n%ve-ZLUcrkLozhRr4Ea5Ig^I!sA~?tLa`lOLgQ;`Z_VzbUnthrM ztsp*X0m1Ltu6^ioRu6?Vc3XmRN_Da26t(BXvG1gmAA{g8mD@$nD#HZdtN4idFP_sl zPVA}}hMp9-O4iPYFwD>_Dm9$w-f`&2$H!D~?$R|grB>m?I=X|)pH8Q3uxv-xCTWfM z>c`?LpE&dtVn7#kp_g(7#Gqjbh_*4srV^Q!`7#sl&y|jSlYoV&ft^0UytN|q?!e$S z{WZgwv>4jc?dj$;ye(VzVr%=R*UiNl-uwM&)aVj8Knd2W;_zXqb_~!tpixDVlrU5A z=)RfvE@8hcoa_lLmP_VyP`VV04F;-#NY`^<)OLel1xm5^$)fq0!MDSEF3ji>jUg*y zogo^#z)ih!KMR~xxp3?DmW4KN{P-aw)?<#)>vxsgeOrpiSi%{(H3aNuje9HEXL?dN zpq+g$5DBAS_)K#)OkzPG=c(-LZs<>F~6iF`TS6h*~87uE;#qY2kIMw8}i%!)Gtf4uci{+q0o-81M0N#)arQl zNArh)ORdEv+DtV!wc&4g3>Xgk=Olx%-M(8`lQ;|PcquG8CUh;nXw~|fT3!X3tvJ;t z8VqdM4e@@H`+UGSxqc~RYq$+l?F)5)c)Zg;Oneejtg#$gkK~kgr9_lOb3B1LzhsJG zfW4JmjbyL1=wI`mmo`L3_VX?8(oJf$?1dHO+Y5Fmnbc6&29)4sV~x}5kk#AdM^A5TiH9W@+Ply<|0!^DZ~}uUuC(_dqQ?-@L-NZXkJ~_ zQiNn`4!2k7kF|%0!g!7psl5?qvyZ-PA~*!6R<)q(Oe2YhZ@G#4g&lIrXDTC=ZI&C$ z{Iryrrg2tx(Tj1T?lE0O%vrFSq>33spT{9%WN2N)+L5@MGVZptODZ*OIhOt?k?gwd3pILYS_Fy>Coa7z_CLg{6LbK;BC~Rmw0vS ztz4)e>KxYzXSfd$u#hs8R0U}3r5{_G9pkH*ccXEgDvS#T=a**n{ebB92WSLiM7a?@y-i(;J9|vUONS053@18-qFVu<5R8pjI9Xs@iSG#5>|EtknBUf&9PnrXKg zDzsJ7y8>b0c#h4?8-*gBnox~@%OxVx{M97)?riEJRuQh^x!n`^JWW7g#haMPN!7Tf z^K`-7H&RijqW9fy^cjTXm417=NOU*4iVR-v3=vDRx%~dx#y-|Qj)!>*8 znx5Hmb^nt4#r<$I)sUoJrc8|HP0w-Q4W7-u+xy;!pHO{_c067LB_;_iG$uhDPsJ13 z@fKt8V2DS@J2OK5gXHx}cs3>P$49`C*VfY%sP*6WV}Vt5I7-+VYXdU~>;?T`2$d zV6cM7ogl*kDypX+!|nw4zteudd5=89MSNRnTvJhBtSjMXW2r(52Qo{6pW?-CG?nPe z)%|y2l$9BEqGr95wY+=tz%Xa)GX%+us37N6i~AOAz5Ev;4-!8T$ik%CEi6P%W*URiTfEVPc3iVBV*j97XY5pd^ekqso0k+P+0K-2OfhSA z1#72}K4r}rALias-p&vzVcvc=|Ho$H zVdV6K6G(&UZ`KrIGUR7r1qYOSrKY91D12J=l|W9gx;6jlF+-WC9jb6nE80oGKyhm- z`cv5&mfrdygE+h-^tiS1$_=E6ylJF}mX*~kLlT+t-aB@hqx3rlI*Vsa{o`WOO;ID- z4Y;ixGTk(`Wr+@{B=&JXR_7%=dvfNN;aXsBLw0*MYm{gsO@Wqu`!dpO4FquwR4Bfz}5)by~p|BM3i0&ztABZRz zMV?K`5lTG!9xp_{&UOC+kS*gx`A)Ff`Ssoc(uq6G6|$Ok7vr-J@~J*5BQ#bC9uV4y z828CfrDqn3MC&(DjqC3cTCsanVES0)td+3E( z;^sB0E`0$JO$dCr(-?LNmfp~*kdWn;$yjdFcNo(4=REE_r~$sGZ1Rd`x7w|r9JW{9 zbL%pLE;1R|+IoNG#%!r>Ycff2YG#1W$M99^8H&yvoQ=o1qSE$C30YIyKrT>KEF-Py z#G%HmANJjt$9oo5beReL?7ZfTZ}zwEK1F<#W9|KGz>rXp(?DDbf&g<6rPh! zOqfHGM;o$ayJ|kEW$=37Vo4HYP|2&za)}-?)A=CF_qi%AF|Katv7Oy$<9;heVQn<` z!Nzx%`}sMYJt{echr)76yUwC53WjOCTdeKihzS9sJyl^*vbt93*o{ZXU) z*@0;Z(y%k6>&3>`r+2di$kS=_S|ujXJGsd#4#{X#+AgPnxh%6$?N&V3QtoyGp~cA^ zWQk@8_V^vlVXYLeySW@z`2<%|db7kFOKEp3EU{9#2T?oRxyse}y`iCpEAhv#vF2CC zcw0Ck3R!ljgguD*J5ORD3dn&@je=!T!(IsrcQJ0iZQo@M5z@ZPi*)h% zC5gg;l>GGB`LWft=N3yBC;h&)_bh3UA!dm--FEl0Rth7v{Q@Q*76)_r0}-Po2ufNO z)~%rAYBkfH13OQ3|LeEuFfe{@UU-&JIb=lGC46UiwFTe;n^87wPW@0a@t?Dv%kRr} zuwk_@m0aJEV-31G0zn5Q<+%#!Xw0|}9;xiJ?q}E8p>8z^O3voe|p-?*k! zm**K5krgrdX$Ne*u2IzmBSi;Lm!~0b!4@q zk^FT2cfRaxpvT=qg-kB`nen7*!M9U~Ga>`?JnQWofc_LG{|pMt!)wYC6ZRGhpdWAI@>s_* zQWILv50<~_7y6-TYHD5~<#NqH1rV`N1F2}GpkikJABQR=Zy`RK9-L1{bW}RVbaaxq zXF$t?3wyzvbki(AAt-ZyKH8~NYH)pheaQSCcTsvqYwh|D94hcYk2+2<<_XjBC~bp! zVu=y%sm>xc+7h7(3T@vs%!qA$F45l`XfD6yaRnmDpJ`R33_EYCk)9R5tu_YVS;j?5 z_sK?T>K&vtH!hZBW6j26EoOQ%{s8$JwTTHx|@8Vvd9d1MaP^1GdTLTP%Y>D1P!{Z-OcHMQ}fyLYHu&*XQ9EstLov*b3_dF5#z8;fwTkb+(m4u zz*z?>UKuabnhMX@`L#8s&pHg#M?QQZ?gNpSLL+_$hjmCa=`R!1kOZ#INZ$m;3a1ny zwb!DNN1foQD@*&!X)seQTypPfhoJk%M3!SLVu!o$R+ll&zT*e#U11ubu{E{yeaGYJ zvlDWsgmI%oFrAkYLM$A_ww#=?zN{c$2cv@K!=tX)O=q9qR`@41cAE^<Q*6iS9#()HY z6v3cOiE@*wO>47EMWu>PlT2%0cYCzY{5Pa}!vl0`#16{bn)@2U(3sW z!jH_~Dbr3}nV&bP(NELA6=dlU*X$6XvFSt+R(-fW%;?7F9{|Onf3HI3fM%NHYRjp4 z5Qssn+`*i|4YSQoL78@C&$(a_T7-ju(ouVh6nuqMUOE zW(CvEcbCZ=_sDbg(VFGRRPJh?FU*veS>eJ+N^osA9Z4Zuo@L{ObJA1nL07H7G z+K7Hp$-|@S^_1x%cAig48J*@GU8k`}+py?(ZL!(Sg^q_nEGhJnP0c^BA-)iMb67ak z_T!w*S@{F=>WDei5%XR?&8aR~2#r7mn*$@GR9mVxWX|&R8cbAGOv}%w5`nq>0?Cwm zw+&*1b(ZD0MSMaI+KmRWItVfAE+fJ*{_Sytc(zm z&$k$b827jU1{O^Z$lg704s*0=rLbXF)$hF$FC%y&JoWK5^5dniK(~)cxg&z8#ffE= z)sD+yy3t*|62fvy9|zFgUh%!7Nw*Lk6dl)ftCxPrKQ@Z|VdAb%Jw^k~4G*q*LT4cD z@Kf#aX%;953$#ymcIH}EyNtZ*LEW1{M4H0 z2XU8leF5sL=PyTEDbkWpn$pS)4f>vkjZxK$f;s)uiGphvQ=3KXw`iuBMv{xz^;CS} zd~YA&iZ!M`*gtPMkq0b{Tnz9>5e=+VkiY*BDM7-Dk0gUcvs>8nbl~@_KiUtE-H7!~ zII*L{c-G00> zdn!+VH9`|A#MJuAJq%1tEAe4GgwQvSbWlR+k|Ya*m%dKP#pc(TypZ&-h3&RG#+o#7 z-`FTAxs9PD(pEi)@M$4x!uB1{pP7;n9JJ zh!8GKY-cs9^R{r}{cF~08BvYh{KGVol9lm0VM1F1iI9<=KDMWR!N24vn_o zD^|X6#Q<6HX*Vzn`-DVeZnfA4y4%gUSV*3(aw{wHcXG0Jv~1dErP{3+j6~Ui2{o^P zQtaRqEf*?KYmhB&W^+O%+&^`k{kU#5R6VvRA{=h_0ghW6Tt8cTAz&$5NW*x3 z7T9*|D})+1dPJ#r06>x-<6`S58~4;}edv9G^~63AITf=L*kPG_iB3*9VgW(~(uUp3 zdGSIiF%h3!LbwfVUFd`e;l_~VVD^lO`bm;2YiZUzlfU6pUE9H;>9bc<KQ3 z6(YnzBgjTC93uXh_KdR5Tqfv?_tk?j5Vlnc|8s7szcCwzpD#~HOeE?nb|hfK71n77+%n)S$d5PguH2jq579` zuGaty^wbmCWDlWPkY;s$GelV0k5EWqo}_ZXp7L!KqJylQVC(HYg}wBBMOV>TZV@u? zo4u<+4$LJeUw*J2O&?LeamzU{I1Z8K<8P|7Kh-h?P{~h|%)`JZRjiY*bkf2_Hkd(u z*cVsNYBc}WnG|Ccg2ImQmN>M4sae`5DA3ND`R)5qW|<-EQVF=e>ns6azacsf595|Y z@g1Hl%9y@WcEr(rC4hQBZvBGpLyJJCyGTdY5Tn&W#|zA95Hm=#IGRZ+5Or_CV$W5P zXd=*F72!@9ngiNnTot+OyQX9zLNE75PZovkr`@XtxpXrimMW?=KgU zjEjVX2_1~u1j;`genXC~fpokU`?}EZ89rs?%g;H|f=E$ze}=*!*G)r}6P$OvNc1jy zXoj{~aAD&D$}R%;AylrX_3)L9HnYm9#QPya(>j-xks(55)uj-lujtksX(+pd@$v?r&PPk$hZr>giq3ep^IpPR%K?Cg=jj0~koPUakCgvmzOZ#-$&^Pfo}?DWHAkkyF2J>1h+~}A2A2@&3%rXr1=zK=F8m&3*?|%rH4$7 z_5(~k`_(=Q^+}aQrNL{?-UbOpM;rvC$V!YzRN2 z^zvJLtbkhfvnG1P`LXDO%fe)(RmwF#SW#gqV!Rm|9;Se01$k(p>jO+~su+ zfl)9!pv@D{Ry!EKQ1LexbMW{@`cgMfWgDbpUOjsu2iZ$fzNjZqL6y|TB)Ix%Cm5wd z;??JIwm}v%s$-%+`ew$NO2#dMxAY(hn1a=4bU}ZJ7UBBB{Ja>MjNc_$EwP!pWRd>Y z2g&K)K0Xa;)!8R~J8!-APE~R)AK$+Bz)U6!Le8kg{3^=ZW(2!_qAQ*;cLymnq>qZ! zjCAlrq)L7c=5_U?ZAuDDchPER7o$L3=d-pY)ej$p&ydmton$u9;-CjxoG{qS9-y)w zx0A%Igeb;%pzBks)JaYIeN_}PAM@joAx+bpG&Pgmv)0+0Mpk>IW8v|pdiXY$%W=>Y zaPJL|a!V*NU5KtL$mxdW!ss~CN-es6B+Ijz$hQAnXeb5Z+sGF#$m8^KW%`u)bmi-+ z@F|}zm1x!Xw1dYZrXBID&uoz=F)yNo+r}DKJVeY$f=c~+A8mE-ZEGPc4(2d5njv^f zSWoP2A5eL~V&XXn0u2+c7+HNu{qA#^B4BDElQAhYdLo6&bE0Nt)o}}`WPj@Q=%01J z_>4kQDLU@`@@m`lp@!OAvCkR+(skY7EHRc|{TyiWK>Y?A=IBraVy&|vTA}Ze^{zs!niPSBq#CJ{ks}j`9@Q%*WWoe zUS-sZ3-xhb@@rKUJ*qfhnB0#~p zx?U6bE9K7+F?%#{0tOid{uy|T3gN|ojNFIl_bqyg;E1AAS0jNRB&7?I0ZxC}BKR`y z_|9WP{DmME3^=0LkpIWnd%#n@{&C>P$lfc3WRJ)y>)4x&va`fsu$~s1ftYc^Me^9@B?|(nn>)w05=R2PHJ-gE*@O~O#?tH&Px<0KytS5+_(aX`7w-7 z9Rax8e~!)k|GSf4cRx)Ags7dH$mP5drzVD7>ULW9y_kXLa@2rV0GUm}0|+ZWQ2q-z zC)^y4=699XUlfq@6vjOO_M!v@|6l&kE&z-dCp|xfLSBP9(!bma)~frn;?Dp?}&!bkB!xySVhko^K0Pg6nLNqTvV0p zhUh5~1QPKt*w+O?g<}!&MEgkS&(kf4#xWrfoqs<}I+ACI5P&gL{)O$+AJJ1|`=3i7 zy!#(uWCPuIJ?nlCJ=^FdpvvLUWc&l!h%%dOLqYo+IR^L;=mD=!1>PC#{(zazp=-1U zP-9&cn_8U@)YgA%r<1Z3D351x2IV;tz(dq34ouGAqPvOL2v6 zYr+3Eb;0le*Z!|tl)aAwmHOW%0$BCU@mg~#2w24jOb3`&Qj+tHZ=(|9e#Ha;$-Kij z4^#lT6=YTeSVB7r$QUPtIkZ@q9Z`#1>OF{#yTq)6Y?SFjOlly4(k?ZQ03m$Ux?>!W zH44bV`~^NA9GXoLIi>k?;uRwJ2Z4CS_ZP2xF)w;0xVandC7>Um$zM8N9!tLlj`@|- z^96yDIAZ=Fxac&Ow=Em5_awv2VTHUY&1T9^bHzj7|KWzgfoRbj?qDV=Q#NI4ZR^jz zMoCYn)ngJ|TOddU|Ki=SpVQJcbWa6RtHd3~A{{ zg?QrnVGW54w9)X7RT+5iy?V*K=08|_&x zDJVosV&jhIhr{T3xNpm=i7yDa-|@5Y1>NEBmO)GkZWRd%(?V#qrkl=>GoWQy-u}-Cy+0WB0{X z{v_`5EQ6(3DD>i=&CyuZf)ACI8l@h-*CE{e>Ql=N=iP*0mi15ai-ZbsA_1^!3~asG zb%v9xa-<44WKEz>OisRZ0$M3008C^nIoZcV!$% zja!0sAO4(mf5wdtE~AFAm6eq>5Q`%{%E%J;eFkh2%PJ}=OiDqVhVXlFX=x2GdVHk| z!2PzBKp3VIWB)!3X`o+e^~%-7MJDR}_jn6N&aMh*W3a!11BqJkJU=FhcYo}JY^*~lF ze;8k!OI!IqatgPLfROOP%0qp6fM%)!%T5{OyP_hSkptJ&?}`Rtkq$swL}YTLNMjuY zYe=^u83mX=Mok8n;P8axzlZ% z-IGE(Z7%3J1mf$4?`ca#*;iEs&)R};J-v_#Vzni(Jx2$T*0^8H^r zfTmuA<==PGB;H`>;7Dl5-5-BV*;#C%t{M)V1J&dP*3T@4g}ZSRfxZNLLBN)pgA2^< zAHStZjO_UZT&Z#wNSdqSEoer$Y-HA(9{%huHK^s4_EN7c2oJ~cRSx#r0ik6)|NOeT z3E*uyH^7zCx%bXx)O|x`X}j}gm}FyPW63$l!uO#eUQltzn@jGyzhckrG?Bg0kBcf; zow{?jEzeT5+U|(RG4-9;$xqa9Ee2NBB_tfW@-iX*)3cW9P;l zPEJ>EKp%f(7Mw>W-W!ou_~`;T*o`gV)4-b=x!O-aEjVjk5%%T13xM$nI>~3_p5HYS zH!e`kd_J!k+y{Lig~1TwYZUfCwP#z%XZ@63HsFDw|1d*&16C^HA8g7vUU}EFQ zsrt;>+##d5?lx;-pA&co=iar{SoNZWh)4DPRQr7(DqsSXLY_5zAKpO%ksHc_3X`-4 z*^-|H0Vw#qbRIpezh6}#SJak>}&{R6gr~{Q|Y@Tjeu%9+!+w zH+u3dj;4WO>t^1zst_FP)OlpBX9TP3vxcGa zMv4f1oiWh{ulLe7AIc|pc zZy{%-pub;)0}yZcUqXn{oa7v1%&C5&$HFGOC4wdS8uI;DPO|6-G869X<>fV1srZTR zV^AJNV3tb*S)O^>5IWUUd?y!|1SCuTTRfSFU+c@ursAlWY~otCt-sT#~{@d25B=KCx($%zrrP|=Zq3dT2?nR6Ro037))a=~-{uorCVvv^BJu018_ z%E`xu#%k`&cN40co4wLZLv_MT4O)^TVAu)^Ow%zZGvejt0P1I%J8!Z=?|LEWkQPID z&9*C%Q4zZ|3l(6WNEOYMkCAEiD3r?MZ6AT^xkmaWxEbLcwfqZjDdU$g~w0x$Z=`KMD>OfJAK&(KtUpNBHi#VuX2J= z)>`<^&)(}#Ek6r;84rufF3V?7nc=o6Gb3*==fTYJ9>4cKJ-RQ>)&F8=QRNZ6YMLOZ z=EEBb!iR^U(vht_*&(Bro(fB*ue|NKL_&eDJaDW3Cw-zJgPGXduPZ50Qc?Ae7M9Bq zmTu(3het;ryAgyj(}gW?Hx)NGXIrFJ8Qp6NA5;}cEPjSMry@Pq5617dbs?$~BROn) z>+!3A&_Q;m5SiRU`R5O=O-)T5!Yi#z^T?$Xu^tHJF_jaD$bSqZHS0Cr3fj2!k;qU% zP^$W~uC8vFOqw!+PBugjXIYG>)c5>MWt-r$#2Nr}DGqxm^yX+Yv24+(4Y0VfZpil? z-BI}1xOqDl&Jg}rgBB=g(9_qbb%VA@BZon5KzLB-=%0dLTh;t<&pXJ-$jF4O%Bm_E zHSYWOT@L~JMkcu``jPgaMvRhl&jTMf>F0edEf5>7(V$aM1hNg7o16BxUh9X9$lI*4 zG#(3S`<9IAy&aiA@~JSyLLKipmN!Zf-$?oo0FxnquH5n>??>KT|H{hKE?KL=|D$p}m{r)t^BudP1c-Ff%>ff9!FqlZLpH<`LpT zCG&5pJ)R=~Xnw=b*kFUT5l~4Mt)IT1vURd{ajDwP0JN50o&cNbns?o3iH-j~Lp~UV zG$FCd58r*0+BD&EjE-xJ^{QxYCZr?3uON&O$z!P1==C;eu77UMY{aj#=_BiAV?ijkkDhGytKYJ>5~T?Bubpusjfs8Kqq*!5$H=y8m*l8Fl_51FEJ z>{qmwW4h_hdS=N{P?iM_|(}6$ zA+ZJU)6!4x6ylOn$S3!H0j~p(G5?pSh=^S({G~-)k04;)jNqoGq=Zfb_QTT}$;he; zh(zj0Li8&sgkqPTR(5?;(^+<`L3(<+>?wZOT?M)Ktzc$wgCV(Or_o*sAtWdmdJ$+> z@BJa9gK2TUlO~q`jqg%+^?7tQ7&8B)PgA+`b8~aroiqo@jrG=H=J;sbYii|HVBV|X z)FD^44a*}Umn5qa;_sqorF=xx$7V-Hh8N(EE_u=tjgE_to*POVWMgJo!?GVhxN*aZ zC8Fn+)5aXV6VSYJbF-9n$f|ks2JPfrHV_KN5Frk%q_lLLc01|y(eHKrqM{-;(WuXi z%R{o|nPW^`3S;BrroLkn6XD(*0*R_qpcJ28i3~|R@$fj59F25m7LAULjYT{uc=s;i zwDt!9k38T(tT8C`VBDglqzpCJ)7Kx@e&-l6`_ftYWky4Os!m+^DP3C(i-ec-Ln$3_ zayx$@*m=iNclRC!dhOlacDUpDZ?Uw;`1$#@&yK)+dwky9m$$Qk+r?uaNpq}2f2Tfr zb$=|s!acjATx2vLOlsdL{YR9h_1QGE@R;_*0?zQ4L}}EbuKV%vv7@!QIm4^FIzHh{ zaiMy}1h5KHhnrMX4*<{Sr^Fn<66c?tosA{a#|a7xhZ8Yna!8w8yqt{OY@M8VP?MB2 zqC)+!^oPt}czX6mi}Ufd?}RT4+*SY{w=6Z@Ya^MVdecqb4U@FWiZ6Yx~q{j+>P$6m33_6)dCob<&oSm z0U@E!)%Or#jz1R`0@rW5^PmqVQmUd8I>@`s}@qEZv8P z9?Ch2#2cEl$c~P5dUWRTuP+t(5Vw?X(jA$4(&eTDj97!BAn^v-?b$Fz4#JM-Cd8d& zCP!*Fkz^cGT6T_(G4r-GNI!{!G-r!ijwF?NezmYLF&QSva4LNhO1!<#$wBOOWF87( zcmAY~-W{q>7PKs%kpS-% z^^i#*!E}Y-0gdAsu0q=sNXct{a znC&gTCG&7uA8EO=DI6{7nysyEFVvWMzbxZMt3tkGh@z-g5EKYi0(}spW}i&YKA_B} z1oSvPVy_Qm!&!vRmN_{&%MG+|zLn{F?&{jfGwvAjnXf_2$?oO?`04&BN-*q7|6Y+> zGTio7_lMHb>%n(9UM4ou_Ge?u0a9JLy_p&Pvz;I9A-h5q3)+Fu)FP?b<1q7z==M?N zw7Hb0?KD}@KcaTq+Gr3{Q;BCnP=Q&Vztmtx#Utab{o-JtE!duU)5M03o{%1WL6$d= z0?-rKX>Q$;i-~(V`J9$Rt%FzE$?jvy*)|aLG{f!>vkV+N8<`U#Ugquu`u%H?`>or` z00>}N#5fn8LcfFH9uH+o;TI5?Py3ETnhWR8kUrxfMiQ-exZLbd9A(KZHERf#v1*W~ zJ)!n^e|mbleF}a{DmXWNrl+SDkAD`ODfpfXy3%qe9I3(I2=m{&7g*O!5SMjdypmDT z$4U{P8Iu@`eBhc%nk++(1<2AM#&((;$dO~$m<6tQCdF;O2bZTP4oJ5H+Z@n^+i4Ve z5wSm_7!P|b6yr|%;?69WqHG|+rmQ$=;`s5OtWzu>q;4d2nkdf96>-i{hA+$KMQSnf4UIv z0D-E14w{zlUx<$U``F)$A&wX7MGy%}43oq2$zsB_4L`q+b*H(H3uMk5wyOw8|8HsqI!LUE}sz;P1^z1}apxaR?MtPOgZjE9_q$gr)P8RRFSSN~(IK`B0Jj!| z*diLC{6Me9|Mrvf)0^Q|#%&COt-B^=8IboR7aap~HxN4ghd+Lm@9ai>7QU;;o1lS` zf!4lSXkR|Y28vqe$_!PFL&p^lbKSzY$F_E}+B1?98_y1H0sKkG@n4kFnORoM~e)#Rl ze_aT<^#D{yNF%f=kth_ERS%zY5H<65jGzLM;3nlmGD)-;KYD+HMV3|mlvA@#t0+Oq zovuiDD(0mi$3?En*dJV$_&{{YBe<@F_P2Mm5C3I${j@G*Z&;3cX=f_!^%de4Om&lf zSzpDfFM7Za)!k%qvE(I>PZeulxbgmUk(T1C-u5GX=$^NXpw^|@(#8F_E=5Cr5d1yw zvL8yor5AqmyBg+j^gbl(Ot+uR&1Ql?HpM|+oT#NoU32Uwzux<@I=Szi@|F#;f9ibh zLM@-NbZBWf`fDI%;G`U8Rr+Ypg!251aid{NjtHGgW>bpHfGGc3ayY&E$ME{w&t_5C zwGAXc90x$}x_)qNovM={|DfLbvwFdEv9)Evb0Fh$pBJC~lE`)OCj*Q^imKBY@e7~R z0l)Ke`~obt_N(olI5@xz4GodtQ)<8=f zS`a}eJux(W;ikF{B606~K_oF5;R8a*Q-G%`xIZ~O%%EQ?H38OCu0XK-8Hh1I1A*QW zAP+zz=-5Id?t=ivKo!qct7>YJe;Z;OH+c;Td^HBLx(mX>!cR8Jj{BOke>%xuNCA@| zu^u%!vh3{91N~fZps-lMKf;94Gq1wDn*hfhV{)!~K)gri`oJp46cR{cXCdYc7AH5) zzkqp=kk|kw?rv&n@|i(j2)=x|^O7oTb5=5(9mDC=DfSe^z!;ZdSd?C=s}-jPjW50d zTKP?A$9){-JBsy{mBU}M#b@&H$9%rJsb3?ncUylR&#VHl+?IGmf*b)8>}- zH5Z7U@9ym>!Y@>Tv9Mu*F^-p|mBJbvpceBIRLpC?rhH-#cUg`Deiokq6g=QsA>c|| zHQ_&iGXM#mi-;p=KI+Ilc?atz>L<~<#K?zs<(9n^69zY_`0ZXN!%2n>xxK#TQdiBSi&kJn6ydx4^g7zLsR0I0qa!07Q?wdyl|P?; z@m~OpG&935&nwVqPY=;oc0nsRt6O}IV=BqXX$3UFewA#oDNenrP|}Do&WD3};~-*> zsB>M_3C1R6hKG@|sM9su3^Ok>4ecHK_L{gE8q_`yBF!#jF$R*CTMF#(CqK6|KjF?t z(uz-yavAm(;2T}k!-C~V;UhVCrCDeAZ9K!);DHXmu&9)y*YA$01fG={EqzH}daIh* zbCk(r9&Br4lM``ru+$}4{`tWf`Y_mh5v&b3GM$d%BB^B5Sihul{96kDr5QcYvk+i0+1yqt*Et%qyLA}OP@ zxw}o?$9-*Li*_ptaC2XIkJX~7%_nkW`eZ4#9BlT~*U8fP8(ms7l(^6NVMxTUgUM?9 z_z%h0=)(T2n7?m@-kG)RcgOB{r02?nko53Aw-N-=WVLwy7^2_+nq4Zz8HrSr@e0eq z>hRx*hx;D`p%-H##<8H-Q9@A?H<0#`-AvOI5~G}+oYdh{4BZJnD&d*onix?{LLEae zS`?UHUsv%24UPo3kxn=LGv0W-SyueONN0JjhYf`8kiBlUdzLEAqwxuo>k1gHPP#Lbg1RC|7%y(m!Qt z>_oL2RnKRvZ!|LyN`egaAx7r8E6nEl+1ZFNro4rM1gFXRA$@Hd3*yd`f9?^&oGlId48@lf(aWoTN_d@ z8$v+ybhxsOrf#s~V};-Tn==ZeBMEMOpipbTG!fC|#uM2Yy2EZ#OH`1--~dmFsOs^dcp|Q)_poxE?C`w|yj@tzTMK?8 zXs^p9#;ADTNTfYH3wN!!DG=)RM$@r_fS>G}1u0BvyOA8min$N{{;UmkK`Si`wG0r7S&E?WX%>yMt|-IGth92At6sYa;Hz&FJgMGM_IWu?Qcg8cSTZHxa6)ei;!*@FYX%dK+Dr9@%kKbv1?ZPG2gY z!E32B|F4)4D#{Et-P!ngePa{aD)nSS)tfeI>-3TC*-a4!#;(<0 zzh)axPel&0p;hIx*=}l!gr+au;l7eNm@C~&pK)L3>eN2Zw|(|ZRb=SeXxM&WpsbSb ztJl-;*}&tU-S;ChC^vW-13fp#=0V6U7Z;u`;J}~7LzF!_+-d4%B+^q`Ud|I*Z}sKo zv%n(d4)biNjGClJtb-VoBdrq)Hpb6gA`o#j#vu(|URhB}+xX$j;dc11{6I1JzuCVK49B`l9MF}>*cOioAdr5+p%zC zQ^%K@Q^3Uj6zFj@Wp;*Ej1dgp3fp&yIx;_&)>o@y2;_6lFDqcA%IC&1%Vl&B7ZVLP zxXo)d890@)k(4Jgt24BM3tJO-cza=Iv>YYru)Ds`t9SOeuiMl&`RnyEj)Q6{q8Y_s z11jm(w6@I!k^Aq{nzHBZwQl`+QFP3bAsq$uidoU-$b2Nsx)sVD5gz1Rh;pe{zzGG% z;r3U>y@r=zfl|8?U>Q`baujU|DFqof>A(&O9kgGWkK+bM zpx>IZe66Mt{y1XT&uZ#D5%#dj87HkP8*S1%DEyQSst?LbxHBsi?aB=?2p16!XrVmm zJ!_Ot(U7shx&pu36==;R3ic~dtSsMrxR=jeUvkHGxHDEFX?3(K>OO^JgU1f@X{PY| zKDyM2JE*VO{5e;=io7Ua1?!Tn=aR&H4i$8YIlv#yEI@^kt?Lwd;)xVd-&y@QbH7N~TxQy9V9M(HHcauMfm>qe z2}Rxqjh6TL=$T}XsVom`{8U$XP(oNsTp7%w=?k>HQTH9|2yJ}H@E5?OR{wbQ#9-%@ z?W*6ie2t#Ikq|-+c6)6tDOEKPRPFr2v3o`WgUa98<&-rMZT4Mt zQMAYgOr1tIqN$-+TJk-Mk2vjwHWk9gvEb6Y{CmDXw1p`Bwkn>RDCGpY`|U6L7%=2a z^ZATpl-`ea^dD9?#1ifZHw7B`Y0G>bP=Ab)jfIzcqQqY6=a4cIhJt~%k20QcEbepS ze)3F37_zMC*GEyNDO*s)YQnj=f%arQ4F-fg_Wz@Z+u zuEdWvi45OY>fl!LMP+Q{a`?V#Y@(FXNtR*!o&a-L!t;HvI@F@vT!sZZi@1s}nV!;d zCb`$lkJiG>Y2$mpr6bF$3|KOcw)&RU}6BFxirl z<9v@<>1s>^Q%^6UOdKmEe(UFHq1?fg&MMtvDT4LP8-BJ4s8(yz#W?=ZdNWtp%{$Ng zsywCwV@~vlG&sgS~}E z=Q8wcZ)V)7>(3g@7qnWG!2Th>5=_Gn%DY@tsk7Ve5H|B*irtl>8HR%-`pmF$Q^R>A zxg$e11Jk);f@Mk|E9k>9iL(NAK_q2|^KQ2v@X!71D8>tquO9Bs-xk4auJJwEEz+w> zisGSgFB;h4e+d=pR7S-LO?l4`d7aXa53j&RTC|T5tNdyQeP-~nrrZ%p$HC9u}exZyht-**HxhOL;Wl4 z@E?b`o3ce%r^?GhQ{*#r*z85q*Q@wmvfbZpNaSU2ZMB)iM}_^ObzN^^vZ3|adCx`3 zJ*3+l4|zKh)o7e2uc1GuzX&=CU-h@zvqQi>cnQ67@4kr;@~)!!X^_MmYmtIya1GYg zy`<^Yl4N2oBYPdlvi~+-2SB^fPZG8zbbkme&R>xXfF~E~j!$&@8 z(O3tI?)t=&mUoX})kEeg&NR-xJOam}nIt>~ zfi|(SPswG-2==NM$XwZ6ugvtME=}J#zVl-&QX3~Hpm}}CZas&9hW-|pLqqVUpcUw zQH-*v6dH|=&iO4xlW}zFrJ%na`kxnoHftilgsuspUOKgQJlIgE~l zHqSl*Hnuai4^F0LqZkHQr@rwpH>Wl<5zi8I7ycezm`ehnKcjEsByj0*+vp47vqUtx z<5m2B8uFYomoJd+zRAsA3|uSuYv11A{I+Zm-oQpGn)o3qogQtxYFp`fWWja@Lt+=@ zq_b+il!}gRP!zJbmZ<(fjn~jPlj>v+`h1HUbMzqqjtM7^NiWdzl?&41+ zqm(E}P=#+gwa!NQi?<7eyo5$|FbxvQT9iV5HgM8wXq*+!64;vwH7e(Ar;Vya-Hb0a zwj?C?-s6R%H9Yrn?B*ro!Gw)Ox$~nV3um&Z2X@Se=~RNx$1p@58p`kf=&QkDu8XzV zYDnRxx89o0Z0i)M!JUnoaIbo#*;M(20b7Xzn6z)Aw$jx6K+lHzREA3B35(6Z;v~&p zCqEa@y{|RfUoAFeh`ys_HGtVXMCGwR3^?|j!W1Y3H&J>1{BZA5#C zcujMyvg&9BthN9C9)Pj8l+-PR^2p>NKKI*u?|E)i*~@YuiQs&>aTY>*fFIQNRJOUy zCjSYko{d;+Z%M=_a_eg_yh0tYErlP7HpbKy^O;%zUU}mb)!8|F*I4*>F@P(~Q5uxi zK(EPZap`s59afo*>KBMtIhfdem*u%W|BJ1}Gg7h@KfG)HxWpMH-9QjN|4C26zZ(N4 zR~4U%`Y8w5K6cHH%*#>PF~sE_?*z+3Gwj2Ms@}V1QzOFO#T>HTy5#me)3jCo+)Ho1 z2=b2Mv4?%4LCo;x)A~-L!@PVmUf@2FIfks*1wc)G5d(HLeYp>JX52E>MIrV4>BVB;=7EPXKPpB4{-LfI;hiSUv%xp= zN{6n>JUrHp1;L(4*UCsp@3wG-Q!*Ag&9cm_YWSSAMWOe~?$>r}0A=I0(tD zEIvO?T%Df*Ei@4$zSu-RrW$aNdHfp9!=tSJnh4XyAEEkfFqVo+5EH-@CaI*gM z?)vUK1~sQ(OJc=2j_*@eMwjF!>5G&hA{|sAm(-CZLSq;Nwp+QOIXV6*p<(`K*gw5J zb8Z>+Po=G_dra{)ky3wMFUd+Kq*9APH@D5Y#WRFs!MYyeY8X+uStU^xV6cl_!?XbzV+=;aq)4+?{3AS1Bu zOV1=ujlBzi<~a-V&vmr6N;3^hV3v__pP5C1^+#@;e%sp`wuY>&Zpn{$cf6e(kMReuD}0X zNVgmBYphe5?D`6z>=c60K~;JPEmEx=9V%ch_ZP^N7T29SZ7?%lUmXFlSbU_;Wwkk4 zKQJ58Lm|y{EU{Fb*uLqBNvb7Q}BC;eLKNIymr3Qy0Xs zC$Jqs^qcRwDlVyL;XFP04MaIdZw2^!FZvBEZZUHCLU~eQ^44ey_+OgP5ve9RvWB6b zV)}AWhLx1JR0C54yI?qt-pAUJ>)1O#v|4G-#51xM!2}Fdjv5Ua7%Gz^t0oNzP=z#_{8w~eT=Y8jhCoTjc`Ld zX7+C(a+=@rrff1ehRYp*>4?V86UG?!x|Gn>XXFSJk{iM3V=XM^hoA{a=F?tiJxajH zdVFn*y9twC=SemyV{8?F-pma!avcot5I>sk<=-T*73wth+(nR02=>g~C;#QiM(5qgG25*n< z!AW_j-k7~#-+&RXA0)Hlq9`paB9gGJTv1bREO>#*SMeBeF1#bxPxcEUFEa-+e`cyy@=Su^!X zXCnG&mS7-JCZkwpMCc`SxFz$OFALCcG#=92?pJWVJ6-yqA^Y*#JMYY+z18Gan#-!h zL)xiXGE!2cS>>12f!z&El337Cbuadc6GEP!JNrvS#f#ZKJMIJB1d_!2;Th1#EbUn% zgSY3&_Fu@iI@a?DiIz;^{TXPw-^G+vrnwWjaTi4HzyJ0gwOWBqkDI5ys8)ekLl%*0 zw|1W>v`&rpLkeI125Q3KE4cs7F~%Yx!BSqsAt6eWbsaB?E`5c?2WWHk;W;n2D7uKb zm1z-!g%6fR5*FbpI&n8~f?B^nfOU$9ub_S!|AFl+|ylZb)gbTl@U&XgElj%)({~2-tiJTlGEuy^i%OW*bZ{49O65IF$x3-6CHM z?K9jL9!z7uZR*fUGf^!|^5Ct=vcjCZCA)cFvaJL)ln0KsMmMX98GwgEf(O)pU z&!ptBYAX3@kcte`4}LZDqUCsR%OXzMp} zPY%+p_1d}^H)e^$G34yP7Xtf!mPuyQ?_PavI!9b4JC_Uhqz`Htw!ahZ{jpS|j2i=X zIaYY#eEQy-KY$2sosJ%tT|!Ypic6FPy$wBrG(t-2@E)<{<2Y(@?;{sIi)GgBHeORz zGkEF!CUB!`JX#iaM;r3zg6XZ!tG`*_Hu5f zMG*C{O5qLb(AN_ukwuTbYi%QR-U_@gAB7`?$q}ue;)Ts$DD!@J$afNl4Z3{y7`}?; zgbQBEn}C=&(;z3(VB+qZ4>eQ6ia7P3E)lcYy`gWDp+!6rR4nl?A~(Tvt{8eRHC9Vz6c^q*y<1L}7wQuyV%tkZidvYwCV*z#mZ?3R)9^OZzIfxj%ay zApzaBsii1-H{{_L$kWebbWhrc0**$_G>_)Z*+tRS{)R_V?By1(q;sTNkq_nA@)5V&ZmY73Xat_^&d+d@vajMh@JAk7N&<^+Z>;+R9SCSZ(&;(knocPxMDHk zyWSbA-43@(hL5r%pR1nw7 zywlQW;23i4`}izDe7}o9E{$u)-NuSFurd7ihf?>uk&+}vU#zKVSpxBG?$Q@O8SUG+ z;cwV#7Sj5TnO{OvhRj|+ZZn7@leW7{Oo^R13$Z%U#%H6iv%^Q2CfD9n7zQ4rw6h60 z6_mL@VQp16?~hwiwF^Cs4NsWH*CTP1r+42aCw8>SzTc90VDLRcZso=R%Gd6wm(Y3o zfN+^|Hiqvb;Y$f;P%Pw==AqShysu;41VETI=M}ZiNtEXhIs9fE##D&d@;$J;kRG^` zz8u_2L)MZlhbWdU77AM%xgmh=V>c+Bgbw@FjLNdtCD=8UxweWJ#c;*LeO(jzslA+J zSF0?K;#Bf>ZV`A>2cgWLPR;foHOI>xNsUDNG>sg4sQ#;_r!Zh-v$s)|$;scYTX0;v z$6%=^L?I{~6DqfF+TCU$_zUd+@DhXVPd8oKFJCfYWQKJviai$s$#;q>0M+ZAa6Zjj z1q$>~B@Lfj_0k}>P3?BTbMo0JMfuopx>wCTJNm3Ue||W1lRnNUrL&HyG!dJ4oF2If z(6cW-;4gaa0US^mWYhS9=L($Mm?$zq2;;maWf>^XN6DO5ia>^q}S=zafn9z{RzDIVs=`a%ut<*kJee% zwW$Z&Qz^E%3Nq=*J3d1eG`7{I7Ck#?yWAFdW@iK3805v~-9EZ%6B7L9>kRPr0}^Hq z;`sYLQS5%?M`kwkJJelc66Xm#Z!ut+?~4*2K_9+vC#sbm`V4u=gqG^`j2baGz@@Uo zkK}fvxxQsZhKQ|B6=<@t;5m}O?u6V;SROhv-H7zg!EO^=+t7GI#!h$hQK1V-P*kW4 zgVY;hVo5R?rbh?YS!E;xjpptQ_=q=KMD*4R2v|Qy-rXltYoDI(TkA*k_f>Ax*Vf*M zav&0)D%nDGoeF(CE8MHrYqQ}-cgja2w;aAcnf_JbX)f42brKM~vRa$SA991NU72Y`2IZU$0nQuy<|qNVZ5x;d;Aw zPI=p^-chz?j?c)W`Ij3GuN;1)5kimF9Vd&TgYnPQ?^%7Hk9?z|U(rkJ*|=DQ?|I%; zWcNeB+|gZmTQKkFaNofG$>2j*>W@vxv08+W6{0)p_T1 z%3Gm3>eSiH1fCr)Qc}!ypsQ`}bdX6K-IQjP`|T;+C8%#sm1hv=F?xD}fjRLTQ5DhhfUNAG0}!AkCsR`YHvkEMDXl7)WYY962l9JHFT4bk}|Z}H4r@Ej3Zo51%%;dH!dhei@ZxZGc)t|Vb;%{J!rG{% zOiDidcrT4*=Fr;O%Bn(LO(`-gv=~*u-%sIF#1G;e`OX z9KN!3bfjl!WTf{L_+fAZ594oBQ-(xZN3ZCQ;90;d<=gk~NkXEc9|g%e8EYcmoEEM< ziw9@Ov#Y!0mALx2yF21kb$81fzr6i5;m5WS@G1EuRMOqwAJq$h?Q0vSM@Ki0vGFc| zbAM7%#TEJ5$=WfmGqc>csgVnZbG&{~hK-^BFB=Py7zzw`DlC}qp@ zg8D0gfU8qE{vZdhKK;d6UVfS_cmCrtw|zdqFTC$ZAbSDx{4ch$x908tGvxEX;7WIs zxJzi`pGpw?&y#W6GXFtIE6~@=BBR zc$us3?gAkZ(98i6eW$SCN;gP7+4$TQlyG(K z-!60Y2@b>!uG|hUK3||$|LN2o{J2?U*6)vaq%Bmvb*$`$bUn8 zY5$G1{ojQxopqkaXEcyCZ^-erJ_Z1Y{gVq!LvXCc2L&9HXZlk+j$ea1`==`Jk;s}t zem|b*>`}L4GI=wmn<|G&LWbT;R!-C*h5PYqn^Ms0-~*JKnhMpCuv!h}cuU<+7CTPa zb47Th&hVwazDxm>K5xj*wOs~?kCXgGCP8CT)j?8@{SHxo$wOLcT0vWb<95GLO^|5K zeb+C~?Mz?Y2LM*ch5F3c*J~{ZOm1Tdan#xC7rHf&*33tUK4T*zg8b1{Cf_Z*N^ab! zs3_infq~DX?ZL=R@H(s9b8&M^^2&^Gadl1nwg-G#4)w_LC)b04@ON2F&GqVh%?%us?wc_mU>kD3#A)AMX%U*;n;;F}nhs2T7Tr+G%plhSW4QWtA2Ym;Stm_x=9ip8kmR z9`KXW8Bnm_(>DcSZub{2aJaa*0-(#`KhnKQMxK5wO50CVV)8u)3>>F#+|$>W_A~ch zG=4JgzWs1*nDxU)A)V3GEUxJ2=)+pOj&3!dMRJa2xjV_d8t1xZ3f3@~00$RWcT+Fm z=mY`M&gRw@vGMg~`O^y(ARz3c9H3edKLt-#%0Kw_MC5}6;zT;#s~ z(ezlka6kB5_?^4(9Qb@d_$n)V3H(Pu?@OxOPOsluSupo&M;`07zxaan&#K0en?p*FN?rsUx$4j1DOnoD81_cO-$uvBq>{`cq-fl?bmjk@}wRF2_d=peEln zRPt8#RtAs$=lT7bXm^^hA1bHwtGvTy!36=I)u+}p+L*-dCYB2TJ1x}h+3+qF%R<6n z<;8tcOOMB!r^m1oH5)G=gMNFO^aRA<95$B#8hoDf!PmUWiZ;ODjEQzz|6HOK|DE~M zG4LSK2jQdjIu-EmsYE=l!|iKx2pJOC$Px(Na4naX@Em-W-6OXbD&hTwVsNNHWuW;&J%})ipk%vy#&Hk7jddAKMSfrx4f<` z;D$jNgX_u~fn8Rz{JGpH=ID*Zh@b1?2OTsMRqkT4=?_>M8V9K+O7}Zxe7pQ6y}zK9 z*?xKv_954}E(zN!4oDN2-%;wyN4bq8Yvfw*JIHst={QGF?dOoI(h~+1J5zj%yHH1} zuafP;=bPuV;3s@m+5tef$893DjIqYg#2e1eCcbTtq0jW4*YBxBtR*+~hC#12S^JiI z>{u;N00eZG-ox$lf&Di8DZM zMU(e{bA%W`k(v1rl=9goGt}bA1?7Y^!k?t7Oe5NY@0RN9DBQbk$VXlWV6OiD`pj;I zKXB$ngXy^7Z1$*t=ljj!6m(XDc%DSHnKlhZwbzMn)Pq_3iG z^~|0cjI&732u07Xv;l5J6Ra(lME0?Jq`rO^hbvWU_FT{XQ0pMBTyY{zPd{1?e59mu#2rrkE=s`8J|Rnxdf_J=ng-0jgJP!xAvE6AW! z{Dn2Y@oSN4b+4ZK9g+A$^MRNT%HJ%L!*!E@i4*!VNvk%kv&$C9&~fILPW zpoSTU`f0%?>X>}S)L~-K8;F6!F)Z<^Jq^m({*p9x)tf!6ENf3PyL5^wq)zQ(HNOG_w4fzsKwR@M+M(JsVnNNobAg2lM&)y zl=iedlqnyT5phOXk$8P;n2~wz9#Qq-`vgLFsyTVp`ErpKvQuv0=048|7}7WI&a8Q< z)tlm$d~hk~ptvcvrb&$cBei=v_{ydC68oG|(Ms=l$@?5-mTpgXZ|_NnQ5~t1|C6tLb6WywW84 z_S8$-kJC6Szag;%ie}^sP>WZEYP6=R!dVG_x)oB#DV>Wt&i5#Cjl=Etegxg}@uU55h&rHa{#vZ zz61P6jJdo^uYhR9WBKoBtt5l4btAP`JCHuh_NQ6!yd<2Ld^8yyS`FwC3!lFJT8W*j zgU6o$ipZx{unr3%oprjPb@JJA7D+trH?-<1*<`daR=DsTXvSVCfg#+9aSAQ&^KJnH z$-Y{BaL1RV0+WXw-*DW?ZRzcBFliku+sn)Rl>BKj_j7Vlgz8{MmJP7X z8!pNU2b#KI;5|PF7BgW3_le+qLOgGL2^zF9S(8`$n2v&-pFU-mCve!tQYeccldL*- zJq%kstDbaS?E@5U;en5qf{AiBG?&+^K0gDFVOEx~Is0;*ghVH{p_6U;yXfc2)%s3G z7me6q_X9(LriV%Qe(GHj`5zM~M2oW3+YtqVU~Q78CCeheDP)uOo8kFGd6qNB`dZe3|+Oy{S5KbKONI_ zYmc!i@v(f3G|GLXiuf0=`W(Jq#u#&L*ZWk0ZkxO14Achbh$#bW5Fh9MpTK%RH?L!YA3{t;kH(Eo~~!mc+O%vIO= zRi#lPY~<&8$cGcK(-yOD-Z|PqM>BQ*m!y2G{iTw&({wnw11BEYN#?mQx+N<^%uNJ| z-0XvX!(KU^KEV^nS&)tPX8VA5Z7_>3&X0qp$ML8I35(KsU)JY8o@$VsjOwb4jOnN$ zNNc&stll;6%pPUYpGAfK7TRzjFgE zGih9Q1qqZXmHheBLUpfRTlUpa{fi$*w`=FIZW~~#n&+*33)y7QT%vKm6T!33!p}GI zp316Opz`>+F7*4*rVs5@_LSI9Q%JDF_umT&(e?ZpU{9@K@)!PK=JVw2)k53PMscgS zr7Dk*z0U`;>h1Ao4owtkD^j2;kyB^flhM=uTV1!;OKaUi8^@6=weBb6+#V{@SSA>` z9BtSZS=(e53Mqf1!c(d@NOJx*VgY}f{wKm`Md<}FONX(9A%M*@S-KYjxXmUF$-ornJz{`(=*~!j#aQKHMy7 z6{*eE;Gw43+ZEy?AiCPWXh_lY1!&|LCK8UBswT(@Gf2=x|0uRkelNFQG?EeT`c>sP zg0esHl$U+BWt}gJb^9=5(%FB8d3Q5*Br322{lZ3YI_GK@LTDcrtaaPg>3D z)<4PK{C0tm%AS5&gnmYl#yOcnX7E}S5jQwd`Yt8dcDR6E4Z&Dt(2(TS5@Kmqdmd-n zzoe6i8Ar+kgUdk_=g~x;a4zOmWs^FlS3#O|?7d-d=ELuiMR!p^`vy>#bN$yG#9BD%m z9EQAJJq)Z<4!U9_336K@fZ9E=4BbHeIP_UPO=#^tRzMu1#bIzQl(#VW9D!_%u>W+TdIbB(hZB*HoaoIN zRid8;6!cz83XGk|I%A)Q*=AkP$lp^!1Who+l84z%%D=Te<_IGWk z)pd!a+_kJyeD??X20{}^p~NMsubgZETkF3q2+{yTksL-HXDW1Hb%HZEFNYu?a`*(l zm08*_I8yEK^XElQCmft7_jLQp)*+A3;&5>`@j^<`^LfOItv3_L-_*zqn5S=U!v^$2 zR}m%mLq|Mfla<-9_`w2{_1l4o1P>%uLxxw7;cD#&Iy^H%%dh{kFxyK`RD^)~vhAV9 z*pd(j|MUCu0%C&aR6{O@&(7f#h#bNu9@d!zi}(9~e9A!urg&l<56i~u3Pe+#p6qLa zx(KZ*lRe()aKw&rEONeEE~+PIfX#H@L#i+CGa_p2N`_6ZM**V2kqY6AM=WiU#PjyU zZ$}$HXnlYGl@+`6ES-x_O}m@@3mSx3L!k<%m|n*RL87rK0#x^q3Wf$AjnsoOhUF&^ z8N#}ZIbQ;|bfC1+ds%Jij2gJG-R8ImTs8GLWC#?*NlCDJa+`a6(+nV6&;|z}pY8b& zj?8|+@FED1(V#+*8V*2KT$8>Xwgd#UzO*Oogxx(WpGC39nse}zVEJNVp+5wKgmcRA zcYMOzG*u$Y(0uAn$cO6%pYCHpQ?fV75 z%uTkSH0YP$YiA*jk-BfmD^haJXRox=(-}Nm{p9_ry9mE9z+>*_TnPmB>&(8TH@rV< zJ#`t}{#GNsDCd6yS8F!8PHzqC-K`fdob=lhD_Y{Z`_j|A=x)4w>m9x2a)(WffpYV$ zR~n2$(Y;Qq&IFPIHbD@emg#RJN2C>0uMsTh{idh;H)iuSw&fE}HY7 zv&($ZvPnz}?Rfv@aUu?GZ$O5coM#tg2`*jGt{a=jYGb4Pc4Zy(fCyi{QggH162v4~ zLCfUK9f>jSv8wOu)EnW}z0`{N=CBNSHnJQz`Wev@(P(0CGq4Kl!xom+G8B-Y#u{0w zmjX9yWO}EW82PjWs0(nDgGGXQ`ki2XXwMYf55(#6Mnt=YE% z?=C#F96r9=%+stLff0qg`?Bx|o(D^8y&J1!b#MwfGT`jpwgW}e2u3v`!h3H;Ya(hW zL%i-5u{OmHLa+5PpiZlfR3dmSZk@35{YP-*k_a4GPsiU=-l>(?`?}p$;+_cJ<$gA2 z9SAsk+Tg}BzTSawfSqti%M2e(ZC%T1@bzL30IupC&&tP87@-_piBtL*%JO_+3jLuE zeoQzAdl*lXoi@)n;9OlH$Hd_YQrg!v!U^0Yn^a2R#N6Sismz%U*8oW5XCo&iVI9|} zNz~u~h;eXq=A+I!THT&fpD)0r{DvzbiUU;qXRHb%@SzwEPpe!HvEO}9Cpf16M!Mcv z!nzFS&-EG7Q^Rzj{4C2RKsdri6|(xZw&d=nkEApCl55j#I#VeC($esq3c9CTMO=P1 zIZJ<#BmL3v{A~}-1EFN9@ZWC#!SB&wz_CQXzA+l+a{>5!5!Enfbpj|$@%W*eRBQ`xW#d`~{nWK~wNv2Q!b@_dJ|>_Z?fBUmW-(+%!xK zS1_|y?lS~7Jh*v-V_!E}Ey=DOqoPyt+ELbn+-4CtNgu0yphM^nmu7?gyXyCZn~VY( z(NUr_YVGYp_Al@e&M(mJrS?-jlxY^>uCDqoyifxo&c%OP`YgMv9-UAHL$GxuP%PfB z=@HxWi!&rv&Cc{EqFdlfKdGm9Iew#_cX@XnjKKUhOO2sEl=0Z9m+DCd%zQ0U2@~mR zzgX1?!zg$D?6Gucfu$Xw)JGI=hmX)Sp$g{fXFww%DN^JUBV+EpRw*=5CW7 z7TtmEJv~|-a6^X@y~aT}r*YW8T&<`0;mCoi-DB~9xPj8yz}hC9@p!6A~fg>u=^1^6Y>ZL}eCxaoDOMNv} zF6xaHZNat~&Od&bB_f@N4nR;^63SrQk8Cv0hBDo}%nZySw$oUm-_|SNUoY46mL@A$ zOuwuCrvIWQ+G5+I6AWRyUXH$=v*>R9MfKZv1A2X1@JQWXVp2DoTf1h1`mrR z2mE>hA8+qDo%NMIpF!t3OIrTk0k%@7Jjcm&0E<*(yqzOLjj%;yK<$xL&Gtc}z>${n zU23=0`*KW#K2RdybJ__u&6WiE6cQlT#0z|A2LuL61I5RxSCEbaB@n=tsxkcdj<)Z^ zOiY(-V7ZYIjjpAVNQtyGy5;Owtlr}hpcg$az22BiZ0T*7KH|>2J)*lp9xW5?woBfA zKA-oVo*&+A^S_04NcmV>`B1>e0gOvhRp6P5)qi##JB}UT3^?}_081^6e3F6#eZBK? zHPiV7{WM>#0U&-hYntTZey*FQ#CO6<0qiS&<@3+?VmeLT(V=_XTqU;+9FXs{*=`^P z{F}C!909dFX%Eq;_aD9yzMDRA4}w@i$DYz>d`PTarK%d~<qgq6cv>2c&M@Gb+V{EqZ4&A-z?gL z?(V8-EYyiZMF)8AI;|7dbKH*S6as6?v*?$Kv*;J71);cqAly+`0N_0O%u{M;$HHkd zz|Jsl;aq$JIJ;Y(goDG(=dn=5ZZp~3hFEEo2AG!SOZsxY_jfSoBqT2(im$h)ZG>Xi zrZ*<9eml|H5T9jH5Z+ao<`d$($XE2u&ci`X41_?N8x_yEwjSujuExd^ZTE8OU03RB z>bV+~sm9nx%G`NeJF{?f&}y77ReodE%vBG#U;{!{0+K>%#M{s%xK}R8yFRWWjZtZk zmf%908#R3AiF12o>ZBwka)nSdVL2c5oQ4eTw)!uA-gtwI3_^o8NzFHhyeMRY3;ris zn*n~9D*PA)+?Dhfx>zkoS0h^dZogL3pHjYT#zk;O+Z$QrWC}L30*e5qp_3|e_o`6V zv4N<&OJ+t9&Tpb1@OJV3O+}>|k4pT)nlI_d-~l=j0WFlfjdZ>4A`&+C;7?RVg3REv zUf1GgCQEUNVhHak|=b!qiYoA zU-U2|nAcBhlo%! z^#bi0{bd%tnmFHlUg01JQMA(R%@XhgFRwPFuVnS$reVNAa0l$`yUht}{Fg3wzdOLY zKppwaTMSmDj9;(ci7Xlh>@AM)-p$nXHHYswOt!G%9|DtVVUO{f^WU+PY(|( z=f60&tzUuw*m=6d=I8l8kxg`k1ZuOUs0}xlp$noF$9FZx)%U}`QM-7EcNg|CUuhO3 z^Z$wLSO02GoA2fM&VT~xd#}5jMvA|oy{G5Yk6uJ60`FQ%8&;VDmnYb<=v5J*IGUpw z*n_1fIe!u11!-%dAK`?0zrkaU_$j6F!2C$xtbAwf2j1`8Hft4SvARNzq#p;fIV&Ey zrkD|jJEjp@ooJt%Oszu(-ULH{vIy6Ha)@0t2#SliaE-VgJ0-?~-ppgNE6x&+)jLx^ z1E##kK@YuIib?=A4d?nl4M+Cc=2l`LAp#e(l*$m?^m9PucyHs}Gr8dNl5qAK&;FdKQWbV24BSOcsr z<@kSBY?WY0)aDX4v@+e>p$456!wH99gjyCyEc(bDDBbW%@p?zaOTYn$90~MV4>|r5 z^;rv>i(Bd(BMrS(b8KEf>5!1{O!==b@tT&b914fg-pwrVt|qy4t2Y9>GtP9bQe>?(P)AFM6iXk%oS^gAA7OTk?za zU$SuS`F@m^knue?bsvGalfjg{3mt&W1@RAtj9<3PxxEx2L zkZDqWGcVJG1nP7`$%%u6P-cl1z*E42hxxOfi+~RlSl1HO=aDI&N1Z1$&qxF774?U+ z0op_*2D*{PR9%_wO4XAea213z z(<25*P!el4*_>SSpWrT&^H|@6P|)7T#^X^~t<5fq)fOdSYwKg zfCEewJn}(k10kUz10qz;IET%u8xw-*g7PEUJ#+`b@9_(S56GeK(xP4W?G(EUcqBwG`Epnge+s`KvJbm!$vFizS)w7YLnuH!a-Y zzIJgiEI#aN0$zt(O=**C;w`C)nfr z21|P7OLz7HE7TqA?<@!fhz|r-)rM`F-#gSwzOdq>f@1$`NI7o`FwCRRQn>AZ?FJK= zFW?8h&lnP=yM%OAbV`YFfgivvyTSMYIHqfwF=gC}F2EBewzS||CFlwa4^-xhwtF{1 zX`?Mf1RPKtZG>=&zKU^tRFdP&BU+>F!MDtz99X{}ogEZ$mL#)r(q0beB4rYrQOn0C zgCZJB0wA)cZAsp<`U{wLwh>RAaZ(3x{~Y-&4)a?a(Fw%4b_aL$MT2nA9Oi12Nr$109iT#8Zzik+g`L1Db+|#K5T>T2nc#3vc^nJrCi{t`n*Givx*8j1!HsY>=c42nuMpe2i^pLQ zhf8|;54$VfU=!~@MwY$VM#UB_6{_gHVUR6*1Nz)^AZk^wRfdbGMs9FvEEOe3)?rXX zg9DY#lJ=aXEv_xW!mRRoD9k7^~4zB&*7fG#Ps@Q}8l zOk(gf&Ywa?aZB2C*VETv=}ZFS;>AoV>Au8Tw@*yVyJ=gb>`T}YeB9Ge9T8}0x$C>u zLmhFpdRwoVH##MfxZBX4kPs)<6e$*6A9d!)f^zezpC^BTo#1Q}`tMm#hZI_GaP^46rtXA!+?uxPm+niU?Yf#n`O9FJ?RWy z{C?^INN$Oi0^4=tx_paT{MW`#HyFL6G(|2NZ9XRwqgV+s`cww0EgFmC%LZ{|cGVtH zGhzxV60FE^>xarBoL>_LQlXn$ndHEXaKt)eogZs)a)GokTz=Gd!ZJdyf5hrx z-9HrqrlhBhN$|Wpf%jg0RWAb=nNbf2h9;w%O;@Kt)o_3Ir@`Y=U*CuOrb;Q^+1X5OXe?5`5tK(<`-6yQvC< z_KcY^^CiL=y)9bEP#}gbi@AXj1qW}h*@71g<)l8d1B&s)sN9tpM8y8!?^3KUFHL>Bomet8=xOe zVkWa+UED_Ye(ynkTqQEa$lEc#-V}FO;Fxyw?@)xjtZPYF118#1eFjgM8eT3c}idei#c=PK;P?0{CfsDYjCc%aw<(j;_`?_n&&l~}mKzEpzC-uof7I#gVri{9QvUuSN8>Gq zMf2v~Ubh`m=vLdi9tsIqkh-Ev!D>d@Pm4la_tGD%s>Kb_^nzsRGn2#^wsarmeCpu; z^DC$#vay$hH1_@=s5h?6RLO)Rq01NWD>iX%s9sPHeJx5@(;=`%8zl42AP1*TZZ1$u zz4sQu<*bA@B))80(nNe_MN*Tbz+SRn{_x$VsB@8^@m15BsB?%>ukbJv^R_GW>vI>4 zc7krOFUBqiD|I1|dYwwv$Tmdv1F2j8Sf=2xV7o*zysJg)clRa@Gpg48zS}Ty6 zweylD_$1n4R=I5!D0yfRGRluTD3Jc(r1?Boq>A01UHG<5j91@ln~Wx1?eCD;sq>_J z<3yfj?P5I?hUs_OCD_pnGtJG*jRtN*GObz_n?pr;R->mG#Qdbo1&4KVdRazXt z)BTEHz`59eU?;~NS`6t8dFX|A=XZQ7qbD`1* zxu*1>C&EsG>Dy#C_*6E|IKvWWws3rE-{tmf8v4inL}?FWrmEJY7F_ky@pRUaUW9T$ zQ(q22^>@sf@^!=P&_f7&$eZ^V*G(~d1Ju`xSM*{-2dV;?M}rod4x-t?Vn3WQFqN!A zjfh1voP{-Y+irhBg!xpRsmFWrE8W=*VXZhAY1Nnv2fj8BhOpYOTpsQx*Ky7AueLto zEZv4o$m*9p#YAVUefEihw(wWHdr?4{MROf7dM|Y+=?SSCTTfTq(U;?7fk=p3Sv0+P zXO8!Qj0cGXRmv9cMlsAC&lf*LAW(mC-&dR+)h+c(tfwL6`zG2iH1tXRdLr5ue78C5 zCMfzaUN`hu+PieOTzSAKuTY$QaGVajB3Uz((VKY z$!4(W1cvAfBfz^98+_F)e94R{0-5u6&8Js#`D&Sg?Lh~6*}4Y1ufG$PY>u7Ak*AHG z`;hydsMkxDaDDuA6n+qS06P*zI!m_r^KD#stc|?eRcu_}kD>4I`9ZP~Md>?W1F`G< zXF3r4XJRl7h4^#hDp%%>dMH~2VBwHVuYQEdSR>Z%XXA|Ke-CQpkNL82hzK~R*)L@YW6R3B z5u57HiMrtc8D25x=tkTuc%K=(9NokV$M7u2JZEkRjbWC$zP;-@?OGv0G9f>P#_7&h zP__S_Km7<4UITRO#gfAb@+T*eX(s9Sm>QEBo59UvAxeTaE+X}^e&kond;ed9ZsM)O ztub@tFAHAR+ScbpEED0PiUA@xVDqJu^pA*8{$zUTj&C~ON~U)UR-97j_B~pL7b}u} zuWyxQ+Y_T&qt#f1i2mPHa!5rt!Kx^w$sWF}9Ad??;l-@0r2j#^udX3xA4}D-|O+W)QoWEf6&{15N}XN!lGkC`ahrJUj+yt zaTMUU`PYr}pN9L-tAqcReNHeQCBnk-V^sck$pVPt|Dm#c*go(-)cF4(1TSju>Ypj|3{Gef2$Kj z`5gXJ7m?fwDMSMWm{AJfWiUr6@X}k8)&bAKLiaCL|9}6_|0VeUS26s5>jM6px(9*& zm)8HcQZt5x_j!3PAIWF5nR9=PPLV!*+%qO~i+$ErEdKEDnJ;k`RR-ML-%!?_Kqx4i zW(@SJ3j9>Z^s^OC|Gc;RPagpWhyUu)zaTr%4maPr8{z+}6aQk?K)d+H{ci*w_y#m! zQ2rZ}|Np)IOUVPxllXfRU{(8(`H%=;TS&uEFpT^d!Vcb{EvMJ}?y7|Pet>Aq$P)($<9mpC@GAVhy~3P_VR1H`)ZWmJj$R@>{-v6u zIOC#v7J@SS4TsYO+&sINSxcsuOs-|WQ{0Hl| zD;3Z%0FkwS%e+{6KiRJfz zO07MBug>NwMEr1GYKHPQ3<76ruHGHs9cZ>70#Z^#S1Y$(fR(-PRi=itaaag|uUm8J zl;KX}_1*&NszcJhKmQ}5P4av-y89n_Rf1T!oQnnEaWo5D<<0G;VG4Vq zk<<>Zq0}p+9Hs<|g!}F5g}{mb*AQo?;1CyuUt_8`v?~RUafG0y^ZI1xSNj)3&Br zsJnA}kXDwK`L+9#y`QUMYRVxIXMifKml>d`!c4jiG${=nNyR3;g3h* zw7rszdvMY$@p}o1e*}l0gY~@yk4d0@^s4N34fHmH%aFW7bbF!^-4kiVaYP zDz{2xGMTW4?W-U?vr6>l3AZ0Wr~a5=W3h`v19qaIqvV(5ufibOg-5w~k4!TGRgK9< zyoDp|V~SeWUp~ovZWWk`URg4jk#3KCAQDQ$a#-?%lr>)OnKfVt=nsZ4jR1i5Y<-2i zoYhwbFv9}8$_!zvQt-$`^YatNlSz%GI8nNHq(ScnW=z`s`*;Dj%lu#Jl;dnjD|7Q8 zs~%t*Bseq+3{LoBY2SRLndn9$JQ#OpdI8|B1qgcwov*CA3qSn!@D6_?T@D}+(jFPXd`1Spg{Uh+p;CoIS<+dOA5TK{*2Tx~1^>nj@R;OZZ2N^Q z1q3{@abBA_#wDB|`Y8C3n%pUPDY+!~#*;yJ;s7F9ko!IBF(b)K zd+?LY)HTvxTG9H#)t~A|pZ1U`Xk$q{HcfRBm)Oq17B$cw7v~Vym|DkOqZUv8vaZ_q zjt01DZ_JZUU$Lx+QcZukQdBs((eA%}u+(>B12uM6rOtUgLNnT<*0Hv=uO2*?0vp!~0&%TLEZ8B>G)Bz zq8#+_nXwsCU+`t%)Kzk1@cJdZ^8kF)kY!e;yr=oXUU-^$AKpHoQ3hORZe{Z)G z`@HNA!2#FZ+g1RgRUdznCQoLCU7uFYVu>VQ&t_JNo;koe(9qU8dZhDp8}b95N+MUl z*u%|;x_H8BNba?J!P!TW{k|t6jOPq8`R+k9z%=8lTxt(k5JTch14lBiPD0KFUnfKg;W=Ij|2VJ0Q?jS7tHI< z|2zD!Um&X1r0}%Xz+~7C;ecv(g~>E~aw=PZWlHDt94;r|@OHWzvL?EIvLJ*mlN)er z`ZnB{tcRsM^)a*0%AoFqHX##f9JNh62TP@o0T~5Luaw`7-i*il_u-g12h;Qz{j~Av zpI@ja8^k~LTNpT#YyRsQH#8o;CHEuBwfg;JX*<~!;G%09`J)NQ0*O56#)$8!k5+lW z{}Z#F-H@#BdGjQYLY2i{4_5ICu zv5Y%Czt`6CZxw&$b^KjeRmKGn7Zs{dF%LUUe>7D$2UX z7(ua`UA1ZQ$%gh|P21*WnJ%+&-oV46&hY1y{0o=Tc4d zSMhSOf^=-V#Cf_rNnATl$yrM1%&F1%cuV^ElZ|=}kG&lB@3gwNwrrnpMj^_PsPse!|Z{ddzXqcGJ5!TdNoK zS$5_g_2Ugv%_%20D3s%gEi!ju%Y>vGMl&rSEn~A0SjbU^NJvjVfk77PI5O1K;u8{% z;~G!be7}|c(-aYx-+C+ouh}bPc@i400w5Tg$-J(Y2a=yh zX_s->Ix`O=BDr5LbyM8t#J%G^cAS28^FgGy<&JKlcTnX1*PMdUy`a-3V2DeJB=cPc z1k3!TJr4smS9=;Uu;nd|kRbM!eMuK$&A1u5<>*~tzm#e&uY?|HHc!Is3D|vt#3!h1 zKiM+)r`}~={qBSn;3E8`m@UG|QH3FX7HTF%SnKU*z-(Ew7!Bl^*SJ@!o z_^3p+(w)RvM=Ol>cGP1Bf8_Zuo~hT~yry)MhGcP?8}q?_$m7{2I~&L@`13h){a^rz zJYjl>vOvMqn=^_Jx*=8i4b#C8eKsvqtCu{SISZKPv_p?E+ZO7=XgA(f+p>cg=bkW$ zzV)3h^SA4Oo!+{a1tjmc2DEu0{hccQ)zhV=Z%Yk;%a*4=J^A0(2^){=pj-xL2Qmjk ze8PzM_#Z_nTA?Fecw!yZiTBv;^dP0-XJ~#8R4-!D&U$o9elw<|ncKu(Jn95Ts>TRlM3S4J zvS6?hf{x$=KwA|?bCMAw+#NYBoJnPcvpl3-Dmpjt`*5$87&k=9dO9>wPyWzJ22{-6 z3C0VM@uCc1vf#?u7Ncqw=r#vL|sF48i;sR^N?nzQoj9BhNIO86O z#)5M|Hz1Zz7i$#Vqq8{w>c3G`ZN_z=#&FyOQ zO|M$+h>5gR$@wWxc1ZaAac6|MYJrY(CGQo>X0-2d;Yp5#ImF4v8!b<~L)N6VscD_M z`{tSc#?6hP6{WvLhYV789$3J>?T62hl1Z#(hsUUVf03iQtxy|RH|>EIy3=NREGdIK z^$~Urc$aEJal})ZMNqtNZS>TV ztL)g>L$*5*B90%kES^m*h|5viR05Obljd4aULrQwil2ySKUo&r&k>D0I8=gm^Dv0( zwaTIUGfnfS1s2<`7}?5&MfBGI&_D`jKK^LuWtk2XcV0_{SYmdV)^qWEqN;q{ik&xc zUwDIy7!^z>shoaj2--qcMl=j_{J}XFz#2ET-A=_?CaO%vAi4%Ubo_-?n^#YbKNH~G zphx}fH$VVgn;R3(s9A948iNk@gW>>ZTWNkJ+eQ9yH>C-zXgnt+jUk55x)YiflAW*+ zV*1jXXIN0r`bA!T*ezwqhZfr7wljTdNt_)+LoKTSLY*MU$v3&)?h+uQ!wf1QN~%IR z08+DrW9rltGl8qddFeZV261B~Cn-!e-Rl z1yfT`E}+>C&eNPzFqytWpO?or-}{aO6;gs|tdU-|s4Ixeb#`-to+oXZFoikzIhaff ztWw(%-v4Q6pto?^ziPln=ueyUetj8-Z`p%zM*7?TCl1%ks91B;B`T9oofaSn=XB}F#gjG^{cxsOGzQh!eIO#tIM97VLT_ux{hU7qx z|9XZ4O$>UPXH}p~ft2~0J~DtGxx^Vv?HOpHh6dcxNGUjld=yg#<^THX?(Y6uC?;}S zb2ijX;fg!g5u{q3tk$B{o#DipoAmSROE;dm4$v}X3K|Vd>Zv1l7{R4jikSgXY#whergcppBIl1 zVx&hH5+iDN*?xd{@*p1?Gaf3k?E$vqEac@+w{DsOHYf7;{gM{X!7fYrS8+!rytP{B zcyQeviI>Fj@t0k`5AsE;`9}D$DRzOZf))ErNd;ml}?4$Yn@np(#83TbxS8hS2Zul%K?>V68cvb8_ zDqSkVRu&9d^nYw@0HP0&86cc~jOdq=wE04g_QEPqoju?gfvc)LQ<&It13F^X5LiAz z_@<%W{wz;uvO6Xxd9!cUE7)SAHjy_Mkm(D|G-BYk5X(i)!ZU(@2e{CF z(erkg;@Wsv7l-ytMaznG@{2>sr}Sdv*)1#D0LV~TrHKwPdRY`uUJN-Y)3c`z_;tp7-A` z|BjQLiB4D~Taej<+B{t1UUVqmC&Sbalv;pY+G)j}((+LGgpAo5A&O@?Rb?i40>iAh z`{?+~w7J>C<}l82e{R)x1k@se)~}dEi2VO*(ca-4y5Pd)v_Fed9FkfWC*fo8c7kQO znay4*if8_wFjC%^x_Dw@1*Ds3mxlmpq?~1&z)00-q~^ zjcd}Kp0|?ijyBXxS5z7@Wd$-%u$L|k@o|gx<@;#z<^H(7ADEegl+qtO%Uu1iUx)U% zF6uP2UXe{w&b0owL#>ufe=YI+b1}d~&YucHrFQb86T6n)0||Z&f{qv{J4jmgsUYbX zI_~`4TzGuqK`Rofo*VsL|6Vj}r%7X=-QZ-}i?x#Y5ZEQS{byzZLy< z3};SETH0p5ssmPqCKYW+wp~=6p~@eo{RlEdRY+bK{@RFDLr`a`GY_N6K`p_A7Ld6l zWDfG6p<<;A>uLYeTiiu(o`@c7<054Cq|r;wGT--;S-tui)oVlNduAt_tj(A5v`M#r z)>UX@`0H)H@(xsjpvBjLz@vtrVR-M2`n}|QxGClAgk#%#&3!x#;7b=xS-27>>`axI zaEl&EB$(Wdo*$LU&b&GV`j`p~3iAaG#O7T7H2{h}3=jd(O}CLVs!C9Tv6bW?a{%_d zMCl`6dcI@&vmsW6Qnm*_94D|uU2%q}AF_)-BGo<&!JER)8*^^k2hIV(x_unyf zeBu9o5-(@U`8yK_N_JtDSrlW5nd+RcaQ6Pm_}R!c4`e!-RvnM}n)0ju{lnI{15nyc zn#v|xvduNuZDiZ?tkJJ<*-YbPftWj`^Z&c?1J;*+MN$eKBCe#zpiu@QAa*C~6ynco zgi}l&=hgF0j3xRrspSXW>sx`&fwGSLP0_eFaP~1n<@r2Z(>;byfjLO|G(N#@Et7uj zWA6peJ77PMMZs(A*|Vlj1Mo5<5s=hoO(NXz$R3)zdFGTsmvg#oBDGHq^(W>Ci*klxww^dq)Wx} z5Mb|JZn2u%37+B-S>^8}CVW&??j_%Lod4ERf-z=@|0UtN???>!_6OuG-T~g#pcmsb z@sP@dYw4bsQ;1|hG--jO*6cuQTgcY~Ui7augD&^voLOuDPK|$>!U5SdR0yDwc}2!K z@8)-Jg;A&|RNwiv+>R=QeAnd3cvW>4t)U1ZO0*cly$Og)v@Jru&9KD5J~9(I1+ za^XC{lg7aoDKC;1M#+Rsy~ehFIEo=kVa={7d2!w+>I>MFZyL6&mupi~(z6B1U29RV zq4T_M86-{&et3$lX@g-067k&4+I5R8<_2nGIDf?{c^$dEUWB^&B>qAIxx~9$fxC?k znCf&~NziLb|M$|lajHQO`Djq)tO8{US0`AcvX_)HkxmVm7e<#Mqmp8UAoo}Xskx+U z64&1&ro)Xv?OE*42;+jah48iLI%LLJCYdfWU! z()%@gk%~WEg-onIDJ;1TmHXE$583-i9bE%_`0)UJ{-*eUcM`SpzxHH1@qq5=Z{X74 z!u9s*R2o$-)H`N&;vm8lw_0NOk@I0 z3w<0@6j!S|>I>f}Ag}RQgD5s!fyT+UZ`>H(iA7i;PMBAv|5glsCh4M2C_D48`LqVzL?`Z-Klwy?D8;$UEbNT=v(Od$=f# z&4ObY;=GRx6*#_jj}a?1aW1wLXoh9jP+)txzN{8xxp**sVcF)I=Z=?)fqQ)U-0CnR zPV?16w&m}b&rK#Oq5IT0hj*(NYLgWKd7@tiPKHn1 z@g}@1NnjDmc8r~r&Cy}EUHoZ^UxdtXJNkyjSW0hyDFi@(nGSkkPDs$1rNW87jn54F zv zQW8pcNeKunDInd5fQTq1BHbN=G)Ol{OA81t2!eDYAs{R*AOcHweY2qNtN-`^eb;yR z{q}PAoHJ+UnK?7hJaONNfs03>Gt$%c99LMv?poi<>gx4{Vx<_nGjU7bvKrrqVEqza zB!qrEuOP~K){k#NGg{d`Bl0ZrZp~TmE9mxV+}s5HrIqlwtoQ2bHI_Il*+D9xXGm15Y)&n0^^r z4sRU9NoYxFZeX~@zGlg!f>7XofjBFX^(2-{ z@DKYwr#d^~-k=tBYY|!oyfARpuB&f38mO;?t)U?{D9Yx+oM4_JcVqEA${RG(NihBn zr%vmt06hRop{lvimYZ^jH){BfNH@gntI7w~cEae+4!z-5>r_NH zhoXrE9%b7<6+1oz%gou?cQw{;(!+FFzo2O&yk9B z>yQne2HyLgJ&eVv$7tiPSL8C}9ByY2=>*a}7zM5i(YPD^sB~{VMkq3@eYQ=zr$|9O z61n^JHcS)L&uUI6s@tdNKT&L2CIa}0)Piavf?O%KJ;`nQLDc;Wey#I5m3sJC{vC{s zB#($GDj^o%;`5@HBfOs*t@`m^%6{?cv%a3`V*`j=Pi zPAT1>+txE}*C7jw^;Dn1Yt!`OV+~`jw+#RC&OV;0U3!N~H3@y=0~!JoS7EuXfp)#^ z_Tt2dn@z{(d&I`P&!zR{~UHM)(zv*t1%?l)3?;i7-~Lw4E;Up23seHIoZ;Dg0x zE5L*);EYpoa5u4UVN|{5Q+?Ro1 z9olW}{_4g@GRi2JaP!N3i6@WN^TURQh9tXkG85O$fNnrbI7b!B-iT`Rth%AaO z7OqCfkgJWXLI)A!4!MPdbU{dLSf*!!vin!giSgd!3HXuo=8Vcc)%kK1L(iS_rT@Gn zdPNpyW`OtP;s-XEed-LJ_yIMr7syMag@#|nXnd9Rg9C>?twP`?Q(~Q4ir4p-}Pt?)%fgC~?x|>(Fa<)4Yam4LO^CitAFTd9=Ej5)z zrmq1DfjMP6Go(?>G;(~p@E;YUrpa7At$o z$qsuZNl7sD8mw+#XQ4?bJc}z#?;2;Ts#qm_F+4n-Xaq95i3zMkZpV$Hv&2`K={V;~ zcCk1Yb5)wGX!iM&JO>pE(5X}uQqMjJt$q3sk;=@U zFuf90`}M3_MW<(!=1GH2XqMQ}$}|}~DO5WC)G;wc&3y>1N6x5w33_yHeW|s9PyPqm z7ivSp>ZGayy(C%SYA4C=!Zs z?o_L>T#7x+_2h~l6i0xBLoe0JHy?ekJiHM%;Nj5^cN|~z7kY12Y|W}Nxs@}{re6x! zM_5ZXSUL(jw||@&8Nr$^QBVs=9*1XX_cwgj`9&vNri`;C+CMH-xOyTx@g>bm<&hon zD(kAIcdus;ilJo;f3@w{e3ijW=8}4%NgK+~!Bt2VGbjD@Q zUlI|y_b4ix%pb?hCq~}j5Z=!yjT;|x6f>h3le965`n7aawE4;6)7d!!rb`_;e_!0X z9uFY(HBfE;T%G+ry~8A>I9 zk@UxtqYrkLh4R;(T@i&_;jYI1`Pb*+l+#sH?wyNS4Z(zT*Nelg#l#N;B5rg-Q$U(o8hdAXE5ANJwNF_%iCxYV-;v$;! zScaIWx2>)E0po+rG3S)Rw;$?zpFA4%cnA6FBylP#{t=``U4U@DjAOivV)27wWQGW! zKY1M!6C?9-T3u~vXUE|~(Ym#@b$)w$dqLUK($d75)hCMIp(`NR;{x3gA1*|ZkdOdw z1R#Q~ACNG;F9UYtDl03^E5GlYUq&?fgaAbwiTJn?2LJ&9{gSe?v$O0psbaZ7Sw4VS zTs}OkdD}BLJ{qrMnc@f#djTW4@`LdLg|EH6y*oQQeDw_tz$$(b9rd5GE}ASp_Q{9{ z;zRb@_q(^5ZLF|w=Yup#;I=en*k@(^+}biE$w!9#T%f>8{0`A-y!M?5Ncj(V>CAs_M;7*P z$dZ`c-%zHI_P-xH^Z6wa2rFu$r26EfgbKlJh7_L=yjIEyZ%U_(yI#qSEDZvgivUl( zz#|4b{|i2A8ixEuF_1Btzt8%B$iX-Wq?wyI)jO>!ikHab zt1b1?T$INR#3k5j+AOe({Jyhy?r%~43HSOPzvW|N*0O8`zFhu}yk$ceE>CwMAb^s3 zas1m)zcE-ZVS5wmp)AN4F`C6EJ6~SnCd=wm+`g3Je<4;O|0fbKrt|+7a_eja5Rzfq zW$YEi$9Y!eUy$KHwP`H34-~}pmw?Eu_n64NPZUK(3DAGF$RQDPgI`&1!YRY$1ttpr z+$A3D?-4+M`FdR-&HgnKeE@0kV(LYW|6I`+3H`eTzeDx@3>q+2FNNw+q_@JED^fLL z)zITjXauMA_oQf_u&>|uOkbihs_U`q8>M8;ooHab5afwIta5}9(fF$VlWPEf2&&*2 zb47G`K(c@MN9ow8@Z{Kv_hisA-4LelI<#0R<(s?)i|aG`+w7`&q066g+t2D(E+BYa$a8E_*vv(89+Jxy!Jp^D5ED-XUTF3 zFdVDSlai8tKj`d~pIB$VsE*(t#N~Gr6BAl2#2SDGmL5WY_@PvyZE`ZgEoOzQ>$!r!pr+&lKS)Fd25`co}HZy4-7=@?e7Obg(_-mlh-t>^iFO1 z=I4#Z;qWk_+f8W)FB^IbwAd^g8ZI9GCdFn6U|%DrIp-}>g#XIQ3ZI}L*=C=`FfIIW zU*^rw#KbLkzPoov`J6mFQo9!H*y0olK7E2Ne!z6*ZGFbG7#0yxeM)r^(&s~T3Bb#I z67HSCFCs#**|)d1=S_-$Pfq4;Z*RNwtrs`c)X0M=5m*71AFZsfug_Q71Hf89+eTM+ zn6?CJ_MNI#UYw?QVtIfhrvwi(0eZvKYutiJ-r$pxv;Px82{)+5_1*Ty##cTk8=E&E79j-}7Z)_PqLDS_ zAzR$|nGE(tReq19YRniBR4zrAP?%qR{uG<9;S zUUpYhR9r_*gzb;)V}1AB2eUb%XzfWJc!Cf0>ZGV* zvl_9_?DVfWj<_*S;aCm5uWWJ{=7DU2#Iz3Y@tEFb3FgEQvY2z++=43PV7SQ(vFiGf zZQ=F5-KQ^sp&1~6)Xv4=x+U7kSH{OC|HEv=h}B2NDX_|z;5h#%EE}e(0kR*aiZ@Bx z)E9%*$i3UHAEOvSah?`$ijCZ>RokwsYRgjsPog!*%Cq23q8+ z><`VwbN*oUk$tmobJNzmiE?^>G=KJ;;UWUT>FVyT_;Nzvp;>OP>7B~JmD(2`o{vYG zn_o%aIpU+HL;yVBG$Tp;#37HJPN72(vuv7%CL|LG9adZ6At!2X9aE=u3gS^MnIAZN^q)uOGP!MpWK`2hZz-I|`>U zgkMwQEXh zE?o5DMcrxOzs2Oe_~VCkR@(Nq{b7ytY(xzc`#M{}{=V~W@q%5Fhqrgp^rzC4ns#8y zF13*@P2@HD&VT?34|b06#of7OvqwiqY$9;1=C3`XA|l0qN#>tEhR;Swv+ufjQm)CV zJ!|VCKorxHRGlp^v)Ut;mhzLKDZNS`qW8A8qHAOurblP{_6o$97`nDkPKx(Ncg)4a z#l-^xq3XoB^I3lhr7`%o@be9Y-Gl@M-#j-Hp(@(s^@rM06zxXL8~5#ssPtE|BX7SkP zd6oVBJwx(4dQ#Ky2RZ6ek#msG%w+`p_CvZ8g0u&8A7TQ8?^|k(z~S)7qsht1C;gPv z)aQbu%MOE6tE(?Px%_x_0i~k-D)3ll5vgpQou!WegzZAFh9{1nCiD8lWGpN!3KjwQ zwau5bx?Af0F?xSq0V!(u;n5KxcmdFy?{mY5_LY>CWu7xT=95>WhNRfg}PJC~M~t*j6X9$+^-S$iddwJ4fA@UCP{XF#d>)|!;|7$ zAUBdCuBxssH7$}rA1og{ZtN2&>FFv}=$q6%2nsnH=JQh!wtAGndeI1OHbtZk2{-rk zMRU%YSN6AkR{z}Gyui)VGrYD3EQ>~yQ&R?2m6e)LLK_te!_E47dRpAn^M?wksHrWS zNpt(e!AgR99l^oLnP)f%m=#ujzi58~$O+)`c0g@b&j`;BQ4wfJTX*`z+Ng1p*~!sy z_w)ql;J;Q^InGe0-HVEfcFo$nibtEFDle%CSt7TjC6V!qqO9u zGWc7suTXkeGd2aNi6{jo%zQtDt%8talY2{u3-M}z_|vn*lVK`dh>GZ(DvT8`Ty)+Zk?mRtza@7xHF zuP>?!mD{i^D%1Np61vBe<~I*I#flsa*PM>azJc6~$mBGY{LSghSuXK+eEx>+)7EbZ z7A0==2102n-Lp12?gCVi;!{gr4P{>9%0nMl7F^HjREqxRK37_pMu5(lF*jLkLifU> zqf{lPNUGXN#6(q-=)IDG=fem4mmCG>S~j4t%g}#6=y@EUaz@_DFyNTAE7ytrII%Y2 zR%cL_87V8ivU{{3EFZ14FPx?l?!A98SPUP;U*@x1$M@rySgQ`3hlOSR;I%;N#mK{FXtvMl=|!0sBv5CtMFRVkF!7(-^zWV989*+{<#KMw z{GZw)@A{`{|0PU7o|<|5M@-0XK%&33*UNAJroZU>LqQ%3*d<*i&CFjH9>iQ-TLMWl zBtZASyra?RCPM2kK|=l|m-shb<)6)Ix{UO%V$KKJ|z zW2lm9mb#=rC!|L#WrkxgUNe{Lk(Lm0|P(%{W(xbM=4KI5W& z+@wOe^#Wg#)mOpSvFd!zfTUW;A*y5n>Om@-XgfS{{X+C0G2~+3clloaQRXjYqhwmH zBa+l>%32n7&h$?Q5k}kgdWfmITAs8Y2SOv;ks8aNf#>qkhS|}}O{ACf|4N5}Ec!%o zXryOfS9@tFlc+zS0F?4>D%yQ!X^ggw1 zxQx2TCxbHY+6dc*!op;aX{(QqKw1|}pL3}AwrGF*uUgJ6kw}j2wxu(350A2yS^$k_ zKH*e{G4J6O$1elBgZs(`V_CCN#fiT+sN35OFi-{;H zOwtN{laBQ{13NxPEjBi%s!Wbl6aaGxg71d zV=Es0QeZ_?R5VE}fUYWW7o)4cuTRO9;@9~{91(3kSY4eGUtf23=tFJo#klzR`*x0w z3p+q>+y*d_D1%ZD9>gKihMc^-4CXzqmN0+4o6zkFb|l=9xsCN0MN!xj=!G1cNFFd4-Yp`SC1+g-F~8@b0AQ2e{g7L=fgvN{naOdi*dL?Fsa`p zV1L+Mkk!Tq57yS!ef-!nQq11VtMoaPZF`iJu(OQN`9nTn5F%jXGT|kcVLM!#i^#fW zo+_ByAp=rRyp-{p^J{1d#qQRR1 z1kb%aJYFs#mY2&5;(_MD@RCedsvud*>KB85K&IKV?0E+V2ed^E2C_KN0}66HM)LQ0 zEYw0idW^(Bry7`?EU}aMf%VBuJpP4+tLyg;S(UUbJn|&@rg)+4GfYto3=H|jhsmPQ zc)Hv2KhzmAnwzEAvA>Ou(j#cVAF{@Ojfs=Gzb-&*qP2rNbf0>Pa#J=j_K`w9 zP~b~g_Au+e(D>;pzV|sB=p|WeH85qG8M$!NI$107Z()B zy9*3h6PwGe>t@+!S85m-SdFkdlz1^!J$#8>1|08|>0h0zi;5K0JyuTuIg8tqwCd*O z<`>!a3&VgdCHX7}I^9Cuzx|sX>vDZVisEgfrU{ljrNC_skr(-5c{+6N8U&g;2rL(6etTemj+S7fr`kfTe-su`9o8UjXnvzru4bQ-)j1Y#h9@i4$}cXj}Ezv zg+sv2(5!1oduPs41o>Hg>8?&u^ZNSw2mN`<-@Z*vovr6|Dd5e4KhVa@;f-u9n7z$d zn&jRQ)0+dV zpRTTkP0RaXMN9w4U<34sU!|mJqg@ax@B()!?_~^fEO|c*wNE#4vZYOD_M^1%$yHQT z-f)Ge`Eo? zv26*ZQ|xfEEuGtqZL2%>r(Xx5y8nPy*lvh%jQi?($+<9rTb=|>$Ol3)*@SsHK8D?# zzDm_6b+<&szq?*M7lHf>3JPLDfREot<4Xa>U%g}$w`pC)DO-V}sk# zAN56$XaV}!L56)qFjR)-%GtDK#B>r&lQFx4xc>N7u&-#+O=`3#A$(QdU(dB(=stPE zHHYUeVrx3R#i0wsAYtH=&99Ag9R-N~`SQa{s?t%z`l#j!W7rtiFqDpJGp2>}T zhz^DAGqe)pOMnQ0*4nA5+OlmBeh@JkHaR{$En6(9Mvhq8_%37mR7v}V?^3#1p#R9} z6r^7IgZwq^hY!nEK;FBOCFcBS*LGe)&Z_j>Eg@$5ju5oieP**BAgYA$w$pWbq=QBqQ>e=4vS z9dreWi^;RqqI%1pr6DYTkqn~7T)SSct6<8x*|H1$!~m{cy#VKZrn2h7#6zj1YDJq zrd(Ingx7p~R}4JIylkQN@Cxg*da?-md2;R$%Y(;#qF0{5rGlWjR7}!TM9##{(%`U_ zMSfl!Kb{)ue*^IaUDRK1&A%ff*J??_l1G!w?+_&qya5xNLo*i+vwjf_1-_~4Oo%sE z?SMIxs=#p@M21<=&H#wQf+|CQqLnMI&kGyq5HyE5Bwq?sgbhM0;>F}29&XH{V)38! ze3Izu**(h5%`Lvh7!?qkP+-vjm}Rj9OQw7b+}u_gRBjOl&>a&GU=v^pnW02`-CS4x z2#eXFXh~*Ozb(4u8SWGNgTl??%RZ5-nVpud1j_uuC+Bry(3<~g&shTnH8p)6H%U=d z#Ll%5$@@R0;p8LONcf`cysaeLdr>qfgzed(P{$rpam|&yyt-im6NdiA)>ibYF!yH+ zL&G}Rbcl`VVbe&U6!4tItxb2ODw4U?)E8O9Vqu*(bX@DSJ2 zU0=9Kd#v2=DjrS3&&skHVY)L0xGLxRC^wA7GT1L>%8N-eC95k1Y^P*~}`vo~=-K+Gq9R=*fTs<{!S5ZlP!f`6C>H zmvT@RU}M&(c_ltaWO(qxeS2>&e!HM&=&G~d*K6$u%_0xvFt=kvL{6+CHf25A`UTOj z>^QGv+G9AsQ>dI?mwZ6mr;Zh5FIg1_Pl=apw=en@Vq~<1tCb+1!cbWKL-2fGse|4( zn6N^QF)?rVh9pZw!VDf_eO*Yqi8V|ZE<4YRb{5V_pOay~GFN|Iu$Orr%S%j+O z+ZhUgI@;eo{^K)ZMrFgh$8v1(*DtoU_Yqg{({t<1YV;R93X^}N6T;2?c(BYE-bN{~ z<&G=dqem05i*Uy}FizFH@r~ z&iECjbRiv0SEX7A(Cx9W^X(g#eBbswHWh)r>NRby$p(|RlSh+sjlkHFiRgr=KuYhJ zv!xtILz2)(i=nHYxQMa;vq@-n@3we|4KwNb0i$r?8(ef}%!pVCya!L8xAGHv=$4I| z;1-^$p$8lT+Gra=3h*g`9;?|e|Bc3vQAAab?nY0bg$0YbKFT$$&JVyoO8A)-)0=@| z0F7~6LYA!@?MB$UK`FH?e?FaK-~Egb_S$ z3QFwa4Zv0)x1aH4(L+@LuwYfCMeZpZ1H=Is!U6|83Yx>^NGL5u&bv`d!d13r^* zbImr-y8h)+Rq8;qPEpr*`_7yA*iYdRHECuPd)&9b0dxKKZKW?ahX}*tEyIRBS|4p* z+fEG=KXjizbm?1j>ZZsmzP8d~Ur|bN%!!3_gSjvAG`^(bqE@el6Y{fV}IfXTGw9#%DD`fdhd_nTYphQKn-@3mIkX!? zzUo4L-UeTIz*k&nh>g{=SCiKhyw}44F`MyUm?wuK6e45 zEGi8?s^r(lU%g4CKNzpDy+tL=Q5E^ZCt(+@yKJj81^80m>p~1aLdM>j-oNZ6WI-=U zAWk?CyrMtd5zz@Ttc1LO1eqz|xHMxm^0*#@hnimTnM7&S)NM+eEcR;8D4*4-f}!>1gP-XZHSj_>Jlr`faU|F zM-c8IQPG0AP1L6ceUlHNPlw$ z$MQ5P(x4pC37T;!ECbRK{@Ffie`vd{{z@(0@#p+=!DFkc5q7%Q8VX1ImlC*qG>Awz z#>!m{pBJ<`w}^k)=2!7vODHl1Y-~l;J*8{dhqq|GNzRBAa^#FN2g;JX z2%_b;GRO~O&P_QQ8q%%PT1A`roB0pY_U9g?x~JQxAE(ZzyZ3FoQt5vLFgk)|p`kdy z68~zZ4{S|uaPVAb1yHj{rwkHkC$+%L{UP1?;V9hti|^UdahhFagphcT>)b)Z4ai$m zVo0MS7}S1HGyt)Kq_jo;@6s~6zmDc)cpU-1F~hSIK7Z~6@xfDrKt@3jN_zb0!UE$L zNh?k3=rpLs^?P?|gs*FyM)A-}yU~Dsk0P+)p4)Iwbf%tgG$=ECn_(5f4 zWTI12*f%#fpNXd8;NaZp9v&M@Ed5wte|w}NGn4jlb$K~0IyyQb0RgR-m)9217BiaI zK==b8CFN~jU*B34Ik_MP28QPwb2BrImXjo;q`9S~DjFIZ41$7VFQup@OifL@PEJnJ z)-63fAOBiesq#=$dlMfS92~W^We1pOw|?5{>FP$cwcVFjR{mCd6%Fm@XEt_rM~8W? ztIj);FANOQS&PEiJ9Y z#?S{UvIc5udRKvjEjvG6wBGT<-o8G0Jq7LRw_Ik9lr^S_^Tth5?9ysQ%&sl+hJfo32X`3UwM5zOBBI;Lx-L=%+!67FT?=j-OmR9WQ+S;T}83??A zcIW5jn%gYoZPC-yTTM@Zga?zSPrneg%QY)o?d6sM9{1X72#@1EWsrmKy`Tddo1P|J z*==c&UjF&h=22u#jgW;??gad%4v0$m<_Tl9xF>OQ<|}V{_D~&2MAW^4Ut*-G8I$DL z?jP%rl-tmdG1EuH!p=VAWMOFu;s}v3ehF=xPj84Yz_JncQ%RDyN9tqo&}wRHdAl9o z1^_Z<{*~~lyW)d;N^jcPvQh1ec|o8c=fo{aO2-qhmI_N;fy~sRLk!!%%M?ifAJBd7 z#Rbd$LX_|}`i%tgscnd_t=ji0Jb_W6L5 zf-`}2P)-uf2+h@c&tvTgMfusTICgJhX9qF9Ht-_3ShDQ53P=`INW%&A7|A1&;^O6R zDI~H^5OA;4EWV))f{lGJ*Zk={*Mc?YC!|OiO)Nt~TkVKi$|EnNZ}wdPG>Cn85lktQ z#Lh6Qn(EpITPxNIRps7`1Lp1uAC~zM#N$#{eekf2WWelpQ!=S4Z+pRGBfYVI1c$M0 zE`q0xEEH<05e$WoE}sP-7raaX1e5Haez}CVpW2sD@L7)YnbbWf<|wbTTj+Y$9e=wO z%+ev*vXLa7o}LRjp!?z0@%!{IK3Po)x3PZQ-`k5;j&q-Mcw)Sp$&s)pA5WuTtx1@q zBmuoEL~a)pqHe9U;Mz@xgvcrPZR9(m3P8BtQxKRWU1vf~or`D45@3 z1fNT{`lT+X6o!UE$Ce#PJbHQl_>T=}r8d8Y3&!mf zqXl6kcvK=mE!l#(9~69704_$RsM<)VC9oUzhd}F`KpqhLhxN%%&UeHL7H{>QgZx2b zzh|6JITqPBm4GcTT_}#+AE2xCo;@$}TF?gxuW1w4HHenjjH#cUnf0a4TC=9I?_JTC z0_Y27@QTy$TXz`-x#Rb4|0Md63V_z)%0Ki14sAZXZQA>bc{@7hq(?*XJg#5G3vb)! zP%cFUo+p>htVeA=;)(|y2E@mKNR+2V_#by-v&t;QMm=sbsf&t=ngjSRjdT>xd;tuL zxzZW+!_=*o7W|iN-K<Lt(`!vin9ETwOQAOkw0qK0 z`vuSj`vq+QPI8sw!VKf|Mv$V;IFK{fmoekvSmFJH7z-MQi-%VzS`jf(>TlpsungU(F6TN+FdOIsMTaKcF?(?f=OI~HU4r_{*`|P2kZJ~7%B>@9 zu(*Z8rFU*%64n6W2Yq!?V*(}{m{xo%rEl2R^lb+MQx@9&$uKTP4$ zlJc$?*7}lg{sIbcj4I9nY+c8eCWcA=hXxypo_Z@QQcnGj5GYu+xQM6gT zL#5%8tF%moyyUb>T5-t$3^}hXq5-K&4XV5|9@UAGiw>d8FjJEGvS4GbJiHLD=z zxoY_>&}FiY(afBq6g5I-DaGikj&WWmc!L<{eXA9c1GYB+n9H1m>z(jcR4XjuSr_~w z4O`haj{}v&MEDvUSG4h6`R-PmpA+_W*fsB#^++4H%sMlqg~76WBxc75Dg_%_bmJ8D zkL3Wg?|@@ZVruq)$)Fxr!hgoDw*fqA#;-%;+iP^X55B=SRV(o1(C9v~O@PNF+23QL zP%xuC3S+ZW)_;#FH37R$(C%Cpqhsl2HSPuA%UWUv zHEW`kbZyPfP7Yg=?=5F*!DscI84$SFuJIE9APG?&Mr~P^gw{fZXODgCBLriynKN4k zfo9tIVVK`GaM0SEno7jt7*K(%EGoH{Y9$jCPI+k7zz*sk@F0J~|4<=|v zsBKB!oQmF&B3)S9b^Urb{?t3RGmXAGg7$L}eYTS;gg+>|;^g54Kc#d_2FU&Yk=7G9 z?Kbd7E-u3)T==g@5us2F}N_@+wN6Qrri7wf-$tMlXu z_5QUM^y8VS5cF&R^QPM=9UgAyx2DAhL+zLPdc!mA9!5)}usz|oo)Y2h_IJtixRIL{ z#4maMdJ5fUUNB55K5B>T1vwQ}#GDF;_cvnRD5Y4K)P1^E#(-g7qqZ>pn#xja5lm@j z)pV_R(o7A7Ac{$RXFGC7Eh;wVkzp(rZL>y-F-ouvkuRaNp-#a_*S}`xA+dcQH-V*}S&^%9_et#E)!!>c==8k4h30 z!Mh9koy56^p%bRbHNGLvX=!+`o&FVGbPqr+`bI_090YZoFE*&x+wcFrRsciokZh3# z`5NVx69VJ!1+#b8)&EfgKzX51Nw{vEi|vA7VG@_X%P}=D>ulwEKbK`?uj(|;rR~Nk zXOfE;=n|`_cfdYP@e9g1jS{Xay}d-kjG$p*jerd=d$8*AJ)P?t%vHQ0`$D}=1NJuU zU}u?V(>B0fA~+vRbPD*Axa38!S8ua)uovF`8prHXJuRTj3iXfGf~yA?76KN`^t~Z0 z<2>v+yd}=c>kPMY$ba0F=X+K>hi9~ylL>bTQYLg}I0pDqHJ`NOWNZb`{k2K3v9Sfp zEAWGXFDqyabNTiM{ET-&erYm|7Uvce+yu<1hXrE+86dp|53v)*)F zKgDq$E^B3I=hYf$+R!JqpL#sDY!Hq;q%{(<-f$R4C6L>6sX8$lF>deFyN|FTZd@Sv2%C>0fS=m^Ybs74{vD%(Q-ea+10{# z21fs=G#nFR`Yaq;%<>X1VBr$T`7yWtLMywxoaetXG0hBy=Hz}G$1-QMtORe5_UXpP zhD4kD?ynoyr{9;FLpcdV_z^%!x(b5x2Fu00PGGweUdF}@9nkFT?C|qq-nP3E;b89a zwk8K-m|Uvzu=6%pdCfO>S04V7`7|sMM4$QiRwjl})@lkCmmIooGE)}Z*fXwBcI}NG z%Sy7S``*`v*2UhGB7&zRxAkH}pdYO!HH6Gt3YRnA#18L)1$3JcK|_pAZf0i}y%h^| zrrvjJ@wXpoP2H8FQ^)SC0jYtp2EXR;qE&7`h}V#bqlr#(o&VC>?(}P944vBD?Z;^2>o@#lPL%a_8Glx4~%K2xZG6VU03Zj zZ5NRKSr7!pbS|8q^CppbXKgD>h*#%%Tn)dfx^b9k$W4Z&aW@Yj6$I}5^&eORoyq7a^f-}3_0<5Eoppny~0IF`G3TxG(Cj3eksXC|N{hn|k zLh@kyEIa)p3|CZEsS50SN~|e1Vc!;;0&oiHMPswt?yP6)DPCmR;I<15iwt9nBpV*D%5g$+M zx}0Ym2%sBgBY?Jo8eC0(uSLVFe+!yq!Ew^}q|T*$q(z$O0d>obF1?rI9}3DggPJP6 zPTh8C{a8(g%lBrS%YW%4Xox{-6JYv$A#$2+6CG4F!Ii%&vY8;$6ji5*>ML6zT}Bfo7gBILKZRrvG3L53F7c8s7=0iX4YV*(9X zh;rK5mcAsv-^|c#JGgqG;H_UMct+3n?#H*Lq0=B5!B8bI;I$1*Sx*as)_#e27ahzi zA}XteW0e(H3zYEJJa4Ytb4YcWCD^=T{<@l6Bs1{Gpf+T@!MlOMug2#mMTmgb{X9uO ziTHvLsD$2oByjaMquur8o>*-TBIlstp(xlL9v<}@&V~Rk&ivNkP0Ptz{+PNk?;lj- z*TMi0wVa#$0!bE|Ji`@vww|Jwsw^&p#c?1eqJ-f9UgsK9t1+jZlM*qjZ{)tiWfBK+ zV#bZZnp5~ZW-Y%jpb^A1ymCMtPXM}-Ke1(@qQ>~9*F&_7#<4N7=!``)kxfPFEK~HF zp!0+|KnzQ4;x{S|O_#fa>9Vk-z{G)dBnR&_=E@2&DS*X}D-t+s^_|@{&;Y7X7*Ag2 zo4pQ{$kw!TLZ?`&q9+HU%5Dov>9mC5<0f->QO}R4R6Xy@%6=2D`T3Q0L?=^})p1Dz zK6}%@OQ+fXl=iU&-u3I-Z`9_+Q~sJpPP(Tk=b7-(AjAt(6oGftY%$!hZ@T4hqgzHd zM2E(qJd+urIlQ47lGVavniX7kkkSE&Q74gjP)a!}A z4VT+bvXW@^L~hTuX1!CBj}ujsnoGllH_ulb_me{$#tqCvh|eCV%kapsM-vYzJ#G2K zTKqhvthyw*bXh+{GNfT#tfYM41)v~ypVs^_?5;`k;O*wm{ugzQCte2UCPzK4&c$u+ zO9MK~_BA&U4gl=-;c4lId&AjA@9a6XU76HYgzsL3YPX-P9Mr#l5;7gK5#o}!Z@X!W z32gQo+=ocIgCiGhqc8;e(I?okPXa|*oOSIILTt`O#LJ*;^n009kuWu0U|;6(E5sKP zai%Y+=#a(RkoU-V#Yl*T%A$el;wJV>X=OachC?e8_+Za?%e zIN3!a|GOk|z^`zdUGZ_nfq+zH_!Xb)vk|xK|G9nL*s%tLVgGij1Om5&50?{S>?(xJ z-A;o{|x$w%-0RI7SWdUCkaMS#80Y=tC?@Q9W z&SE)wII96qzE$}?x$EBh5l;VA8d@B+y!TbO1F3U93*5c8#7NO>lnhQqf7j@}-_Lp+ zq?hiWV#zpOl>BcEPX2_C&m`6X-{fUw4bcbmi2kQh+&Zxu2CTWNn3d=0ld7WYvlX6> zULsd~u0%{YMw(tmZX4c=vL`{_IkTQ-wZ9;Ka_QwuRICFIQewzcGM}vab5D`L`jC6= zTC6`;wdp*4KRWkpn3tMO;WHhdtk!4Ds)(vU4E-$2y1Bf5`v^f>K-VQ&u3qaixqQwW zg<#B!yajDc=hSYLi|a13WWj(KKAerP*S)+oN5gIXqR|`#2;36>%O5)~cgGyc;$Q0?ni@Tu6_;20-zb4i{r~gS;_^WjnW&Q1j)^S24UIi&#Psa}I zt@x=xs(KI#0*tsRA!y)&HFR-?WV1t(f^#7p!Pp!J>n1vyBBy5nlIBx z|GX+7k2uS6V(g^=Pewm|cLiBwyDH!V;UEd^*cQ=Hptk1*gdk&UPO2*TLo?USr?_&5 zCt9;uu8NahRN_Bf;bkM1VN`D*K9~9gB-v6EZ1k8`PEG6`pCWH6e^2WHGRGbqh}ATCMRsN4*A3nai&k40 zaLf=tx-S9I;Vg6hS$-@8H&U0d0tMNBHC?f7A-Ewa8NMB~{UL69lkKz#SLb4nu5x`u}Ct*tF)5E+C!&6#fF;)3Z>6LM!xC+6_b6=c?` zx5bC~3{w3}k$l`gG4TlaPf-r0^IhlfUW;UY^5{_v$S&kcurxA~;^gGCfg2Vdlbu-2 z=T-IfshC??k)$SksH(zhi%y7-_p4I2<&g4lZ{6LkqCv*5*J&?a!Nr;*x(iuHAi9Qz zV#FiqU!Sk#!=!8jkzy5p1ozc{^+3Ww0qX^bx#t`5Yi*SYfijG>QZ|3Qb|Ll6h8%7| zK^Blh&RI}Y^r~@tg6U)2R(6EsrswR1UO_AM^`bMSqJ937IiDW^voQw}n z3?jnAGbxgnF|J~4num+g<-x;CC>wbQNL-5b%Qa~d~_bT+9Q2&P>qzQDMuU9snUBD=DFH#-wHf5$Mz z0t-tp>06rY{rj&?sTdg|A*kt`uyhBz&^ego>dW1yvyLay-dTLPSeaU1u`Mis#<^@AM8cern_KNCoz^bpH=Cd51^sT+ZhUUn($=;^4(&e!SqzmFJNe!{J-y^95u3ZA z+Em;>syKVztD=+bxUKuxC}G%Wtd+xFdDv*AB_^@}uMFX>TcLJy1ov4cKBmLxP_CGI zX=}#=k3S~oO||qNKa9W7U}$r=xw;Oxe+(@2^0fp)XU_iJd9LFUsOC-?Bgx0%-(B)Fd~c5QJv1>?V$m+BN!t``7Z z=m-N%bamQXm*-$)dYWsDbex5aP2sD`w{PDP555Fc%wkkyAxA69m*L^qdFkugIBLp* zrNzZFBDxmQtNE|CjdFVupJRorzXb8#$SFrE6dGhgkw9#-Cd|u=qAtrwF~?68P6+ly z)$GwG+`G3#lGXS9ofH*AnO%*PRII0TRBiDYF{8OX51WN~Q{34Y$23gUwl01qGBVvx5CYY= z4V=S6*4QfyhAMPVQO*Z4A4P1)o22!;L;Q(n4oM5ngW8k-hpzVkhpTJbKxg#cqXbcd zB)Slc-iZ>C1kqa%ZPd|?PRJlih+YyAiRc-iAf4GB>jg74~g`u~zQ}?}- zi%aAgm{PGXaLd>xZVgDB+Z)E-f0{MBo`gh-Kb7PyiizRTRI-yE`~q-D+RJtd$3F)) z7xL=@0G5F&3{EI(XrH%|g+=Z>^Q})C9~FVh#?yRa52#6%e;4ig1NfXW$Dj7ov;7l!^V!Kq+)GBDbbutZ(&HwNz0lRbKA6;GCFgd<6*VD8drH-JjGkp zQZ@gdUnEX&ZztD3(?4v~?8{2K;a^@@_o=S7_T|PWU|XuH+>xV{ll}EJuU(^#-`rYB zb0!u{qYw*|3PH>;gM>iQOCXfQCe;9!qr>iiDhu;0u$?`A($T>Z#B}n$3TcvY%$fiM zGUxkbFQqd4fVdKQaKtXV@6UTLGxwVx^j3_Fj6KjNU(j?8RSr7ee|ulcaqk!eYagVW z6b*K!yZH8k&>(+P*8hlN?|Zv7v`2k1!K9hF*<2ZQ2U^&i`54H!OD00)HiV03x{Zd) zYjz*{G_A`teizq;;^zqBNw6Dl1AF_4A6?>c$1U|(mG-C;qqLft`+&{0GEMOm%_wm^ zrMr{MlP*o?2iw^}7d3=P?A4vDwrvs=7_Z@Q#t-3gcffmi^Cr5tcEgGUW4Dx!Qa$EB zV^_R-vc~(tyRXQE2Naj!30jRPF$`U}D_vp5vg~3~@kqy&q)%C?mXcb$EcZ?e0Ghlw z{puvH#Ki2ZKR#5#UZjv+uXjtFTb6-BEXd9Mrip{j)u3rL#+`Y8wCf;Xzi>C-39!EZQ;?By>!f|liCidKXOmBU6z4~Nmaop8L zYOfLVpm4U_T|J#pcV7$-mKX5EW$Fx*;Fa5y`4rk=pkkI@fjhY{_d1c?Am7ZFo5KiH zB4H0vSoP$*h%v{0Jw{t?B4#<SEb_4d6m( zStE2WhA6R`88>OXNGgw>VIi(1bJ}x%b!6aov9P02U-Jzk13o+$Ag;7HD*^5JUM>%r z>N+sW^23GN{y9hpg-q2e9>|%PdE-C-ry+=h#Wt9ysv()7Yfh>b>$YSMJhi5J-mM{D zke|>u)v7Fd&!cJ1J7yO@vAfj1t!K}W)M+EY3%WdqJ>(9GdDD;3>LCso*qoW5g*5h8 z_w+7jb#Xgo_njBYk?#+6Rd$Zv7>wDXU>aKxk0!gz$87^qS$(b7abD|hajRV-;7`|$ zr4WC0vdV0Cbl`5!gN(_R;KF6x)eLgpJFh@#tQ zMD^T5zbWxZC%3)CBD=b35)2?^p;UJWaJ)dyl)k;}J4d&K&NDu6#(~Qv4W&tV9hfSK z=+0t-W}2Dza|K=?fG>ZOvAdAK)ho3cgd!r##rdgXo4&{|LrBB?t%_Ul=5QpOAExt9 zdfd~`=C7ceEVAkne;O#JZy0J~wfUjiP5;5UClDvT*|jMl?KipzsLJyP>5cok=Jd;q z*gtNRA@xHpOlH;f74}8kH++&g8qsw0JLW>XE*tAcz&n6{@yqnm1m_lJ-ipBG@|U}?u9SM|3MCc4k;tE)Jwo{su} zU>ELt)*mk$Pt2}OX33E521+Q(Y@zZJ9ZIhGf-i@`KkF$}^=6{eyO{6uzZwIE%YNj{ z1z(FO@ng1-Ap z%TcZR%U^U4{A9bq{+tsltul(bjsWLAEF0)()#!;)X|RC?5Rus%XvZ7VeQ>hM^O{fY zvG*T92d1}g6(B;LY4ZWYqS~Y@m{%*9v&(%}E$_A(KuUUr8Osv_a`8AXwmVqzJsho{ zvE;6^-@QL26Tc&R#8J%XU~kx%AWiG}O$VF8;N}-dWhV~T4I7C9qpi9h1JbsU!^-0- zONH5il3ti8m!>~VE`yhffeun8ru%g-vTYgXpGnuCW_NErH^V2{+vxD4+P13WlXC~V z89hIkD3cK7`KUhSxHQfn#tw2539#jZ7P18GZ+>RRUyAK=SjR#C>cwUB>6+)XYV04K z*eM{ZC>!dWOc1go;>7IQ6|(JUS#qnK8&8LLtJpWY)lDW~!H7_IuA8GdQv?rVFpMqY zcvAv5!-C?aWgu-cb9{KG5uTQ&JeOQV!0=Z)K|R4I2Hyqkg@oK?5`xb^pvlDV8Sx6o zXNQ<1L5NSY-s2}BO;am19o=yYCh$;TqcpknB;OCTGJ&fSngxUEUBC#L0T$G=$dJG) zU$Zrf=njUgeTxI~0v?mA$|3Rl?ejG0W`dtxjXIBuD&Ht+5ops`$s=6JDoOJWkAzJE zUz#b@9Yn@1lr$-L!-1&$i z%5`0cr-ZbB1BIvwrOxUp^J>BK@_HxdD1oHr_z>LOTBVGcz(#W1Rke$&MyNL<%A~t) zFs+yjC+p5S4os<7Dt_SzcZ=>Sx6Hmc;(@?CqzSfRo!IZPH+DGgtTUW?@ zDERtffqo^OuT>rH(cn@VeKEK%RF22cl8GYo#ycbeHqb>b#6eK*;oUgdi{- zi=_rN=*~HG-*u3_OEkie=!pJpRtoa~ouU$B2Q}1omTIoQV$)I;FbHVj^|+4(BV$2z z9_}>yAMfx_?HQ=;-{@rvf#l`FrBq@lnzM90Qk`rbN#9`p(HwYRC^h6I??C~&m!7{f zIEh^LOz@ZO05^z>eV?;W=v_w4D-pzt`{Wq!2U%7x$XSAmuW{M zc0M*0^-r^(TTj`T{O^FLADBKXQH0;T-p8=+x)H`k`x0^wth4fV$ekoDZQ|=K0 zKRR81%aNhA$&TVH4V7s3KU}1T3WygcHKeaWSTdwm;d*AL+lP!xJ=?%~^Jp%TWw?nP zQoj5k!+)y!kVIuiim41vVb%bQ=mfg1{)VtE-Yos;h%XgVo6XUiH&-=Jp@FLnuA1#% zgxD!?)zW+gMnjDizvq1B%cO&vXU*Eh75> z4<~lruh^?v#cF`%E0m{tkz+-DmTr7V){KT-Vufy+tRIF*EKSN%4sVDyU*nX!j4!x7vw8PZQ#z!r!Dm`Jc zmE1S$hpi35f_L$nM*ZdmLrmO;g``vRw^tnF zXiwghb>A7M^x}_YsGJT*lhv{2#d!_1KbHq&Rjn1r-|)UJoA2_nu9@*YKk*EP;9`=l zojtm@^CzpG_XK^wi<$*ddg=Sn4xAT+$Ju{w_*lR7#jPu7{w{lYCX#+w{I#*G%R1PD z5XrHyh>`}~x1Rw9l_s4fX`4Q9)H?_LnO{fefTBoZ5@mB9Q$wX9b&5p|Aj(LM$%|!p}Of18mXum<}(&lelBQP`aiJ9b6%}^*H7{;W*fzPNq zz~e@N-_C@bFW&T$l6>N}R3k~$%SE<@JLQQ}ctIvin?1SnN(GIO&>w%fO8N+whq|N# zK+@F!;PC~ z*BM7UdxA+;^3$ksKN5582baG&$^Ov%I3T4bk)oIZ(A_W|rL{>Bq?LBM{OgQNw#OZ; zugf=xFi_r8$Yd+e1_w6XNDsjaR`FW_Kd*|+J|mmJD(tMdWohP8tzMVGm`Lw;Bt3=?aBrR1 z!Fw9*jj9V>_hECMqnU-dv@m|-CC{fEuW31PI;OG7%cJ#2}t3mNZ;%eiun_m^6j zpBLE26P*wh9fHRm z6%rM~YMIAUDr;_MUY?K;oWuK*ohw8(_Z~M757Nc->0!5r);uZydauXf-V^x7>(($1;8+CZl7eLXG(Wp z3kNNV+zt=zjPy3U%4q{5QU!I5=hRBm7Vve`5m(dzrK+K1$X$CA9MtTK&fe|)aW^NWL$ z!`y?)i_Ew@(`~=ZP{?GveLl}gPu}a&(GhG6MS@EJT6-pf;c)m)W1dx%IERgtV@3h1 zkh!`y3J5IXV0fqF1jWEv$$|OQs9gR^1UTbOEwc*d-E15^kO^pYRk?VuQBuruGr;ml z6Ay1e(|AXM=h@&8Vainc@bD;}DG<96SacbHuen_SsVhckGJAxtY-qF>bw4ubva!Ao z1Lkrg6(ZCaw0Vz4XkwH%e#nuK(Z)^?Y7l*L+n_}E!cW-H+Adx9XdxsRgX`90el?85 z$^$JFk%$O=N`1S1`9$d14XA2+lj|x&>~o=Fn=wBujCoF{_O7w7a}#J*H3UNR@uPQ^ zQZ`tyMQ+rC2$-yNSU78^_PO=)23(vOl9n5Q)I4jJ5BqCbf*~h9PDO-YxfO#4vtUE;t&b;}BSPReykkCC`*hg~p%RAYL$=l1WR9iaw zd+#bLMC=(Wm%>je+3T0kpA{tdeySoUkxx3D;1Ksy%1{U%jQ3YL?=N)kYD=au$;r=7 z(o%3(d@w|5T>=NnJySDEZp2C`F?k2?cwm2rKu$6@e%A$s9KeA&PuJca)1VE`Kjn-q zEby4jh5J5+E&)jY#2(_3V<0b_xCC~*^Jm!|s9x+NkSB@msveIuC%50=`o z0O@zP=gbTg>dPj=l&*XTWDc!f@Xct%DQ&-8`wr zH)+)o=mQ^IbNj|ck&0&VRmE>!s8(`A3lYDu$NTSQPpfjg6A+_Qmo9L|D~)BsS4uE7 zlNTG+$eTBaNviHVd{2I75>#E5!XU5wu=1EaaH$LCL}*}Rf<`#w+}1ENiBoS8$3^A# zEHte+K_kVipoK}>e$!|-N~o=| z?3=YgAlyWXp+}WnwM8f-k`1qiR}C8J zq0RX95Hk%yP=ae7&2)DLA9Fw>CH!ukvyTC*$o!x8rGHOogJEt%b+W(7viWpHB{6UN zOo>o)7rRSuu&K*q0^g&_D4jV;&A2lZE(ft-!do#O{+1d;bQFBC`{Fs+7?(&inYp~r z>nEay6CfBN{5};bSL(Oe7=q&cVUh3AmPG(rUt!_&5G?;hbkaot2VbpJE`7}Qmy2XV z(I%><%{4)Ak^D|=N-X}nJ3f1dOcaH5s5t}mZ@63SH-&RJ&RG+x7|~?#td`8j?TYQ{ zK3xq|`=3UEWQR2?JMQ|%zCcb5<(uc)ak#L>LkNMkMdD^3>?EtdJb|Faz&#S`^!QV||liS$|?M zgjYp}k0ZHNWSD!kivVg{bYk)M-n13R*;vddb_(u2c@u!)wppspV00rM~DpSG%W0i%WG-tkFjcj4Qa)RZblh=T2 zJimu$2^nObgf|g~vAp-EEL*El>TWqh*GHqtRNS2C6WK1O+H8iz8FLaA!yh|uFm}=_2+y)6V{Jag5{@F_Tz!fZIS1HC z=$d)1gjlIEi$t)T`?S4RSGWM}LDsq+K&(xJ-WcUT**>0!?gbEiMt!(?+8~xLpAB>` zpb+a5LG;#cp|oo_If}%+h1vY|vzyH3OefPOa0t($r!h2A-h1=VykB6+y(i74 zn_X@%(?(VQ0{~L4H@<-y@KCQwl~O%46u<$m{5-V@sjONIg-Ffb*?)N-S@Gmt@b3g8 zrRqgdXH3!eka{L=mE?Cz&9t7cXyI^PQM$U-LRFNmB4hp&(lRlm;;RoQbs!sS$_Ery zPl6*UI&j-n0!Z)-eB$~ndE-11)s5hI^tay-#U$ay?iRwYC}iCDx;uGWaj%v+DHx)8 zB0>BdT*Z${gCE)6xXElKXXs2o`6DFuNgk=n1er==?7oTj1^3NQJlm2Zz(92@eWe4> zj<5(`tz8~%V{_>D7JjC~fwl6mC2<_8PXY3YkURbC)NVa!GCX#3bCk-+69kBaRcCD9^pgC-$&p z>)j_*uVcOLZtCg7*XO3 z1Ruw7v&)ssWJTSk%+RchuPP@(fe|sIxoZyCKF-*f{#%#$P}@E6==-+s1IM<=HiE@& z2Xq6FNRR{Vvg#m#)o@z^y0`i~isP!UUX;PwCCl36X#UK9G*wm&&v}m(jYo9%E)FS+ zA~vq5v|18DBeu8X4~-Oz^>6VBYdUAw_l0l8#SzjDehAyYBcBlnQ@Y)MqsZFs$bl9# z;NBIE&rTYT;)pSkUcuSwC&J{sQ@~u=L zwF>-^c5LkOWM_LSm7S=+@Jisvi(wix+3e>#U4vH{$mm|<^R|B{Az}|(QU>MQjBj!i z{hnt_S)XW}By2x@c5~bx)8V+|B#Ff4ZB6CHB|*M2 z*CP_t%)&)KLn#`rGacl?ec!b(qj}26*IO)%N+Xl*edSpH15l(`XMbc_3p#No!+hlX zprwW##@DVgS3u{dQ!@a}ee{$R4=+mNs?MLvMcs;o%u4&2E_QQI_dQB)In1)^w%zPA zWwPNUBCzaxiQOS-_2@+K5?>I$QD_o8uYuH>W72xt;Q|OFnVOl^@LOeEv}8oi7FwPq z?}Bm~0S(H1fwi9oFL*CZp|;09Z<-sUK@N*$9LFePIahV4tvuw7!_w*L6dLi@(F!9) za>>ZZ*gK>WAqwvu(!AY>a44h;GmP1@?j0JNn!f0>r4Bt+35I<8_AOwu<%QdmXJXrU zuLx8I+$Rj|a9@rmeFm0fVCt3mc@Ew>fc-D`V`8nfB{)G!78myXu|FXV4R0&xMSBQ@ zxAbOnrr}8kARx5SxoFF&9j3&p?;%jAo$p&Je{))g@!ZEY>v}@kWcs-Xim;}#+#*=wx_-c2=*oHXlgr?j{?&^yf8LMVMuun+mMTx|FMWX)j#7QSKLLD+YK_^B5d8427dR&C!SBnZDyU5jv_L;;+ivsEj6?P)Pv+tdBK=1Sg>oQ zsZcELUKuHx>=VN`@hqckRau*}YB)svrNe4^Fg{2%K{|SsC0LzE2#}2m<1@R7qMY(| zch?&-i|!ZsK8LWIZ?v^Ci&ndB6ODY6M6G(%ZI!i6%=5@t$BD@cR?%L2p*{NQAok!7 zk~X-JDCN7Q6hTr39AcOx7mORkhpaf0if%L!q5a5m;EM-gw^G62s~FLs?7wl~0h}n; zi>a1OJ}g+OBORGfFunMfe!8kcI6lr}p0*rzuD1gX%^W5P&(#t|X;JbZIk%t${I@4B zK+47A^n}@`z$ldsP&<~mE!30$0tm`nrNJ zSKDmY^Fvi1Ji4!6mZY9{Zy>HFt6otQDWZRwN{zCh6gcG6jb1h~IaJ+8536|y1T6MJ zcCK8;uM=uZtycRT8CCHj&?`{`0;Tza)w^a&&+YMG6bLR)_@9*q=%6wMK5K$_O^h(E zN3O58cXQG3A2k}=VS>mcIn{y}_E<0)Ve@IW9HQTsQ8F}2sV(-m-rrdbdj+6jj&4oN z;ZMd&ct6Ri)KRMe2iv}SO$jaZ6L^^CMMLIs z(8`%fB4{vI@S}Gm6moLtssXFIQGE~0^$A$vb78}t=dYAEjoKuR6BFhe7@L{>I%htm zF!6fjv60qOs3lB~5F-3sIh@+D()-~1g|+ARiAEnQlc1*4cImN-P|tCh#aoof#1=?w z!c%Lx#x{CrWOZ)V8VFX(z=tLa@Mu8y?(A~1x;VZV+nl*uwJhbOl74>G9*?>2#OL;hV^O<$(20$7 z^{8|GAo^Vy3W*{eprW!&3{ev{*=#9FSD!pSIuiGlvCkT>wA6dKKNA+-Bga$%tUpLB zD02(KpyQPrj$Z*%*b^1f*btDfL2QN3Q~u|6Q#{f)H?){XlkXCj5w_394?o_sErHb^ z#wXcUvdm+_qh3qXDhHr@nf}+!Z_?~4Ec$`Ol>E&_SnNXe+W2~BNYfhL1POMN@P6}oXE5V)k%J6$E#=H$VFR^xF5ah=MPkjjr1*~hVmPl<$1Udp_< zaf0Lh-qdcWUCJ|8dt5iL&tCCBsSG?NHX7W2q`huZMvh0&da60XVn@Bs)>qJw7Zq|Q zYZc{pc_yV>IkC&xe%C6vhW9+mwW*%DbH8I9Xv6Gmmj7tO%2q;wtu{6AE0FWgSuAI@ zF0g`J1Tm-3O_VNc-r@|Fm;K|58KNed_=mg-yF2y1P}PR6RP_1e5dhV*yRSu&5E0VXl9`xDt)-O` zb2e3#s{;0qXvD8+zax7G84B>LMGj-bBc!^ogvt5-0E5pOhUK5|%GKh67qWX<0lVbw zR0cD31Qa<97}>eHmx(!1vr!PS`xElbJ&I98O5)t6SFX!E>REbrpvrRVjD&^#_XXq- zZ{pmI3=L!c2f17k47E)M>*U3`C{s*3KUn)z5)ccRni7*4pKUVT*@z=|wi2_J{G-t- zZM*)nohAfBdC8GV$AgLqfD-vLHT7-HpC^?gGNYuhor=#(CLOH%!kPege%;s`ahop_ zu*r=B{YxIZ*!Gk2i^(N7KR+WtU8gwUzVw5bAM8_{-p5j?qWlU;c85SN>R3_fq05D1 zda}roYCk`}pc|NlQ}VuT45nn#rEOfPq-I-6GP74T7u_4A27rG-pz#l*FRQ5j;-f2W z10%%k0o4!$4h_H+_XnWP;51e2Dl1_kC}(4FL)cfRxB>k)KjB zGS8;YI8o7R%^rXXnwx+~Z_NcjL5XTJi}P#;5^?GmYJN6~4Cx?5(=Fn2&}tW` z=w2}15zYPSDCS^{Mo|5`-IoV1^O3Lx{c>I_= z>gpXqsn-ngt(jWk5XfZ*%XTr;cB$hU5&IYLe+6WgzXqTK)V5>KfgIg`kj($0#2pPC zs-FLgv;GIWZhp--bp-~#g4|~|K&k;5j$EGQGe=_S=qlozXnhpcu)sBfyuRz78VK-YlpYg4;EydshRhXd;R_*z2i zFBJRg80}O1m)A)s!}amifB!E?^K~6ue~gW- z|L@EH{r>+c(pBO90e4@0o^=JJXN|C6rN3Yz{qc@Fp6f^Pq@^KB?Fx>L{g=%D`aZb0 z-2V+X|4Wd6jnlrq0Qm3v8=3eb*w-iI{P}{!^%&h-U>H*Dg2EY&Y zFI4?#yvph($eb0=vM+mAo$rzUyttQ|^^R-0W^ zgL7rqKr0JM0fj{N<2GZk4{)Kkz(>xKeR}_*{r{dHrvuJ!U5m9{7qB^;ALPYSDgfUl z_>U|heh+ZzDLeq2q54;0;=e9MOJQx140)RTQzdvu$gfow1&19#Y^D-{KHp^rj0VfP2d##x$zr;Ho3ex$v7Z+jxe^j+O4pcqs1$*8b+ z7Y*cAOI*_5yxDaoPPJLU2ml}vczAeuSy@>hf$?>9W22{D3x|1ua0GzB(qv?|9UXZa z`mo6O1A^25X;)OWNRXcF8G(*icZADKee{?y08me3(Hm)9SSXlC-Y+>)R%8B9SZH;@ z#K2%*WP1t_Ztnpc&*ECU4XgwqIgBt)6eW&41r8zXojP)8c9w;doP6wk36OTIyt~*# z3~m|e7RU;BGzSg_fL#XpMW=yRtaA0xxz?Y%M!?8}aRC+?2aJ)1lFeGJ{@AibFKd7( zR4kDC2gc9)&CGW6sNu_(9>=>=SdjoBZO6p{z?#!NmCtmgrUs-u#$3f%=d>e&kJ`Qe zEOaJn{Wt^&aAMYi_E(>_IeM=Q-ZpY)%jkt?f9Njh=@TH}bmT8qKL&(|lD>$mk&d#q z%h!2rygXU*xx5G%o|s4*H{&;JlmeK`cKtMt^UriCaO7K=31L@nDBqfSp9t0n@Fnq? z){DQB4GP>_pAfnF41ZZ6vlK{f{^>pyf=kSaQV)hk*7SGDxGh9^s}d;a>(f;yK?^~> zS<((-a|dNt#TT>{R0VQ!)vs<2e4!B9aSAA1@^uRUpthGUM*-j)@_6b#odzb+LUX}#4@OQ`IHO@PQHbz)+auPs#zY?yDuT4KZv-{ zT@{A%rU1&~!upm;gRkd@aL8DRNrfogMm$s%C@U`Tq`Uqndv_LZHveJ+_~u??W*QhX z0ru)yOD5Ss<5uS5e84Q6|F>Z370`~x;GP?`Q84G(Mw1F(f?MmT8j;}J4^udCB#o*a zJKv^YV*ne$WzHeiB9%!uH;z;*eBw(8d!^XQysiDRC0M-LxZ;J@q7&O3A`GJNDGYMU zo&xhz2YCBOatdHV*xlwLy+VHyK7g9%@-&7{z(3=)j(R7{pSTcKg4;q%z>1a%qXtw7 za7cx@e^g<}B*->X10+J5tjV}9YAQci>jrf3fG!aQ&>}GdLRHV>e`jwm^|4^l5xl)? z|KPaup%GZ0{6X8dzVy6IG_Y|}W8M#_s@C`r4nBEDH@?9g8~6BNf$y6NOO6Pw&kTD@ z(bkz+qkl9P7nhpRW@G6Ar8!^n=q`4kI?MZw(VOdb#vSSLukEnKlnFYU(!`s?bpY;u zbQ5S!y9b$EgTr4`%R--e51@44p7$kyeT=KNJk;1}qgz-?yEo^&GsvC1L*}zR^obBQ zIXQVDO1?|xQ2yxj?ajDY-1f&s+bTw_8?$Gv%>Fd#5eE;25Fv>UcL=sTE3r03M$y39 zyg(F^xGsjErujN$8FtP>(dB_$cV+?<$YfH)BiQnVCH^3kEJ~z zIkFJ9)6q}=sI)2U(9>Hcn{F1MPHd%3{$5+EW5;4`%E=Bi%2ksB?Gh3Op~BFN8<&BV z>j4?Zbd@=>^t&OX#$ZQ#>7&eQx*OYDqq=~%VA8~W_?G`QzcI2oBKkqHxT6SXm^7fg zFwhc;9}svs!6-whA}6A|nS4+G$S4*-t`4^~qVkkLp=(XZDWK7}|C>56YsR~>2KZCvWdu?-Vk%XUUNbmLwFp4>D`MBG7dU_6UGnD#EC zskwRD#(JD|T${rbQvi;wCFe(5`yO6imR|vb9bXW6AwjaWVeTl>xN{2!V5N$+eT}Vk zetsSc%a+yO1UAa~KDq>Cs_g>12mOsaBQi>1QgH&eH~<@P}}1o1IM!473!o ziz_*%U>mnb9XkmHE%H6trIe+ms-A03^q=`Ur#W8(_?B>J@o{=`hXvTbJ$ZkDO#nr` zihltQk?1(h3E~%zZ7li(wS_|azYV$bNWJ+gx-gJ_rZ|M)9la`eS?g6IeWWn=| zTg`ZM0;ZY^4D|F-r=f~Jmin^ii7UsZUWuh}Y$?Mn|8O0V_b*)%AGK;p6}re~PyA*j z6fzv}@$Nt{*Ibg5k}3l`c((qD+1LSC8?Ftp_)3i3Wd+;&k5XJNtDUC&jYQbEIX$<{$rQD zCzk+$8H#3Y3_2bYU>l$PCcCPxPf>apAf;%~sKE;=^YzhQ6vmk!M}2!O%Jog2wsHX2 zqe>&*-3ZubyqKYCWnL@?Lh_60zr3}lSilFuKNfvYxWi=8p+SZZk9-=(e}RUbR#K$5g2Ev z&LprKLdnLQDguGr`=?W0mccNR&g&X9(zEMdp@Ns_VJnC$dJGgX1SS!OEwTtfF+^ge zuCTWnkF{9TDMCY}r?o`W2U z9!b)EbnStq0G!Zt_Sk5`4!$uhL7H{L5_(Pp@l#9auiRH1%o|f|)UxB*CpB(*=5-u-Tc(0Ub#Oc0ptPl^UJPFEbpfc*MsIEzBn&(*nDUF@dp< zYU6a#a&^rV>{bf5g;RUi*moo4i`7{0o-x2`Mypt2pH%KXj1c}BHjm>&rkGf=Z8Diw z;vnAk3DzI+ZI8!S^H8BXJ|_5_mI*o2`|U&o@i;>hf2tLb1@K2fj@AJi=B|Bem5H%Z zBMId-|6a=SQQ#-V#2?*#5j+^60T}0FwqAd~&AHI!XF8}0+U4Fv89SOqW+#6LEqJz4 zFclYmA^1_hiR0&x#R#VE{Z}hqmA7|X+-B0DuTMs1D-APDwlk`l_&1m1-|}2Kf3`iV zi9%ceTv&T-`-46Eg54c1<3ds5D(tVbP63*l!_#59`G&qGGHHQ$*DXZx=xH+|r1zyd z9CvGQu8UuKUu>j!*VUd3lOqD!N75yHAqu#BY(-J}D7k&@w;NiUJXMf;$TOqbsQ;A< z$O5}t;ylS^bM#wH?uRkaRgshQYULn-*_Oj|U5s(+i`+iG7gJoK_dBbc}8(z&f4DD z`z5{*)w{r@Bdc++eim3YnDpdVPRY^=MY23M)+g+gwbh?Xz^HsK*@5_m&b+P9c6k=Z z2W-<~!2l|;a_i1FxESNO(tezomS%r@04!v?0VT0Gbhlaw?88@B84B(nS;~Sh0=ih@h_c00(BL*ZdUosGUtC@YuPehQqjS zrN=HY;=?x=AeE&S%b7>D@y1rzjweCHZ04A9rYwmAy*`irq?674)@8; z9Dj67v0qwouautoM3irxEcw-_oZ7}VRbp!@>=7|>f)XoZqK@qFe3ZtHKR{oJc-?D_ zoYtcu15wZ1*ra=;exLp|-9c;#;VM)8{HJ}HTB$ra@PbHI8b*bum=D@9L=BFwOfp#> zVUZ2g+*|c9eis7iK3|@_s1qOY4UlCC{i!OR({I0*-7hglhT&XRk;Uo-@Q{8W{=x%2 z^C#J%xqfMNJF$rBg#KqbL!wx)$A!7GKKQ$7QCYh|30_!RAEv2_;Qr-!)qdYdFvzyF zo&4B-fl9y0`or6co!tTVOZ-?qkkquJpn0-Umfm*!E%XpD?gwAd+wuwhPRq1=Aa_t* zWNP5ezPMDyEt#MPvz-rg$-1Q_2@-zZDf{rD_h;yrdz4OIbx~5%(tjuo&J&&&%g(3< zp@#78uLva8W__^IVsWQg4He3e+O6{v7|ne4>@0gZd|0P@i3F84zKyxXWLOSF_A!6E zz-?lbAsP{_@DY2nG_^qx8QLO{)dE$sbCnmJR5l1BMw}8v+=_ie-)Kleht~en&&KC1y|w~hQ@E%lT^s|aP`ADcqu0E9 zLr67Gy7=fTXzvSxxyed?si5f+SeO2xQ29{;Byw;-w&As>?Q1>6lA8WkVP|!6oaLu* z$k{_q0h8HR>kEBZvyci_*!P3pX7_!OV+mYTwD4QSgyS-a9dc#;*q>nwvvM9?kV$5DbN470<}Vz! zvQ}S8v}Ro&BBa^}kS^Nfdrr8ReinOjSL4!^ryfH}U$r~l6CxEE_mAnDdQe}|Y!Mwf zVaHF%eXNCiUhx>!9h^!a6=h$|D&Rj)Dk;g1pchw^`7I-t7%Yt^T3y@T!rW&CPY|!) zU>Lh0F6WsUScx0!;H)8&COgZHY;&m%%pu7;*Y(0n`|j#3)RDGEt}aTR2FDUpBKR55J=(XSEoVeHZ<4k@-~yxK?*-qY9VCzbO7`@jYQW57*v6SRiDc6QIiu zcSl$@b7V1g*zXSB%A3SZM!D1yvp$n^`fL=VC4QHK(Y@vtx_2C*&!XjpK`mo~ldkQv z_YJV(1MIaq(+C$fZs975?gi#v^E`Jq$i(oCq^yXwOyW$`7h^5v^n)S7ayGa z{?QW69A6l)79T?<#6#|4*4AIfVn6rS%lgs#B6DaXXze)7+HM|d8A+)_)*3r(wZTP; zb`4V#e`5Vz%{I93M{`4!9VVmB9m>7?%N1&p_7dNb-bwJxvEcS4Lv6j7w?#qrsPdu! z=|@8J^3OC6b_?E;6Wl)NK$han3Pm(Vv{VMM~*r@dfP>x0wk{ap?kfm=7hyCE9hg4_F z;F}DkdT*@*>cIC~m{%X})0E~)i-2KRWFGljz`C+{ws~tkM;{ksZT1k#P15bAZFFB^ zWe=)t)RdZZNrP$`_b&+~vnvC`TJ0@!!b63elgL#i`<_2vo!qqbu`M9>c(74{kF~go ze9`b#hPqb_o1$MNinSRBM&-Qa36EpjdmU`b{QBWi2qackoUcKr!~gT};jU1WU6M3s zrY9D%!nP?p1fp~|w(;|8jss0>jFO+zXc!nDth-+esGEGMmcs!_&-Y7P=`StYa=gmQ z{p>=F>hz>O7ah$DLBBU!hLQ3JroY`FC(_)1snP37HQ7TDZi*Q?PT2EyFVOjHt2mY$ z=OKOK?1-|@uhFp0%5VB~c4|BtXN*}an9(DR*>2an&Ej$zvQ^JjTbPG9FBgM z%JC*rv_V09Td;wPj8g1;czT-txjpw3BdP!c^f* zGzC+W6J~ITYhNZx+}yD8nTnPK$mBrk8@*x!Z8>ws)66-|3wEYJacQnM12<{fk!~N& zo8_P~DwAAT6c%>V2BlSS85}~7KgYmL{9#L)3EuRsOvDzI7P+%Zvajse=LAy*UYOD) z$WB^WX=2~?Knf^)q9&3c5pFPFfQ%8mF&|?>xmuZv`+RPy)pl#hUz=%?>>xea9&KUn zDsUI4^8A7iTdW-c^mMR(RkC;+$qgT1XXG=-HfM%4vM-1uQsCe3Vna8LbEPf2$r{=o ztol7*QjMJ&eu|Cx88y8+cn6snabB;l#Pq6G1>ZyR?VxXn8|rRfaU3kp|xukrJg(!AKz!% zeCeS@h_U-zB&Hg6i=sfy#E9WUI(fh;6vD6)ayf79ytv25BiPSU7mAf>nmMZHufw(% zu*fz}`(c+uxoCBEql0&;_NH7(FdsISMF|ogY2^v%7D&y0$*lica^cN#?yAGsU(iId z>YIcRdVK;jikyv!`n+JoD3ASUbqoQA=s{*4hC*VeVdMq8651EXz!d$ztYk@JVSjB* z4k8(J^sbqimY-h6;Cnfg#GnuavyT6a5Q7TlVZopZciohu!1CgLH1{cg53d^HM zd@*@cy6o+5DDV-bGFldUb_gF<1$Fwz-MI?s%U(WK^);Nxwr#=ylhK@HMkm2V{qtGU z8=3N&D(OBZJVED(qw#P~Ips&;-(&ABlZ2}`hIIp3VSFnI523zmlFe@zJrq=LP$h6V z)1X$GZ^k|aBRm1j(>EeiQCn2ssXW3KAgIRE$nC2Q-hJPkNs^G&6bH^QiNi9g%S=D- zU5({$bHRT5C_5?JOHV@*s_HqsUdt2;nK6SHaOGdvWoXR%G&6j`W`lVxmZdTLo;m%I zbA}H*z!%CL_hlt-nfFFk!l7Sk8_CoWx&e0W2V-Lm}&u=3t`A{QlxG9E^Et{I+lb z7zRb0O2>X4!79OmQS97~RO99a4e^vWl_DVyx<^2$ov*FlJZd}ulHze>xeS3FiXzhZ zlRjtb2gaZgy+350jz`yhz48bF7ry4f(|Y1u9!(+MP>N|@B(`{=9}pDWE{~blrr~%bZAy7CUMsV1A+<9sgsfxk*!LXq_ z_3H2zDKM_=HQ~U1x~rV7OhKs72)Pe2aERyetR!-N-T+pgqv|>!+Pw3{gQ<}EM4EG7 z+8uq1Ahl`!-F{=2No@|52hT1TA9ga0o?zjbDl+YHwGVZhMDkDz^)!G0;;l27mWc>e2WIoqt8@}ZrpGz?DH}K9>Ia) z%+g@RmGPXRLZ6!B1eOlPyS?tJtaNE`>FeRIS0A`MS`nZrrq4-aNu|(L>i;c1VbdS)c{bOh@mLelN1 z6;O65p3ABvec*geWZzwi8)P(IRt{ZF4oNKQ`mqu46Q&EywDMu`97knN5B)a8CikZD z$a#_ZU|U+18zWZhJM019Nr6%I;y02>K0!qu&&?gVmH|)bAn0R&s=^h_zTqJi8N`6R zZKHL19QND$L?<^QnF5G1M?tW=9S{22mOn2QfQ30QQp(&rvCysMfK+N|27%yuC61B| z15#09m3eMr6WVHq1zRT92f=Rc2L(bSGi!g%-mz{BJY5~R$NUxK>Y=BiRXjA7Bl40p z=Q4J8_1yRyg$X>$v_#cdz!VF1mTvSu+)8G+zgQm(OV!ge&06U=2!#|46*qRcV;E3u zZXuYsz`$-4Nq@hZ~LKJwPI{s&)AD0i;HVtHKJn!c|eH;QWD@>P;wbD+qrO$#Xu zK1_1{{lI9$+7mkGhdG-500Q~E>&56pPkEyP=OPXwER4s7H;gALr|nb{mfl0-#YEZN&& zP`Ua}@PqSP587JFwWvq7WNY5MDQN+lfLyb(7Co9@e)%}89kebq`!Px>ALkXv>cNb8ADq+46 zSd=dT;a>>wZ0ty$XEhfvC6r+@HBW@^LjxV{8A%#(-}cR5-Q;=ajN8_}xh~q+7<2)` zMTH?(^IW} zhjMQQ;TU_ERA8el0$1$opKXzzV;0cbX8#XmUmX`^6Lw8WiAYK}5=u%4EG69%1|TUZ zC5W(eBOtJdgdnA)N+`W_3P^V@xze(TbbkZrQ}6r!@s0j&-Q7E8uBmJ0KG!*b?O2Tl zn^oHn181vMSkq4sLWuu-dqp;2bHX0`{8eNIm#bQ*H_u)UG35v_-mQHS=@qDHReE{kF-xiHI&osG&~kB`IZgO_`p9jz={ zZ9C2+C%Yq2^h#9jtnf`RccVi=AHGPAi})C^jRiFC=+tNKVxzw7g>6#KhQC4*qQ0!g zm=U{#EG|*0P<%myGFNIyYv7%r$t}}D*c@LfI}+M`=rOqGyZ_5#8*=SbniDr%3x^J& ztU|_iZJ%DkKP7$g*^Rin-*b4Arzm$zfKhK)n&aXG^1d76cMhK+JRUUtG;}zZ`7ZD1 z#qL67ib9gZqvt8|w8%tUNX@4ujpcXU2TcuLa(uXJn9yg?>mHYz-nnXilBn*Pag5dw z%u#7n8Dyw)-Z#Q)>8`w5eKa+>2>?cnoLW2WX2+#NnuSz7-oB)9%$s zHtAcdCns~QFP}%`U_b|bNa_>c9k$yYs;({v@}a4$t|0qaH)F%?&HD zV~x7^4)nKLJoQH14KdM=t&;DrZ-Tg+RG;E!@qEMSb$AH88vO|9N@4!P-9QNaB_6+* zN{OWui1!4l|C63+^hHYw68pYI}F5XT>Ig zgBcBK$?MS%DRDlf+{Lt!5u)c2K{3q0vPk(jaB^V0MQw$7o5ZtoPZf|rJT{PqmYMsN z^yNq5?O%u?mXirbX8}{nWRn>zTmjxrv@h_7@{e0gudX0uZ7>;w`~$6(uyITWtK=PV z8r*67#x~vDDhSwH{T6xyx{rX1A)Yct5Bloy0(HPmEDl5wnjzIByO4s+NNxuV3q?h~QTKay**K4+HV2HqJqc$I+r*YRIG9dYW3W zQH2uE!vqR$SXBo0y>+JEBIV?!97)9BU;LWAI^H8vJe4y!+0X&G zhP=X-b zN+j}Rd`4~5==-!=3nk<;Eyk0q zF2M&%k@*c%8O|A1INAzq_lNx;Kv@eUOjI(6Si!j4QX&utK^gpXtmpDVw|>y8PRUz( zldqYU@3JFKplb4fgJrwZvi`L$Z^xXgK41J;We6eej^buB8}eD03S4EAP8&e&q;1Tu zg@8(qnxsV_;zqq_1O=@(3Ne3*QbMMPps>Xbgdm!szutL-rYPUWN0~H-?Phh2t~Gm_ zCEzLU8D*&Fa36H13kLNXq45vmF*S#=WpA1)O)vs$2d&EFaJiP;QgEWLKs7K0GrbXf zG^n0QT&?36^2dMT6;$>oyRYY+gz$aXBj4A(3-vP8-1$=>eZZ}$9^zi?=}y=0MLK1wdb3P503tf_{ZVl3_v2nX+T9OU zcQuTgJP##$8^n_;(OnM~(|l=VM#Kld`Ub+%X+?0vdPJ(GMsx=gWa{20xwCJHdGpb^ z82yP%dnHl&m~)1X-}z++ws2JmFiE)mz_c%)$<#&`}J zQaqCTeMJ}j2#)c+F*5^Y`&&PX;*|$N^1iyLIf|GixeGbBhRWSk(yU}%**-FPmjg}j z>E|V0z+FlX5+}ia*G1X16jpoERua2KjOr1e?blr>*T*b;FA8IU(qhn|w6Lj(p?!gpe(|<~u%#S=@<|#H#1r z*yYox#@omZ`WB>*V~&6ktjuLqtX5tuTj+|}rHNMHU`o`)2P(-OUl!4;4Bi*ART&n^ zE~iB-pYQHc0l&3A1@Q5KgbqEq26x_#sl%Kh4CjQ6yzW&OY1b7o+T(7yPdn{|h{&wR z84_rc_PFjCqWNu^OdSM}(Gn>Lhua|=3vOS_Dhw=VsyO7h9(h=yT1j$1T~}w}s90rk zKl{LYm&X3Uz7Kk>P*$U*KpUu7GzozXZ#4P}No77pSx$C?1m?XNBS7(0`*oDWuD>@Q zw8aqRdpI}WJ1~!z@fob1=u+7x;De#R^)cp$4C{ypkn@-$7*8RKjG?P2Fqv98{ z?WtRS8Bsl$sI657(^jjElsam-j{nq4lNa4?H{eY|gN81?&v$MxXKS-oej$Z#L=3EY zFMJq6u&2;rKy4(DguL=0ODjXG0T4XI&8%4vZt&%c_V@hnlJ%henN@}ilt+n#10kL3 z+~apcrxGQXTm5*SUP0W^-F$F-&y1tJ9%7sT#1=yovz2>UStJUA$?g#1Z*AG+C((9u zvo4qb_XWe?K@G)j)PW#+S#B!t7bHQ_#tk}*dKQz%El7!o)??v!EpAUrT##bzv_J2? zEOnBbwU8Z^^$vBe7oVzC1m5U6J7wZMJxtV*6>a73aRoS|HsF2KlbN^VtSnaW%OeH> za%j(We>|pb9aE^9Q8TAt8mVv;?|)_(iuRqegfI1HatPRvsI6MP)+BaW-Ppk6JH`~l z43kMTF2kcAGhe0qxN?(k) zKls7=gSq1&B%H(6&s|PLk5ZPek<)8aC>ZLAi+KNewR-3&ZYA3Qd#2m*j9g_bb?^Jo zN7zuoR9l>7u@VW*D`sF!f8k*04}r9xLvTL6iKD+BColKTB*z*|Lpt<*cRyyton#yO z(K6r1*v=C}Ry`$L;CrWyT{S!#O}QChVLqs^oiLpTZ`p8Yc@!9L=O(wo_D_;;K^H_pChRK31U(5D}xL9QQ(l+^G)` z{(db28lQLLTKZiFLSA8C&UtB!JRl6AH&gNENp?Z=cC|PDyi=!Y@aBsbOx+(s?ieNe zB!B&Inh~1%>4#R&U?loseVY{~bWcMH#C)pJ(dAUvp-5#;AA5VqjT}YQ2)I7Ejyw0$ z*xn37AFE$?JTy2MS2?9+p}t=qgOfWjGAXwuETo~THvm>iq# zb|{~q2e>et9Aklkg>}4z#%i3g8Ip0uQwacJA=(TyX8n2P8Lr`pw7B~H$M{G^yW2Y` z68z>9>efsxE>)ork%ieBW1@$9>yv(1Uwy+G+}sGq)!dYvw~auqwhP;BJQ@o;$ADz4&5 zxA&zxLrW$E4HErnGBgU(pBO|}2+sWOQR-2JuXncN7i!8u<8mW=^ zb3Ex3*8+c4JNo$!#j{R?$c~f zWey9uolIvov{Ivl)dRh24+*~tN2nL7!cY6|AnJ`PjDIK`ZYv%aMm+Q7bRU*A2;mN6 zXK?>??@r9>eyxNkSE_A0I*#eXX>p@%BxB(mtdTAl!pxiw>Pkc6)t7hTj^q4YL0qiy zW|uCT?7DB4LC~%RB(UxVua+p6DQ_Pu`_KkMeifU~YHXLi>cfM6DsN2-J=BBi91D=} z*M^3kaP(jP^g}Z)q+K@6o=fgI%}s&_PX+#Z4l_RV`%RJ8*{dJW*i4cIy6UQ6KR!)4 zc!p-V95%X)vrc3loUpPzmj?}}kq%kgNJs2sz?m-f@<@0ivUrI+BI z!H^bjQ0h6jwwy27zP(7xEeCj#M$X#M(0u^=nlLgj;7dC`9a~=J9Gc86PjtH%e_^Gw z+`kj311!ceE-9S=z!~Q%CwG2J@N9gjT=O0?^FhWvjR8T=o7Q0S+w-XAto4JAASWj` z3)>172un*%9S2biuBG~GpH@M{LD|Gwna4yZV7!YbCIKz>8+BczrdAbXvEnO@U6M@A1j{s-uO&Z~X_hd(>vf8}Yv5HdLX`6qy9{u>H@ zE(HGf2^R=wKeI)cJsMC#f3bg>MptsTM7G}F7#HIYT>Pi>ziI5z&J`8@hM)gByqWjwF_?&zHH~9Qa597iHet(60{Ncc7_$zSY>@1Le{*FYL?Y01T zaB96q5i#Ga(06ww_ZXQ2LOD?B%%P!hrW>JEaApTaFlN)-fQKIjcxoEU8p8}^u5V-y z#Iz_ooYHp(4j39vjASQ^?W+JH-e$UUa_--UTPP(&G@J#{2`eFp0Z%V03Y_v6ZF>5` zo9Xp8a26*;@a-!&l2}++I1exwyri;l;(A#a2~ZUT0U=YkM_6Pfk$#8+%fr z5I{dL+}rzVXnfqJw7mQg1OhQW1xT)^U7ek}H*eldO$(8lJvo^YlaMHNc>2`Pn`fPF zeHN%8Czp1mCMD7AA%B4=AY*S|FR$kY!;72VaX_KfrQBiBtbPzxDwH#VByFs4OabtV_$d|R?{bOVP@RLF z714>^|7*&(Z{HZ6>HvZ2!OW~Iy|}ozqnzqRiY}RYKqz zynO7PSI^<>z*ZmZEG^kyQ(H{tPSCR^)Qj-dMx*583vzQW=K?dgzD-E@A}J|p5JjQ0 z1j1Sg-wJSWa7cf5W=&@#&;l~o*X-8os;f%?UOgXdfqwDn>F)MJfm5ek>lmw+V|aZ7!fwx@HbYjBE9D)OY26Gi^8VAxgINc{W@B#_+6mC+rIL<2nell!)<@i6!soCi-$w% zU%XkQ=#iO&g*4OE-tV!zz4T}cj|cTCD_}IZsOT0<*tGZ>mGqnjft`cH>nDvsK(kXY z+gb)YVSe7Cs374nz6-NGd`OrT>(-Ak(*nFsm&#Z>(#2qZ5_56GE8X_!JCoBLk4w4YZJjdXWO|DM$Lxp-@Y;?VTNc5FNtyz`bYdcHnOt zHMjBBP4hY_{_Zo2yba~$%Ef8P$(@$XQQ;7~tuIWNd}3nl3+kiMc$xrcu>ZCA6@2qE zVrIXP2=6)y3)b%s6bCooHq+rCCMTme0RoGQ&(4C0K04w=EM=K*nDr=giT++bDvz?OdJt)ZOg-)?WDc!tS-AlSIng-pNRA~ zAhw-!RlM=cSdbY@25-baf?bLJRA@nZ8u5~1GnCytt+$NH(CLbZ!y05C7{8XvR>9LL zq(V`Dk$!1?{c25!*-)nylZYre5|bJ4*xbPI6AjbX*oO%*nql8!vm;pM=WrJ{+C9y8 z4y!Wi+q#=w7=^`86L~HMlf94{h;`hYZj(sr)5OmtSP%U<(pj1DnQ!o8%ShjQB}+8P zQGhiW$}=YuB_7 zDD!cykA$_yJCC%8VI&CCm#$HVtDoyUp0GFeQlI`h0Pylt5N+b~Jc9R_irH4?@bHqT z==2OwI@>y39KH6*j6gB_QNE$V{b4}p*)3v8MfCt$Fo-G z%h|R~<s!gOTMh)p1^F}rn%3eAQB z`PIXMCQL&BDnVucA;njEoz^%h0CFHQ5Y<&MnT{j+0)&kmi>?+H#+!8UtnP?rXmDqt zV}8l=KFJ?B*szHhQ|!0qiM)+mY4)8px}c{$<2Jm6B7^IkfAohK6T)K_B%1_lze`Cv zvPVvd3rAjmf{|b-=aYX$X@7q)ZF6s!mO>*9eL~^5kMIenFn__gy&y+!WPmM4ILMlXUJ;u}63|}hG9obg^+E30mVkEsG zvkR$SfTQ`;YzgqVxU6J|?|_<-bDlvT81+%nQF@6tk-3hJ5xV!iSS;*^K-slOPj9!A zBlkY^*d$i{L7+tF@y?%2bA^SIfzn+Kk{wf2rc=gQ9{#u&lQbfZ%Pf_zrQc1xLZLN@ zMKTn9(Z@88>|3(;JO>chA171OsDAY7=VIMe86O=;?*tI=4Q-wM{Ba=`S;^=WqFeMMEH6 z;)P8oKOsJmvzhP|t!DIT_g242Br}{NW;N9QyQnx;EkO3Fr*2BX%?X=_(kdkNWQT_* zV-^Po@0vuQF)6$X+!uvvtQX+P{xGkucH3yQB5|@V$lG=*csjZ2S}p2>htMjxO{j1z z;D$|jhuv?KXI0oCdsjp&8T;p}LYbC^%Ef8jRHvzL@>bNeTQ}KW za1c1x^;;5fr^rgEMHyvcDRg|{3S0$mBr!9daAf0r$qJ^K8xO&Nnd?QPkDEorG)25^ zOkPi1#e3&xSQK&i`c*{X1T0HDWRReYltr=wwt`*WDKBhl5ZV5^vdDo%qCxW4&0p7e z&_oG(+yRT~j+d%My|Od$UB7E+`Qj(=db(rN&glIC8SOG&JhahAs|Lt^ap6Tz3$x?p z)e&P~y2SpORKa2$XS4wRv==3Zv90Sj;iN>59PD|pr(tGm zdS$#k_DvNURIVb4aqv!1gj&>dR{O*GH^#9!^tg)^_6%Q(O$qGGSGvGkxt=b_7hEL= zEF_zs?F1XKuN=9-Bj~4KQ|YL(NNL=4>ckeC@b25JC1 ztnD)eBI??xh-vQ@vmfK7wp9f*p#nH3l;MUSY>zje<`rA@+`!!1*#FX!TPtLQ+gVOF zrsAn_M1eYI@jtJ$t%`$%G~bhdN-&I(qSN~vZM2^HEPKy3UP_C)jM(_MP-Ap%# zAEHFYMEC`mi5|5eSwW+JjUx8cT=kbyhFKU|11__PEy& zRnX6uo|jMwGE*zr9;jvvU4o96S>Jx8NXF3aEjh4Wcbe)rb}D~R;j$WU=0{&~HF*RR z>hbQ49=^07AD z+t5)F>jmQKJsqRBGBf5VQsVbGs1oOK1J~n2TvX_5i{vuDWc!v(Pfp>&iJMsx7Pjqf=2a8G#=Lswr)9b3d&MGZaTn_0Y$iG}`6j5Qu~empBr<4KoE<`9dwT*p{M z{SWQ2TJ6?BU-fXbI(9RRhko?pdK>?(Jt8}Ys*s^;(G<0?A~ z%_as}{Hp%$5u1VujR+TXtva@~yZ&xqIOSkDC+*gicB24D;z2JwUFu}dY@pDdhZV4a zIOtfr!%BW$(Ng%?|H8;74(o87e;HBnR7b~SLe$A?rn{*A1D+#gZpdVr7q(tqWh*T$ z?daWwX^6-U6{} zS`(y8UlizD+%}T8kL()cHizHV0@#e_T?_-~kqqC3cMDm_nkIUM$*G&(Z1sz@e(F14 zll3-^9^s(!7(OEx@zL%XFPxI2nX#XA?3|0JoHJ41_=l#-zF0k-G2-rwM$nXTuBfxW zwYfh3#+!P&79!p2>dB&x*CX*;WG~|_hYY(V3oOwlwV?ZRklbo>#^K|qiB_x#qR_Ut z!K*8{cKwC*4ckucJIZQ3T%LdllNe6Au0?nJm_1g)R{D~vCM8b>rWk?-90Qn8gH4X$hh_YW9u$ZQC!|8` z{*4B@UE$X217kb%UNNnDyY@M18E>QwRZ@0HM@_ilW(M{U_8oErdIXvZ>5WYUq_a$q zM)B2C1%p)yhz=u6lt@>uke=1-3c98>6y@xNk)ztHnQhl0T#yeLz})`qO(5eo%q}Qs zc-e#v?Oe96r$eSxDAfq5 zvEJ0m^H`WMXA`)KEtQfuClFj4%907?p4gdHp&T_s$GzxzGydPR zzvr}p7sP+3Z16z6=BVk7Ko4n16;KW$&@o{RgbY1wsOPTHx}i_}R8YwOt!C^lKB-NY z3+Wn1#9g)c$Hoyv1#!Z66K}-v`NF%=ILo*Qo<}r%^#K0w<;M0ybik5M2pF(;18$>> zpiz@KYNE;ud9z>pLEbJ{k;6BEC{5PxT_(9BYQ!apS1sy|2irfO-NP8f_Ny~rV2x$#7XB^9=($Q{wn@C8KQ5q z+^+!PI(+x5ORIP@HMi_^(uYSG!tWoD6(M+^Oe`Hy^s%_JDR+jT$J{OxfL`4i@o}mR z3=AZgSwD_K{kFb^p6c^?0F?R%CDPh@Ia=_Mz5v>kBzxt9}8nZ ztc-r&BZRglJ&T|f>&KlHfAbID%{&P{e{JM=Kq{bis-MtLDQIKa`Bu(`_GE9vY~uLp zjKvkW?7d(wo|%l$DVAvW8;*)h@hSTZ;xg`=eR&)T&m>NoO(PQo;1*%QrT`|s7CO#Tp7L-oF>k6sZ?Yt=A5n~&ZFi#{_h~3sT(__G|Io50S^E5AlCs zdZGSl^naM!7Z<@qM`C#l`0+6xjFG@~Xl7i#zfSg1DK94wmHYJaqqF4zo{qeZp~XPn z!xAP%oOC;lE?yYGF}@^j@Xn1OiImGQZZaD*ePOHOzhQNYNA0>$Pgukt3qf&en?NRf z?%<|l&SC`#G5gN((Y~`Jy;`U^1v*}^J zEv10qc*nPIVcA#vx8-<$yhkby4k*9UGw+)Tra+@_;WJZ1J^ z>0wmv85<}3B0g@sY_&yxpw=07Wqzw@uwqxL^3k8Jz}*)ID7b0o=%ujKL#ZRrZ7f|z619YF5^UNX*%xf(YQHq){Pbd5Xz%p)7_8+@=;N{#raW9Y=2 z5sBNB_6r0AD^PL1{kZlQ23#W8P`k$4*H{PDB2vG?M!$XZ9)CkjBZ5Db`uL(`!&9@5QSCvcK4ub^66yvB>4&TL-E4DvJg43zn)McX7%KJ-=kps zZ*cjXHA$H*9@$MuPx^i2q<$ot(^^{{61ZvnK%=$YqiFOgB|=NHJZ}*&u~be?9WqRE zpR+K786keD-PbOxfi4hoUPF#nH~{82GM4|Syt}}-j{e1h1tt^*gFQ>x*!~iVOJLX0 zqJN?y1<)4StV@zXt2xPJy!x;Ta<-SaEL!T8%rAnRI-P>4tC6=K!+) zt`Qu-H7m#`DThv&Qas4`u$MXf^S=KxBCV%nGCE(d9Roj~jPbUIwrl%AMBBAAx-yqR zP^+AKTsso9wF58yJ%xspDkLuWku0#i>JD>(?10W#ce*jAO!SdGQ;Osol z?G^I*i1*GDNxf77aG?~X@7 z&1v7{Ij7xOh5gOy>fF#k*EtXLEK(=oPAvUb3}qZ~_!!j)g%bFFjbT4W>x`ZNFH3sW zMBg=%wH#q`xB*)CDHtAP3|kFPjdX@ljaZ1fDC}&5i(nq@qjiZt!IevBjm7G{L~Hg` z)?LpTpeRHl#@lW~z%YatNSnY$B{0^8Bd*0_$THoF;$*~Ual{8yq+;Bp5x6RaQ9Dt9 zvPI{CeAWB3SLy6Izw|DndKZRW--UdlQ<}IlO??Rp7uF`sC$9hc*b!$&vmu4x^}J77 zWY1Lw$q#+w0ZV9&&#OblMDCj2BMruLvjqZ;1$=D#4b@{;p&yY5^!_-wm zUhPpBXx>oEFh*C@dcl_$zU6{HIN!!t$jzKTgwdjVs9{G9V`9dC(ntjK*VvT=hi`%% z64Z2^aXrO#f`+>M*x;>Td^A&|$TdIb$W;@EPP3Pd75EDeR*A-M>F@S)nCfWW;Bz34 zChFB|6=-=MeUk2&ULdNEhQFRPMU&nh2^1`nmX7oMm{7BWl{iX#J7!$b_DP>;J}8WM zxD%TD4pSChwXJU0e+q~b*FPhSBV}UmwA0GtJ8^rSW>j5xW8nCAhg+OeL^Nm;3LL!- zfY_An;!`pzVxV{HJ|k2!@_TE?wyn}YD5UjFj)9kN#1ky15RLc5!-i+%yj9fY^HY2t7wETOt7K%C_57hrG-lTt?4~Z>}I5V zjuPzOgN2ooI46x?(Uuz)Ul~vj5Fc-8YLZ2%-@LC|k1tJ<;Nm+5@yu-TXN-8Jiy`we z)-NKNMpKSVz8yWZ0*ivbYCB-*aL<oG%%HBR z9|l-yo7>=R-uQJ=?Knudd*zU}UWJddxG{S;Mn7KKqXH^FzQpT3BJuNOxG$Y~(p2O7 zRcE;u*=?Qh{Ufd}Y>7EPuAq&vmBf&4zs{M~TYe~wA@%~^Bu;H{X&Fyfl=oY8V14UB zJ8hR}tJkUoyZQ7_&)B=HRo84~%I{r0{IR|55??gup@db^lNz&n<=NEhvNvHw$Xo9t zpbNP6EVbqnKiE}URofqgvW)TmNDP(9b20Bk>Po%|Mid($l$~(FtIal>W@j;5v~|vr zC9Q>bsjpn}%EmhO3^qmI?ndIa-yJTkmMprt$USj7OaXN>djC}0&kFU$ir-STF{-e7 z|J0EJz%T$11EF<}$3QHHLy~;=9^FBF8{C)pl{Afo?}?##i5&YUBvewze^qbS5xlH? zDU0>MG9@WX=9Zgb1B@Sw@`)$6Q%Jo|m>?#u_)vL19^bP(&X|~2HSDUEOycTc5Kipt z`yM{Bl9=UK)0K-^TRQ1ovvY)rY2&uc*XP$Y^pnJz8fraR`p6%wGMsn4goSYnCn!8~VnB(qo0eB5FA6;fn zNuaknTD_J+7$~5l#zm3iQ7u0`uOAUmep65~_Uy@t1TYPsZ<+jgyB!loJTk0#S~bEt zKc$oRzHKz>3wUI!@PANA<{^W=el3o}xKQFL&8vF<(cET&{Q$#ON3BoCMw#8|BV&Z} z@{?|ZJ71=b>M8(C1SG=w$K?~8C=OI8Ku@9UbP%&rfQJHQ{wkE&>rp1?oHKatTI`Qo(sfV+2^y9}g|_M)s%4 zpp;D!{sr)uky zXxnzaVtUlKs|O2G!>>UzS0Tg0P*H|h@u18fod4nLzRJ`WN9wdqGHmpDJu>0^+&Gwv>u zJzk7WU~W>6&uaI)^U`9SYSL4gfK(4PY786L(G8-Eb5bGBVV5lA9V%|P8bwjiAwnzh z=Dujo*qnwXJ+12FEzx60X;dd?SKOjyHjl`I3C{3&`q)DXPc2WWj@`OKxInjDnOC(X zr%bB4X8@Ijox5++>25rz7m%o+WKd9BWZ)l-cmiFbWVU))znXU3P8|2(r?sTGavzu& zTV1YJ;+Rv{9-nS(@Gl<4hH&J{P9)#0kL<^DE+%>P{)H#ohp&e@yc$An4OX(!sk`62 zAH=6HuzpPuQ&1*srZ;CeSgp5Z_S0g^Z5PQC878K8V`$@Y#(bB(9|x%rK(6XLf;f_NU+@(7WVXdN3$F9E^0n$Z>aJAw?6erZzT-Zenq_}_O`PS+ zwOd!0WKoOd3qn-hE8d%n#&$`0UZqj7;@=fMJP&3*V2S`#n(G`z_ztf+qrKnuMVvv&PH9U+K<=p|gYN z6CT&9lsVWO0JVTCL3hG^)rY;21+T6a5Bo(IDh7}zF2+au;GI7Zy`Nz4nL;4Gd+YEh zI-NnbE)rK<7CZya;yYF)OG9AF2((KDF!6MVNCcGwKQ{zCzKp$uzq?34*u#+OR*?Sj z)7~?ja|l#p%4y%O_Y2-~q==Do`D!KVzV6-1g78i8&FTwK&KE$G$%43mBBjP^KYp97 z8$ZAKbmQ7#ZEIygq5UJk6;x$h(A-|ChZJeyBZ!W1eWubY-lJt3Ijo;bdO?~B@IkP@ zkU)UX^M3sKr?k_f3+R#w;LZe4qWL!0owiam4jfHO;+C-CDR{MFg# zJx2TBe{ql^FKqVbcBqbv+tF9KYGR_{Vxo7}HQ|ZSs7hnKirCVkActPuRT@DIMUzG( z(m+4qpMT?5hX{vEa{rrwwHgLF{6ora7=LLh%QQvcM-f9mv%#r3?>&TTsL1==bIo`&0W6b?2MVe~GA;L~3< z_m|!OB5wWb>2&`g<^3;D@GR`>zw?_M{C{Vyezf9I-2HSuw2U_&l=A}aM2vF@`rK-4G0hdo4a%KEZ@2*ugH>R|%yER#=QsP_7-#mEUTI8O z+~mAy((J*r-EQK9NnHyg1_qn^M^_^iST^76v*r%nn7U6ke`Ln<(q*n<`T2p9j|lDD z=A^@${hIT;_(}cYD&GIJx=r0a|BQnSUki;Hp`19ecCDsvH&rY}WA2Beh#CQuxlbT{ z)1!u-K1y+6_wrdD{${E*uHyhYm#;eJ8@(hcP?<_~(^0$6#Zpehoa?5WtOZ{#k^mI* z1kB2^Snq#c3zbZhE!B^lOW{}{LS;&meFu`AyDZPR#H zC;!3r(@dyJk?UNaSIC(@$N-WR4gWZh?A%)5@B=Oj{(!D7(U`K%0TcK zAsPR%zMdNwM~+N}A>FD51(UjXeWUDzZ&ndgZGtI+WX3JJ12AaDVhOYyIf~rN(pJvT z&;OQu{aX0Ps;a6Al)B|SZBa(|>S}J`gWREetR(z-1qBCy>(sHo6|m0+pWL|^JkIN_ zkq&f8=_O*6*JmJNz&i!@TdtVH1rpWn73#+Be{Q>(mG@H~UT~AYyHga;a&7~@F$Q4A z#Pw~ZJqhIt-`}1xed#X-m}MVJ^M~|Ov#{QgnvxRM5%Pl&&Oc@K@q%xjoM@bv7m$Ab zY@*|iQ*qzWxRpqt4`=y&@LF#gfc^5E<(P@3ii-j5YCxU~o+)kX{_~xMg_hl82}#M7 z3Im?KqZ;P&MSC;a8?)dZ|m!zxBLDKjI1W>UHi~#s@-lW$NDw*ZiesyXLqgCH* z_wjL!WZAtN!6&AF4Ccqcfb#P4B0>>?$xj--+0h7?_G9XU&vclEZ{~Z{q&jtOZWiks z841HwSxE+5!mJ~NM9ZevX%^kw+!ClzYHXyoYcn&_K%0Q7YN4%0JxiN&de_a%%MMF1 zoDiy?A1^7Kq7m}<6G7KHuOcvc%@TfY|7B`xyIP#kqRl}TfuwCsEYSO0Tv=IJv=IEG z(aifzg{8M5TZ~qzzWUsIQ6B~f*E2(C-z@B?^_1|f%qibFxBFhx6kTyPVK6%7$(yfV z8`1-0wTM0jj&cXe)6!C2 zx<4|%l;~Jo=EI=f4%Vq%jF@O> zj@z>uL|8$PvoW0r#VU@7ww^yZF-oVyQ~9p{x{)qSckpE2%g1%CwKYV4wXz`W0gU^N z>R3kG7A-j{Sz}v^Nh<*itFw@mCz4f@JSay+*p?V9T>JFaxk1j_>6vxGMC1pG!fq+% z0W|AFtgS?7+e@2+A&19CYrgL8qE)WNV;@Ry?Jpn(Q#ahWM#x1KR6-yJ6&h?YP0v#w z7gAEA+N?ZMDV9BlS6#LttyCy4U*DusFobXO>ip>pXHDm<@!&#np*-?)R}=G?`(Q7{ zUc26VP+=fBb_$4gT_>#t#q{2ES$5Tc{*3e_Y5#BxO!JJF3lXg>QfD3zS#h$xO+H#i z_kD9eG{#%7qo^|#c+q=}bP|zDbD96J+*!Xk8)g_0Oo)n6U$V==0df#av97pa2ubac za`sx=3E*{AK}cN z@%Vyv@4NRwFb)4+o-iHNm#USbv9B}(1w-G>Fdoyn^Gb!tiUbEXdsOuN)I+6udSpHd z;ypPx#)bcP=AY=G<={a41NVOHD{`Raw+$|mq7a2V+Rg>QV)b&&!i+8!Wa1q3Q z6bKjqKy2|CS8B)2mjw>$oBy-z?4q5No*EKQms9B_S zIJ@C)&VVwQ^el$$gp)cypl@e7Hl}-M3{${NXd7plS1{dJ=VW2L>+G?XF#6 zMfvIx;LWFd@ji8Nf1GO9JIdK$F>!H!BiY4p-E%%!>U8J)?j~S0aIspTCSn46)h=(d z#bC(NNYDp7Xcb8pRXK-h*I8=a#aK`j0*{9mr)kbFvjbTI8XI65+Kh*IR#wr~L8O z*=IHWs4o2R>Hn{ols_wl|2_xJkpxvjeo=%!=hXj+69KbJL<&R!oe?X55q^5Q$z^22 z@qt$R&T;w^khQ-;P^`FrFGFUoEH*?{i^Be%T>i7J zn|i2xF7t3UM4w+^v0Zxo_kso+0@Vc0GA6&>i3G(0vWDs)f3Z9!P51x3{{2Uh z&mIYWyjUEbX8=tc=#^lhp#@6kok%p@mxow{hD*)V|FPTe^>7c{zXe17r8oiZ^VhQI z;-3G_@PB^(*OE>b42M`I96{;U3(trYpQ|!iFc$7*1?eQWlk?oYSpNNUrTZVL50o%>zM9m!YLeJ;8=D9+Q%&_*+v&msKCzvq?T83ma*^Fc1W zkc+&d)u){eHiw6YWrc;NiE1sw!^3t~R;}IL-9M1XjD_yS+1Up03h>9=95cM87lh#$ z8XD^HyeS5=qOl7P2%C$KkGFGiDSf;=}zYWB9bOJSTt>#355K$RlT2}~B$ zi>hnH6< zHLK{%%RoDy{eL%iKNSQ6-6L$q`SJ1&OZfe<6YuA&2kno_ad8<6ErA48}_-7>uF40$24L1reDJvVFnliGB6%rF$ z0ehy4Af_T%h%*vzb#3jK9wHo8KT_nVrR1{m^6O;0PvVnU1%fl_O8{R=sJfck68$}HgjSoNpy0htAd-VBXc3Ll zVaCM7ynh7>eCKo|H3LIASk)|*R#a4^w-&7D?Kp$_3EqfoRqNc$Vi|M)CqfBFTtj z{gl3*)b(PgkASx}>hX)lpNdCU4_`8u#J+Cy*S37o_Zsq&P*9=RYw5I5pk{ATNiuDu zFx*Fj7l9Ww!X@{9Q=>9A9*1AsPLEBV@wz<8>yG)ChMSu2zNaw|LK_9DgBnx5FH$q0 ztc@c%ZzKA=ImKu?!D2a0mQN1w)@|^$lz-*Ee*Jm~xzAsO0ULfzCJ#SA4(Y zKK-3a+--fzb4|*ML2=iVo0#}3-9I*HqaU%NfW*f~C}M1ks-N(6SHkm{QLw1Pf#P!|(__`gm|)O^EJ}^0lK7Otn`wSr*(1tt4kju z7oUz!O*zQ3Hp5)6t}TW_yczcd=elQ#%jO8x#u?V-nNWlL=o`Ti8e#50YGqA4ymm8A z&%V}czcG=0jdxCivkA^8ebp61XIjHcqXXnxU9?t0E#s*=FG88n&M0nVM%>nWd&~ znxYw%saYOqTB4Yl*J_$+UU{^5x1MONu2%;?yfE?v4P87o^c&P__uGKq%li&9!@M)^ z9M6CL&+|-=fGPoYxr+Jl`6lwe0qTR}Z;*1Pq>U(NTcY`i*ZI-}D!ECFM>;js7?#!W z%m)odUGIE5*ugV$*}$a*RX2oT`MN_9AwohkueW5Zyen&**)AvSt(F(JN_0PM%9Z`# zeU$VnMdEx%S$TPR(;^W$lb1FN{A&==XXt?DU$bm!WmWr^mX?;c`D8IUi2)VA<)yTj zEvkEQKifZrc&r64i0L-Eo?S0Vm>6pyj=C&zxit;O_Y|v`kAL!Xnwn|7BNMZGAL)KA zvCKgI;zyhT_f0I0?nvv%A;;ObAfj>)%#H$jsBH64IdnKFS&gMr%BlAA^<`e-ln^cg z$)a*>`Rvq+)}cdDZ35g>PZ-G(54IWBHySp-Huf9bw|1Zsi?pN`jzZC`8GvU%_DB^M zofsY*WYNT+P%pz7Z?^@IoEU5IK$QYcIW2ZZ`IWCHP!y|$Xs zU7I2UyX>`&BsExwEbg^5A`u=eDnW?Ks*)twNQ*S_%>Y{%5OD*}19c?pJ;GY3KFw+5 zZs=S|ucZ&oSzmMy&prlczx?dQn(g_MDbEvXe`zh7DK3?iZ#si~?CxYH>OtW&7?)>j z%7vgW{-}wvHY*M}yEdCwzUqP>2f1(T`s8dp0h^5<@Ny*j8ZzS1BNg{FhY@{uaF>9C z;W$qUQgQ1n$5oP+`8E+@{-EOtNhU+^whU;ZaqU_*yd9FI(g#no<_d+6m@&FI91f8i z8M*C)m+~&NPA#TfveQ*~9q?*Gp33Ey$r_reQfxH1(u40DB2$l_G(mjw@l>j$Y&MyV`~ZxlaB+uO3j;4b`!r7ciAa;$VbM3v+h^Mkc!csPI8r z6HvnvY}YgM8LSUZncvH%Yplf`Q`s2IHg~)?dv7%pJ^=p+7Z3;~`?Y2OOR5eZsq%73 z0lMh!7BOC>V?V=kzmPJIPNpQy=*3%O)A>GSdV71lE?jsF6DmQntredVOGlwVuAHx* zEuoB$GP>p}j>W|+^x%}<;bj8>2gre*O+M*I2MPkT84Yr7tXH2w_46C3UG`O1I%qWK zmJA0)#*Ip~rza?}6zmf_hu7Uh&R0>3v%~k^YHCIWxR7EIZ+LHj2C_vK^$}>D|MpPM zfnd)*n*o`&o}V4Fo@5Ka!Ue)zli)^(T#E{Q6ZCty*MlD?B9X9G|Mzf%DV4Xp3PIgb zpUgGpP<(Q$ym~-m2JZ?xx8kbeGAp4(P?i9ae31( zUv3X`zlUc)Z@rk6AP^-Cr?QJkHJ9EtH#Ie-B5%IeBLhWPW)G-;@%@isgoG2E?`>q_ zZ)3+!V+m3RQR4&3z)EA4G@@tEH ztPm3!s8rkFK-bpRHYLl$kIOcRPkS^Jv}S$Sx zo$jb@9yFweR9xxQS?{v{?X&_z^}+4Y*Lu&#vy5|huWU)wZ^?LG;^F64di=*AhfpwX zr%A_Xo(XfH7xdlus1;OibR~Q*OFF03j?o30p11-=n&TMV#iZq@m))vU6!H0Cr9p-2 z5e1xTph)mY3|--j-!2yw09cN_=Tskxa^zU@=D0$zD!9v%mYSk1=zM}jZl0;D@8eFJZ%sVjOfneaa~onUFvOC+ zSR4R}JX+)uH32;CpYQ zA%u7V`0b-3z!KZdCH>tDs;h!;guHuJWql~`Va-Elifc)Kg;K9ckD!ot5)qfaGr$<} zr@!ZI-qAXEt8IlK++#ow%p<0!exvj?z9Hh|Qr6PCU9ixrU}mQSUfjIu^<6zzdL3fX zCWlRVfhw2@|4210%=0I+En1IQESIL z6b(9;MwQKhc?^;+Kw?llm5GJB!ft(pBOxo~03}|)@z)m%)xT9E)f)ZYWo@@lhZ~(B z;>MH}WO&WsNh-(^XvsXQ%(kc*M#Y*pQh?)K4RKrX!wIH)j1#DhLi@*BGJxWJBv{6W zi(;_Y+jbanR|$KyKo|@qVIEBhbP0~9?Bd+uk}=|@#ML~?cQFEiELzp?@`;ZNO+%zk zo_)P7+5Oh(q@T5i)78j~F5p=03GU8aOS7D0>x$nh~FD)&taZUT-NHMM<^5Im`NwJU}Xm1@VLP)MS-5@ucssUCPm|t2m zfD5XWZu;=>JxQ!99uLbd(D{42m||r))IfNe=-P{BFU%(6?%jxJX_j}!N8njH=H}*} z-o=?Z*AiE^94s*VgRHG`cv~m+kejMjS7C;yuWvlA^PSj`6!-0dnt@nPplr%v{`z7- zOC&R9&Q92|F3Ceg3n2jeE1wm%w#xfd65M8?!iHz@B=Gr64)djycj?r=9~vn0^W#?E zYS-n(kw-ct=UZJ#J!P@0uf;%El4AYT%oQ5f0+VhhRZGQ zf(GA>jw+9l%20BKE0{L_RTdPo~;=UZf%3^zi!GIO@3+!+WEV#SkN6}+?OBYgo`4J6=hhdl?W?|g{d(cQXB>`FFv06P zh^WB{|26$oh|bQOh5}~G{+pr8K$hB}6NQ_e-7>PtZP*07Mxig3#z7E(KfbUV;G1*` z*CU%i*;wbRyF^9%s#QhEI`Xjrz3}BTKFadXq!5VozhF3sD*%Q7F2>*bMghw#qRp$i zv_A`i$8+@0KG-lY!_Ves5R|a@3mU=?fk004F_y3Y#6JVz-&elGMsdF)D^^Vz>kNsn zN&e5f?#*-KP)i6=e~$<)%-lubZ_Fg37CG-KKlceHO5ndC8uWvkNF1bDp;_S{*CWt_ z*eF~S4kF90)7Xr2Q}-jj9muKcQP+#S1cstS=- esr~u<*Xb>BdCJX`4y1L@%ce&yjY = Lazy::new(|| [KeyValue::new("handler", "all")]); async fn serve_req( cx: Context, @@ -33,15 +24,17 @@ async fn serve_req( println!("Receiving request at path {}", req.uri()); let request_start = SystemTime::now(); - state.http_counter.add(&cx, 1, &[]); + state.http_counter.add(&cx, 1, HANDLER_ALL.as_ref()); let response = match (req.method(), req.uri().path()) { (&Method::GET, "/metrics") => { let mut buffer = vec![]; let encoder = TextEncoder::new(); - let metric_families = state.exporter.registry().gather(); + let metric_families = state.registry.gather(); encoder.encode(&metric_families, &mut buffer).unwrap(); - state.http_body_gauge.record(&cx, buffer.len() as u64, &[]); + state + .http_body_gauge + .record(&cx, buffer.len() as u64, HANDLER_ALL.as_ref()); Response::builder() .status(200) @@ -68,7 +61,7 @@ async fn serve_req( } struct AppState { - exporter: PrometheusExporter, + registry: Registry, http_counter: Counter, http_body_gauge: Histogram, http_req_histogram: Histogram, @@ -76,29 +69,29 @@ struct AppState { #[tokio::main] pub async fn main() -> Result<(), Box> { - let controller = controllers::basic(processors::factory( - selectors::simple::histogram([1.0, 2.0, 5.0, 10.0, 20.0, 50.0]), - aggregation::cumulative_temporality_selector(), - )) - .build(); - - let exporter = opentelemetry_prometheus::exporter(controller).init(); + let registry = Registry::new(); + let exporter = opentelemetry_prometheus::exporter() + .with_registry(registry.clone()) + .build()?; + let provider = MeterProvider::builder().with_reader(exporter).build(); let cx = Context::new(); - let meter = global::meter("ex.com/hyper"); + let meter = provider.meter("hyper-prometheus-example"); let state = Arc::new(AppState { - exporter, + registry, http_counter: meter - .u64_counter("example.http_requests_total") + .u64_counter("http_requests_total") .with_description("Total number of HTTP requests made.") .init(), http_body_gauge: meter - .u64_histogram("example.http_response_size_bytes") + .u64_histogram("example.http_response_size") + .with_unit(Unit::new("By")) .with_description("The metrics HTTP response sizes in bytes.") .init(), http_req_histogram: meter - .f64_histogram("example.http_request_duration_seconds") - .with_description("The HTTP request latencies in seconds.") + .f64_histogram("example.http_request_duration") + .with_unit(Unit::new("ms")) + .with_description("The HTTP request latencies in milliseconds.") .init(), }); diff --git a/opentelemetry-api/src/attributes/encoder.rs b/opentelemetry-api/src/attributes/encoder.rs deleted file mode 100644 index 415050b73a..0000000000 --- a/opentelemetry-api/src/attributes/encoder.rs +++ /dev/null @@ -1,76 +0,0 @@ -use crate::{Key, Value}; -use std::fmt::{self, Write}; -use std::sync::atomic::{AtomicUsize, Ordering}; - -static ENCODER_ID_COUNTER: AtomicUsize = AtomicUsize::new(0); - -/// Encoder is a mechanism for serializing an attribute set into a specific string -/// representation that supports caching, to avoid repeated serialization. An -/// example could be an exporter encoding the attribute set into a wire -/// representation. -pub trait Encoder: fmt::Debug { - /// Encode returns the serialized encoding of the attribute - /// set using its Iterator. This result may be cached. - fn encode(&self, attributes: &mut dyn Iterator) -> String; - - /// A value that is unique for each class of attribute encoder. Attribute encoders - /// allocate these using `new_encoder_id`. - fn id(&self) -> EncoderId; -} - -/// EncoderID is used to identify distinct Encoder -/// implementations, for caching encoded results. -#[derive(Debug)] -pub struct EncoderId(usize); - -impl EncoderId { - /// Check if the id is valid - pub fn is_valid(&self) -> bool { - self.0 != 0 - } -} - -/// Default attribute encoding strategy -#[derive(Debug)] -pub struct DefaultAttributeEncoder; - -impl Encoder for DefaultAttributeEncoder { - fn encode(&self, attributes: &mut dyn Iterator) -> String { - attributes - .enumerate() - .fold(String::new(), |mut acc, (idx, (key, value))| { - let offset = acc.len(); - if idx > 0 { - acc.push(',') - } - - if write!(acc, "{}", key).is_err() { - acc.truncate(offset); - return acc; - } - - acc.push('='); - if write!(acc, "{}", value).is_err() { - acc.truncate(offset); - return acc; - } - - acc - }) - } - - fn id(&self) -> EncoderId { - new_encoder_id() - } -} - -/// Build a new default encoder -pub fn default_encoder() -> Box { - Box::new(DefaultAttributeEncoder) -} - -/// Build a new encoder id -pub fn new_encoder_id() -> EncoderId { - let old_encoder_id = ENCODER_ID_COUNTER.fetch_add(1, Ordering::AcqRel); - EncoderId(old_encoder_id + 1) -} diff --git a/opentelemetry-api/src/attributes/mod.rs b/opentelemetry-api/src/attributes/mod.rs deleted file mode 100644 index 77eaebcd6a..0000000000 --- a/opentelemetry-api/src/attributes/mod.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! OpenTelemetry Attributes -use crate::{Array, Key, KeyValue, Value}; -use std::cmp::Ordering; -use std::collections::{btree_map, BTreeMap}; -use std::hash::{Hash, Hasher}; -use std::iter::Peekable; - -mod encoder; -pub use encoder::{default_encoder, new_encoder_id, DefaultAttributeEncoder, Encoder, EncoderId}; - -/// An immutable set of distinct attributes. -#[derive(Clone, Debug, Default)] -pub struct AttributeSet { - attributes: BTreeMap, -} - -impl AttributeSet { - /// Construct a new attribute set form a distinct set of attributes - pub fn from_attributes>(attributes: T) -> Self { - AttributeSet { - attributes: attributes - .into_iter() - .map(|kv| (kv.key, kv.value)) - .collect(), - } - } - - /// The attribute set length. - pub fn len(&self) -> usize { - self.attributes.len() - } - - /// Check if the set of attributes are empty. - pub fn is_empty(&self) -> bool { - self.attributes.is_empty() - } - - /// Iterate over the attribute key value pairs. - pub fn iter(&self) -> Iter<'_> { - self.into_iter() - } - - /// Encode the attribute set with the given encoder and cache the result. - pub fn encoded(&self, encoder: Option<&dyn Encoder>) -> String { - encoder.map_or_else(String::new, |encoder| encoder.encode(&mut self.iter())) - } -} - -impl<'a> IntoIterator for &'a AttributeSet { - type Item = (&'a Key, &'a Value); - type IntoIter = Iter<'a>; - - fn into_iter(self) -> Self::IntoIter { - Iter(self.attributes.iter()) - } -} -/// An iterator over the entries of a `Set`. -#[derive(Debug)] -pub struct Iter<'a>(btree_map::Iter<'a, Key, Value>); -impl<'a> Iterator for Iter<'a> { - type Item = (&'a Key, &'a Value); - - fn next(&mut self) -> Option { - self.0.next() - } -} - -/// Impl of Hash for `KeyValue` -pub fn hash_attributes<'a, H: Hasher, I: IntoIterator>( - state: &mut H, - attributes: I, -) { - for (key, value) in attributes.into_iter() { - key.hash(state); - hash_value(state, value); - } -} - -fn hash_value(state: &mut H, value: &Value) { - match value { - Value::Bool(b) => b.hash(state), - Value::I64(i) => i.hash(state), - Value::F64(f) => { - // FIXME: f64 does not impl hash, this impl may have incorrect outcomes. - f.to_bits().hash(state) - } - Value::String(s) => s.hash(state), - Value::Array(arr) => match arr { - // recursively hash array values - Array::Bool(values) => values.iter().for_each(|v| v.hash(state)), - Array::I64(values) => values.iter().for_each(|v| v.hash(state)), - Array::F64(values) => values.iter().for_each(|v| v.to_bits().hash(state)), - Array::String(values) => values.iter().for_each(|v| v.hash(state)), - }, - } -} - -/// Merge two iterators, yielding sorted results -pub fn merge_iters< - 'a, - 'b, - A: Iterator, - B: Iterator, ->( - a: A, - b: B, -) -> MergeIter<'a, 'b, A, B> { - MergeIter { - a: a.peekable(), - b: b.peekable(), - } -} - -/// Merge two iterators, sorting by key -#[derive(Debug)] -pub struct MergeIter<'a, 'b, A, B> -where - A: Iterator, - B: Iterator, -{ - a: Peekable, - b: Peekable, -} - -impl<'a, A: Iterator, B: Iterator> - Iterator for MergeIter<'a, 'a, A, B> -{ - type Item = (&'a Key, &'a Value); - fn next(&mut self) -> Option { - let which = match (self.a.peek(), self.b.peek()) { - (Some(a), Some(b)) => Some(a.0.cmp(b.0)), - (Some(_), None) => Some(Ordering::Less), - (None, Some(_)) => Some(Ordering::Greater), - (None, None) => None, - }; - - match which { - Some(Ordering::Less) => self.a.next(), - Some(Ordering::Equal) => self.a.next(), - Some(Ordering::Greater) => self.b.next(), - None => None, - } - } -} diff --git a/opentelemetry-api/src/common.rs b/opentelemetry-api/src/common.rs index e8a9df0060..835c4390d9 100644 --- a/opentelemetry-api/src/common.rs +++ b/opentelemetry-api/src/common.rs @@ -245,7 +245,7 @@ pub enum Value { } /// Wrapper for string-like values -#[derive(Clone, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct StringValue(OtelString); impl fmt::Debug for StringValue { diff --git a/opentelemetry-api/src/lib.rs b/opentelemetry-api/src/lib.rs index 1a71749889..2a07884bc8 100644 --- a/opentelemetry-api/src/lib.rs +++ b/opentelemetry-api/src/lib.rs @@ -57,10 +57,6 @@ pub mod testing; pub use common::{Array, ExportError, InstrumentationLibrary, Key, KeyValue, StringValue, Value}; -#[cfg(feature = "metrics")] -#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))] -pub mod attributes; - #[cfg(feature = "metrics")] #[cfg_attr(docsrs, doc(cfg(feature = "metrics")))] pub mod metrics; diff --git a/opentelemetry-api/src/metrics/instruments/counter.rs b/opentelemetry-api/src/metrics/instruments/counter.rs index 903dd0d5ed..fdbc392df7 100644 --- a/opentelemetry-api/src/metrics/instruments/counter.rs +++ b/opentelemetry-api/src/metrics/instruments/counter.rs @@ -1,10 +1,10 @@ use crate::{ - metrics::{InstrumentBuilder, MetricsError}, + metrics::{AsyncInstrument, AsyncInstrumentBuilder, InstrumentBuilder, MetricsError}, Context, KeyValue, }; use core::fmt; -use std::convert::TryFrom; use std::sync::Arc; +use std::{any::Any, convert::TryFrom}; /// An SDK implemented instrument that records increasing values. pub trait SyncCounter { @@ -61,22 +61,13 @@ impl TryFrom>> for Counter { } } -/// An SDK implemented async instrument that records increasing values. -pub trait AsyncCounter { - /// Records an increment to the counter. - /// - /// It is only valid to call this within a callback. If called outside of the - /// registered callback it should have no effect on the instrument, and an - /// error will be reported via the error handler. - fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]); -} - /// An async instrument that records increasing values. -pub struct ObservableCounter(Arc + Send + Sync>); +#[derive(Clone)] +pub struct ObservableCounter(Arc>); impl ObservableCounter { /// Create a new observable counter. - pub fn new(inner: Arc + Send + Sync>) -> Self { + pub fn new(inner: Arc>) -> Self { ObservableCounter(inner) } } @@ -99,32 +90,49 @@ impl ObservableCounter { pub fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { self.0.observe(cx, value, attributes) } + + /// Used for SDKs to downcast instruments in callbacks. + pub fn as_any(&self) -> Arc { + self.0.as_any() + } +} + +impl AsyncInstrument for ObservableCounter { + fn observe(&self, cx: &Context, measurement: T, attributes: &[KeyValue]) { + self.0.observe(cx, measurement, attributes) + } + + fn as_any(&self) -> Arc { + self.0.as_any() + } } -impl TryFrom>> for ObservableCounter { +impl TryFrom, u64>> for ObservableCounter { type Error = MetricsError; fn try_from( - builder: InstrumentBuilder<'_, ObservableCounter>, + builder: AsyncInstrumentBuilder<'_, ObservableCounter, u64>, ) -> Result { builder.meter.instrument_provider.u64_observable_counter( builder.name, builder.description, builder.unit, + builder.callback, ) } } -impl TryFrom>> for ObservableCounter { +impl TryFrom, f64>> for ObservableCounter { type Error = MetricsError; fn try_from( - builder: InstrumentBuilder<'_, ObservableCounter>, + builder: AsyncInstrumentBuilder<'_, ObservableCounter, f64>, ) -> Result { builder.meter.instrument_provider.f64_observable_counter( builder.name, builder.description, builder.unit, + builder.callback, ) } } diff --git a/opentelemetry-api/src/metrics/instruments/gauge.rs b/opentelemetry-api/src/metrics/instruments/gauge.rs index ad3c7945ac..cd1a38d454 100644 --- a/opentelemetry-api/src/metrics/instruments/gauge.rs +++ b/opentelemetry-api/src/metrics/instruments/gauge.rs @@ -1,24 +1,14 @@ use crate::{ - metrics::{InstrumentBuilder, MetricsError}, + metrics::{AsyncInstrument, AsyncInstrumentBuilder, MetricsError}, Context, KeyValue, }; use core::fmt; -use std::convert::TryFrom; use std::sync::Arc; - -/// An SDK implemented instrument that records independent readings. -pub trait AsyncGauge: Send + Sync { - /// Records the state of the instrument. - /// - /// It is only valid to call this within a callback. If called outside of the - /// registered callback it should have no effect on the instrument, and an - /// error will be reported via the error handler. - fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]); -} +use std::{any::Any, convert::TryFrom}; /// An instrument that records independent readings. #[derive(Clone)] -pub struct ObservableGauge(Arc>); +pub struct ObservableGauge(Arc>); impl fmt::Debug for ObservableGauge where @@ -38,50 +28,74 @@ impl ObservableGauge { /// It is only valid to call this within a callback. If called outside of the /// registered callback it should have no effect on the instrument, and an /// error will be reported via the error handler. - pub fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.observe(cx, value, attributes) + pub fn observe(&self, cx: &Context, measurement: T, attributes: &[KeyValue]) { + self.0.observe(cx, measurement, attributes) + } + + /// Used by SDKs to downcast instruments in callbacks. + pub fn as_any(&self) -> Arc { + self.0.as_any() + } +} + +impl AsyncInstrument for ObservableGauge { + fn observe(&self, cx: &Context, measurement: M, attributes: &[KeyValue]) { + self.observe(cx, measurement, attributes) + } + + fn as_any(&self) -> Arc { + self.0.as_any() } } impl ObservableGauge { /// Create a new gauge - pub fn new(inner: Arc>) -> Self { + pub fn new(inner: Arc>) -> Self { ObservableGauge(inner) } } -impl TryFrom>> for ObservableGauge { +impl TryFrom, u64>> for ObservableGauge { type Error = MetricsError; - fn try_from(builder: InstrumentBuilder<'_, ObservableGauge>) -> Result { + fn try_from( + builder: AsyncInstrumentBuilder<'_, ObservableGauge, u64>, + ) -> Result { builder.meter.instrument_provider.u64_observable_gauge( builder.name, builder.description, builder.unit, + builder.callback, ) } } -impl TryFrom>> for ObservableGauge { +impl TryFrom, f64>> for ObservableGauge { type Error = MetricsError; - fn try_from(builder: InstrumentBuilder<'_, ObservableGauge>) -> Result { + fn try_from( + builder: AsyncInstrumentBuilder<'_, ObservableGauge, f64>, + ) -> Result { builder.meter.instrument_provider.f64_observable_gauge( builder.name, builder.description, builder.unit, + builder.callback, ) } } -impl TryFrom>> for ObservableGauge { +impl TryFrom, i64>> for ObservableGauge { type Error = MetricsError; - fn try_from(builder: InstrumentBuilder<'_, ObservableGauge>) -> Result { + fn try_from( + builder: AsyncInstrumentBuilder<'_, ObservableGauge, i64>, + ) -> Result { builder.meter.instrument_provider.i64_observable_gauge( builder.name, builder.description, builder.unit, + builder.callback, ) } } diff --git a/opentelemetry-api/src/metrics/instruments/mod.rs b/opentelemetry-api/src/metrics/instruments/mod.rs index 655c66e40b..16949ab01f 100644 --- a/opentelemetry-api/src/metrics/instruments/mod.rs +++ b/opentelemetry-api/src/metrics/instruments/mod.rs @@ -1,7 +1,11 @@ use crate::metrics::{Meter, MetricsError, Result, Unit}; +use crate::{global, Context, KeyValue}; use core::fmt; +use std::any::Any; +use std::borrow::Cow; use std::convert::TryFrom; use std::marker; +use std::sync::Arc; pub(super) mod counter; pub(super) mod gauge; @@ -18,11 +22,22 @@ const INSTRUMENT_NAME_FIRST_ALPHABETIC: &str = const INSTRUMENT_UNIT_LENGTH: &str = "instrument unit must be less than 64 characters"; const INSTRUMENT_UNIT_INVALID_CHAR: &str = "characters in instrument unit must be ASCII"; -/// Configuration for building an instrument. +/// An SDK implemented instrument that records measurements via callback. +pub trait AsyncInstrument: Send + Sync { + /// Observes the state of the instrument. + /// + /// It is only valid to call this within a callback. + fn observe(&self, cx: &Context, measurement: T, attributes: &[KeyValue]); + + /// Used for SDKs to downcast instruments in callbacks. + fn as_any(&self) -> Arc; +} + +/// Configuration for building a sync instrument. pub struct InstrumentBuilder<'a, T> { meter: &'a Meter, - name: String, - description: Option, + name: Cow<'static, str>, + description: Option>, unit: Option, _marker: marker::PhantomData, } @@ -32,7 +47,7 @@ where T: TryFrom, { /// Create a new instrument builder - pub(crate) fn new(meter: &'a Meter, name: String) -> Self { + pub(crate) fn new(meter: &'a Meter, name: Cow<'static, str>) -> Self { InstrumentBuilder { meter, name, @@ -43,7 +58,7 @@ where } /// Set the description for this instrument - pub fn with_description>(mut self, description: S) -> Self { + pub fn with_description>>(mut self, description: S) -> Self { self.description = Some(description.into()); self } @@ -69,12 +84,18 @@ where /// Creates a new instrument. /// + /// This method reports configuration errors but still returns the instrument. + /// /// # Panics /// - /// This function panics if the instrument configuration is invalid or the instrument cannot be created. Use [`try_init`](InstrumentBuilder::try_init) if you want to - /// handle errors. + /// Panics if the instrument cannot be created. Use + /// [`try_init`](InstrumentBuilder::try_init) if you want to handle errors. pub fn init(self) -> T { - self.try_init().unwrap() + if let Err(err) = self.validate_instrument_config() { + global::handle_error(MetricsError::InvalidInstrumentConfiguration(err)); + } + + T::try_from(self).unwrap() } fn validate_instrument_config(&self) -> std::result::Result<(), &'static str> { @@ -119,6 +140,141 @@ impl fmt::Debug for InstrumentBuilder<'_, T> { } } +/// A function registered with a [Meter] that makes observations for the +/// instruments it is registered with. +/// +/// The async instrument parameter is used to record measurement observations +/// for these instruments. +/// +/// The function needs to complete in a finite amount of time. +pub type Callback = Box) + Send + Sync>; + +/// Configuration for building an async instrument. +pub struct AsyncInstrumentBuilder<'a, I, M> +where + I: AsyncInstrument, +{ + meter: &'a Meter, + name: Cow<'static, str>, + description: Option>, + unit: Option, + _inst: marker::PhantomData, + callback: Option>, +} + +impl<'a, I, M> AsyncInstrumentBuilder<'a, I, M> +where + I: TryFrom, + I: AsyncInstrument, +{ + /// Create a new instrument builder + pub(crate) fn new(meter: &'a Meter, name: Cow<'static, str>) -> Self { + AsyncInstrumentBuilder { + meter, + name, + description: None, + unit: None, + _inst: marker::PhantomData, + callback: None, + } + } + + /// Set the description for this instrument + pub fn with_description>>(mut self, description: S) -> Self { + self.description = Some(description.into()); + self + } + + /// Set the unit for this instrument. + /// + /// Unit is case sensitive(`kb` is not the same as `kB`). + /// + /// Unit must be: + /// - ASCII string + /// - No longer than 63 characters + pub fn with_unit(mut self, unit: Unit) -> Self { + self.unit = Some(unit); + self + } + + /// Set the callback to be called for this instrument. + pub fn with_callback(mut self, callback: F) -> Self + where + F: Fn(&Context, &dyn AsyncInstrument) + Send + Sync + 'static, + { + self.callback = Some(Box::new(callback)); + self + } + + /// Validate the instrument configuration and creates a new instrument. + pub fn try_init(self) -> Result { + self.validate_instrument_config() + .map_err(MetricsError::InvalidInstrumentConfiguration)?; + I::try_from(self) + } + + /// Creates a new instrument. + /// + /// This method reports configuration errors but still returns the instrument. + /// + /// # Panics + /// + /// Panics if the instrument cannot be created. Use + /// [`try_init`](InstrumentBuilder::try_init) if you want to handle errors. + pub fn init(self) -> I { + if let Err(err) = self.validate_instrument_config() { + global::handle_error(MetricsError::InvalidInstrumentConfiguration(err)); + } + + I::try_from(self).unwrap() + } + + fn validate_instrument_config(&self) -> std::result::Result<(), &'static str> { + // validate instrument name + if self.name.is_empty() { + return Err(INSTRUMENT_NAME_EMPTY); + } + if self.name.len() > 63 { + return Err(INSTRUMENT_NAME_LENGTH); + } + if self.name.starts_with(|c: char| !c.is_ascii_alphabetic()) { + return Err(INSTRUMENT_NAME_FIRST_ALPHABETIC); + } + if self + .name + .contains(|c: char| !c.is_ascii_alphanumeric() && c != '_' && c != '.' && c != '-') + { + return Err(INSTRUMENT_NAME_INVALID_CHAR); + } + + // validate instrument unit + if let Some(unit) = &self.unit { + if unit.as_str().len() > 63 { + return Err(INSTRUMENT_UNIT_LENGTH); + } + if unit.as_str().contains(|c: char| !c.is_ascii()) { + return Err(INSTRUMENT_UNIT_INVALID_CHAR); + } + } + Ok(()) + } +} + +impl fmt::Debug for AsyncInstrumentBuilder<'_, I, M> +where + I: AsyncInstrument, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InstrumentBuilder") + .field("name", &self.name) + .field("description", &self.description) + .field("unit", &self.unit) + .field("kind", &std::any::type_name::()) + .field("callback_present", &self.callback.is_some()) + .finish() + } +} + #[cfg(test)] mod tests { use crate::metrics::instruments::{ @@ -153,7 +309,7 @@ mod tests { ]; for (name, expected_error) in instrument_name_test_cases { let builder: InstrumentBuilder<'_, Counter> = - InstrumentBuilder::new(&meter, name.to_string()); + InstrumentBuilder::new(&meter, name.into()); if expected_error.is_empty() { assert!(builder.validate_instrument_config().is_ok()); } else { @@ -176,7 +332,7 @@ mod tests { for (unit, expected_error) in instrument_unit_test_cases { let builder: InstrumentBuilder<'_, Counter> = - InstrumentBuilder::new(&meter, "test".to_string()).with_unit(Unit::new(unit)); + InstrumentBuilder::new(&meter, "test".into()).with_unit(Unit::new(unit)); if expected_error.is_empty() { assert!(builder.validate_instrument_config().is_ok()); } else { diff --git a/opentelemetry-api/src/metrics/instruments/up_down_counter.rs b/opentelemetry-api/src/metrics/instruments/up_down_counter.rs index 40605bccaa..4eb3b63323 100644 --- a/opentelemetry-api/src/metrics/instruments/up_down_counter.rs +++ b/opentelemetry-api/src/metrics/instruments/up_down_counter.rs @@ -6,6 +6,8 @@ use core::fmt; use std::convert::TryFrom; use std::sync::Arc; +use super::{AsyncInstrument, AsyncInstrumentBuilder}; + /// An SDK implemented instrument that records increasing or decreasing values. pub trait SyncUpDownCounter { /// Records an increment or decrement to the counter. @@ -64,19 +66,9 @@ impl TryFrom>> for UpDownCounter { } } -/// An SDK implemented async instrument that records increasing or decreasing values. -pub trait AsyncUpDownCounter { - /// Records the increment or decrement to the counter. - /// - /// It is only valid to call this within a callback. If called outside of the - /// registered callback it should have no effect on the instrument, and an - /// error will be reported via the error handler. - fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]); -} - /// An async instrument that records increasing or decreasing values. #[derive(Clone)] -pub struct ObservableUpDownCounter(Arc + Send + Sync>); +pub struct ObservableUpDownCounter(Arc>); impl fmt::Debug for ObservableUpDownCounter where @@ -92,7 +84,7 @@ where impl ObservableUpDownCounter { /// Create a new observable up down counter. - pub fn new(inner: Arc + Send + Sync>) -> Self { + pub fn new(inner: Arc>) -> Self { ObservableUpDownCounter(inner) } @@ -106,28 +98,52 @@ impl ObservableUpDownCounter { } } -impl TryFrom>> for ObservableUpDownCounter { +impl AsyncInstrument for ObservableUpDownCounter { + fn observe(&self, cx: &Context, measurement: T, attributes: &[KeyValue]) { + self.0.observe(cx, measurement, attributes) + } + + fn as_any(&self) -> Arc { + self.0.as_any() + } +} + +impl TryFrom, i64>> + for ObservableUpDownCounter +{ type Error = MetricsError; fn try_from( - builder: InstrumentBuilder<'_, ObservableUpDownCounter>, + builder: AsyncInstrumentBuilder<'_, ObservableUpDownCounter, i64>, ) -> Result { builder .meter .instrument_provider - .i64_observable_up_down_counter(builder.name, builder.description, builder.unit) + .i64_observable_up_down_counter( + builder.name, + builder.description, + builder.unit, + builder.callback, + ) } } -impl TryFrom>> for ObservableUpDownCounter { +impl TryFrom, f64>> + for ObservableUpDownCounter +{ type Error = MetricsError; fn try_from( - builder: InstrumentBuilder<'_, ObservableUpDownCounter>, + builder: AsyncInstrumentBuilder<'_, ObservableUpDownCounter, f64>, ) -> Result { builder .meter .instrument_provider - .f64_observable_up_down_counter(builder.name, builder.description, builder.unit) + .f64_observable_up_down_counter( + builder.name, + builder.description, + builder.unit, + builder.callback, + ) } } diff --git a/opentelemetry-api/src/metrics/meter.rs b/opentelemetry-api/src/metrics/meter.rs index 23c4f12f3e..0a7efe8985 100644 --- a/opentelemetry-api/src/metrics/meter.rs +++ b/opentelemetry-api/src/metrics/meter.rs @@ -1,11 +1,15 @@ use core::fmt; +use std::any::Any; +use std::borrow::Cow; use std::sync::Arc; use crate::metrics::{ - Counter, Histogram, InstrumentBuilder, InstrumentProvider, MetricsError, ObservableCounter, - ObservableGauge, ObservableUpDownCounter, UpDownCounter, + AsyncInstrumentBuilder, Counter, Histogram, InstrumentBuilder, InstrumentProvider, + ObservableCounter, ObservableGauge, ObservableUpDownCounter, Result, UpDownCounter, }; -use crate::{Context, InstrumentationLibrary}; +use crate::{Context, InstrumentationLibrary, KeyValue}; + +use super::AsyncInstrument; /// Returns named meter instances pub trait MeterProvider { @@ -37,6 +41,7 @@ pub struct Meter { impl Meter { /// Create a new named meter from an instrumentation provider + #[doc(hidden)] pub fn new( instrumentation_library: InstrumentationLibrary, instrument_provider: Arc, @@ -48,35 +53,41 @@ impl Meter { } /// creates an instrument builder for recording increasing values. - pub fn u64_counter(&self, name: impl Into) -> InstrumentBuilder<'_, Counter> { + pub fn u64_counter( + &self, + name: impl Into>, + ) -> InstrumentBuilder<'_, Counter> { InstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording increasing values. - pub fn f64_counter(&self, name: impl Into) -> InstrumentBuilder<'_, Counter> { + pub fn f64_counter( + &self, + name: impl Into>, + ) -> InstrumentBuilder<'_, Counter> { InstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording increasing values via callback. pub fn u64_observable_counter( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableCounter> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableCounter, u64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording increasing values via callback. pub fn f64_observable_counter( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableCounter> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableCounter, f64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording changes of a value. pub fn i64_up_down_counter( &self, - name: impl Into, + name: impl Into>, ) -> InstrumentBuilder<'_, UpDownCounter> { InstrumentBuilder::new(self, name.into()) } @@ -84,7 +95,7 @@ impl Meter { /// creates an instrument builder for recording changes of a value. pub fn f64_up_down_counter( &self, - name: impl Into, + name: impl Into>, ) -> InstrumentBuilder<'_, UpDownCounter> { InstrumentBuilder::new(self, name.into()) } @@ -92,70 +103,126 @@ impl Meter { /// creates an instrument builder for recording changes of a value via callback. pub fn i64_observable_up_down_counter( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableUpDownCounter> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableUpDownCounter, i64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording changes of a value via callback. pub fn f64_observable_up_down_counter( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableUpDownCounter> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableUpDownCounter, f64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording the current value via callback. pub fn u64_observable_gauge( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableGauge> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableGauge, u64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording the current value via callback. pub fn i64_observable_gauge( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableGauge> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableGauge, i64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording the current value via callback. pub fn f64_observable_gauge( &self, - name: impl Into, - ) -> InstrumentBuilder<'_, ObservableGauge> { - InstrumentBuilder::new(self, name.into()) + name: impl Into>, + ) -> AsyncInstrumentBuilder<'_, ObservableGauge, f64> { + AsyncInstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording a distribution of values. - pub fn f64_histogram(&self, name: impl Into) -> InstrumentBuilder<'_, Histogram> { + pub fn f64_histogram( + &self, + name: impl Into>, + ) -> InstrumentBuilder<'_, Histogram> { InstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording a distribution of values. - pub fn u64_histogram(&self, name: impl Into) -> InstrumentBuilder<'_, Histogram> { + pub fn u64_histogram( + &self, + name: impl Into>, + ) -> InstrumentBuilder<'_, Histogram> { InstrumentBuilder::new(self, name.into()) } /// creates an instrument builder for recording a distribution of values. - pub fn i64_histogram(&self, name: impl Into) -> InstrumentBuilder<'_, Histogram> { + pub fn i64_histogram( + &self, + name: impl Into>, + ) -> InstrumentBuilder<'_, Histogram> { InstrumentBuilder::new(self, name.into()) } - /// Captures the function that will be called during data collection. + /// Registers a callback to be called during the collection of a measurement + /// cycle. + /// + /// The instruments passed as arguments to be registered are the only + /// instruments that may observe values. /// - /// It is only valid to call `observe` within the scope of the passed function. - pub fn register_callback(&self, callback: F) -> Result<(), MetricsError> + /// If no instruments are passed, the callback will not be registered. + pub fn register_callback( + &self, + instruments: &[Arc], + callback: F, + ) -> Result> where - F: Fn(&Context) + Send + Sync + 'static, + F: Fn(&Context, &dyn Observer) + Send + Sync + 'static, { self.instrument_provider - .register_callback(Box::new(callback)) + .register_callback(instruments, Box::new(callback)) } } +/// A token representing the unique registration of a callback for a set of +/// instruments with a [Meter]. +pub trait Registration { + /// Removes the callback registration from its associated [Meter]. + /// + /// This method needs to be idempotent and concurrent safe. + fn unregister(&mut self) -> Result<()>; +} + +/// Records measurements for multiple instruments in a callback. +pub trait Observer { + /// Records the f64 value with attributes for the observable. + fn observe_f64( + &self, + cx: &Context, + inst: &dyn AsyncInstrument, + measurement: f64, + attrs: &[KeyValue], + ); + + /// Records the u64 value with attributes for the observable. + fn observe_u64( + &self, + cx: &Context, + inst: &dyn AsyncInstrument, + measurement: u64, + attrs: &[KeyValue], + ); + + /// Records the i64 value with attributes for the observable. + fn observe_i64( + &self, + cx: &Context, + inst: &dyn AsyncInstrument, + measurement: i64, + attrs: &[KeyValue], + ); +} + impl fmt::Debug for Meter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Meter") diff --git a/opentelemetry-api/src/metrics/mod.rs b/opentelemetry-api/src/metrics/mod.rs index ead65f474f..706d78e2e6 100644 --- a/opentelemetry-api/src/metrics/mod.rs +++ b/opentelemetry-api/src/metrics/mod.rs @@ -1,5 +1,6 @@ //! # OpenTelemetry Metrics API +use std::any::Any; use std::result; use std::sync::PoisonError; use std::{borrow::Cow, sync::Arc}; @@ -11,15 +12,13 @@ pub mod noop; use crate::{Context, ExportError}; pub use instruments::{ - counter::{AsyncCounter, Counter, ObservableCounter, SyncCounter}, - gauge::{AsyncGauge, ObservableGauge}, + counter::{Counter, ObservableCounter, SyncCounter}, + gauge::ObservableGauge, histogram::{Histogram, SyncHistogram}, - up_down_counter::{ - AsyncUpDownCounter, ObservableUpDownCounter, SyncUpDownCounter, UpDownCounter, - }, - InstrumentBuilder, + up_down_counter::{ObservableUpDownCounter, SyncUpDownCounter, UpDownCounter}, + AsyncInstrument, AsyncInstrumentBuilder, Callback, InstrumentBuilder, }; -pub use meter::{Meter, MeterProvider}; +pub use meter::{Meter, MeterProvider, Observer, Registration}; /// A specialized `Result` type for metric operations. pub type Result = result::Result; @@ -31,30 +30,9 @@ pub enum MetricsError { /// Other errors not covered by specific cases. #[error("Metrics error: {0}")] Other(String), - /// Errors when requesting quantiles out of the 0-1 range. - #[error("The requested quantile is out of range")] - InvalidQuantile, - /// Errors when recording nan values. - #[error("NaN value is an invalid input")] - NaNInput, - /// Errors when recording negative values in monotonic sums. - #[error("Negative value is out of range for this instrument")] - NegativeInput, - /// Errors when merging aggregators of incompatible types. - #[error("Inconsistent aggregator types: {0}")] - InconsistentAggregator(String), - /// Errors when requesting data when no data has been collected - #[error("No data collected by this aggregator")] - NoDataCollected, - /// Errors when registering to instruments with the same name and kind - #[error("A metric was already registered by this name with another kind or number type: {0}")] - MetricKindMismatch(String), - /// Errors when processor logic is incorrect - #[error("Inconsistent processor state")] - InconsistentState, - /// Errors when aggregator cannot subtract - #[error("Aggregator does not subtract")] - NoSubtraction, + /// Invalid configuration + #[error("Config error {0}")] + Config(String), /// Fail to export metrics #[error("Metrics exporter {} failed with {0}", .0.exporter_name())] ExportErr(Box), @@ -108,8 +86,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording increasing values. fn u64_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(Counter::new(Arc::new(noop::NoopSyncInstrument::new()))) @@ -118,8 +96,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording increasing values. fn f64_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(Counter::new(Arc::new(noop::NoopSyncInstrument::new()))) @@ -128,9 +106,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording increasing values via callback. fn u64_observable_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableCounter::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -140,9 +119,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording increasing values via callback. fn f64_observable_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableCounter::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -152,8 +132,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording changes of a value. fn i64_up_down_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(UpDownCounter::new( @@ -164,8 +144,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording changes of a value. fn f64_up_down_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(UpDownCounter::new( @@ -176,9 +156,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording changes of a value. fn i64_observable_up_down_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableUpDownCounter::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -188,9 +169,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording changes of a value via callback. fn f64_observable_up_down_counter( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableUpDownCounter::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -200,9 +182,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording the current value via callback. fn u64_observable_gauge( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableGauge::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -212,9 +195,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording the current value via callback. fn i64_observable_gauge( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableGauge::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -224,9 +208,10 @@ pub trait InstrumentProvider { /// creates an instrument for recording the current value via callback. fn f64_observable_gauge( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, + _callback: Option>, ) -> Result> { Ok(ObservableGauge::new(Arc::new( noop::NoopAsyncInstrument::new(), @@ -236,8 +221,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording a distribution of values. fn f64_histogram( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(Histogram::new(Arc::new(noop::NoopSyncInstrument::new()))) @@ -246,8 +231,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording a distribution of values. fn u64_histogram( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(Histogram::new(Arc::new(noop::NoopSyncInstrument::new()))) @@ -256,8 +241,8 @@ pub trait InstrumentProvider { /// creates an instrument for recording a distribution of values. fn i64_histogram( &self, - _name: String, - _description: Option, + _name: Cow<'static, str>, + _description: Option>, _unit: Option, ) -> Result> { Ok(Histogram::new(Arc::new(noop::NoopSyncInstrument::new()))) @@ -266,5 +251,11 @@ pub trait InstrumentProvider { /// Captures the function that will be called during data collection. /// /// It is only valid to call `observe` within the scope of the passed function. - fn register_callback(&self, callback: Box) -> Result<()>; + fn register_callback( + &self, + instruments: &[Arc], + callback: Box, + ) -> Result>; } + +type MultiInstrumentCallback = dyn Fn(&Context, &dyn Observer) + Send + Sync; diff --git a/opentelemetry-api/src/metrics/noop.rs b/opentelemetry-api/src/metrics/noop.rs index ae8a1ae88f..a8440819e7 100644 --- a/opentelemetry-api/src/metrics/noop.rs +++ b/opentelemetry-api/src/metrics/noop.rs @@ -5,12 +5,13 @@ //! to have minimal resource utilization and runtime impact. use crate::{ metrics::{ - AsyncCounter, AsyncGauge, AsyncUpDownCounter, InstrumentProvider, Meter, MeterProvider, - Result, SyncCounter, SyncHistogram, SyncUpDownCounter, + AsyncInstrument, InstrumentProvider, Meter, MeterProvider, Observer, Registration, Result, + SyncCounter, SyncHistogram, SyncUpDownCounter, }, Context, InstrumentationLibrary, KeyValue, }; -use std::sync::Arc; +use std::{any::Any, sync::Arc}; + /// A no-op instance of a `MetricProvider` #[derive(Debug, Default)] pub struct NoopMeterProvider { @@ -50,7 +51,30 @@ impl NoopMeterCore { } impl InstrumentProvider for NoopMeterCore { - fn register_callback(&self, _callback: Box) -> Result<()> { + fn register_callback( + &self, + _instruments: &[Arc], + _callback: Box, + ) -> Result> { + Ok(Box::new(NoopRegistration::new())) + } +} + +/// A no-op instance of a callback [Registration]. +#[derive(Debug, Default)] +pub struct NoopRegistration { + _private: (), +} + +impl NoopRegistration { + /// Create a new no-op registration. + pub fn new() -> Self { + NoopRegistration { _private: () } + } +} + +impl Registration for NoopRegistration { + fn unregister(&mut self) -> Result<()> { Ok(()) } } @@ -99,20 +123,12 @@ impl NoopAsyncInstrument { } } -impl AsyncGauge for NoopAsyncInstrument { - fn observe(&self, _cx: &Context, _value: T, _attributes: &[KeyValue]) { - // Ignored - } -} - -impl AsyncCounter for NoopAsyncInstrument { +impl AsyncInstrument for NoopAsyncInstrument { fn observe(&self, _cx: &Context, _value: T, _attributes: &[KeyValue]) { // Ignored } -} -impl AsyncUpDownCounter for NoopAsyncInstrument { - fn observe(&self, _cx: &Context, _value: T, _attributes: &[KeyValue]) { - // Ignored + fn as_any(&self) -> Arc { + Arc::new(()) } } diff --git a/opentelemetry-dynatrace/Cargo.toml b/opentelemetry-dynatrace/Cargo.toml index 439e188aa1..c296fa879d 100644 --- a/opentelemetry-dynatrace/Cargo.toml +++ b/opentelemetry-dynatrace/Cargo.toml @@ -57,8 +57,8 @@ getrandom = { version = "0.2", optional = true } http = "0.2" isahc = { version = "1.4", default-features = false, optional = true } js-sys = { version = "0.3.5", optional = true } -opentelemetry = { version = "0.18", path = "../opentelemetry", default-features = false } -opentelemetry-http = { version = "0.7", path = "../opentelemetry-http", default-features = false } +opentelemetry = { version = "0.18", default-features = false } +opentelemetry-http = { version = "0.7", default-features = false } reqwest = { version = "0.11", default-features = false, optional = true } surf = { version = "2.0", default-features = false, optional = true } thiserror = "1.0" @@ -80,6 +80,6 @@ features = [ optional = true [dev-dependencies] -opentelemetry = { path = "../opentelemetry", features = ["rt-tokio"] } +opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread", "sync", "test-util"] } hyper = { version = "0.14", default-features = false, features = ["server", "tcp", "http1"] } diff --git a/opentelemetry-otlp/Cargo.toml b/opentelemetry-otlp/Cargo.toml index b1e9e7165d..b33630f84a 100644 --- a/opentelemetry-otlp/Cargo.toml +++ b/opentelemetry-otlp/Cargo.toml @@ -36,7 +36,8 @@ futures-util = { version = "0.3", default-features = false, features = ["std"] } opentelemetry-proto = { version = "0.2", path = "../opentelemetry-proto", default-features = false } grpcio = { version = "0.12", optional = true} -opentelemetry = { version = "0.18", default-features = false, features = ["trace"], path = "../opentelemetry" } +opentelemetry_api = { version = "0.18", default-features = false, path = "../opentelemetry-api" } +opentelemetry_sdk = { version = "0.18", default-features = false, path = "../opentelemetry-sdk" } opentelemetry-http = { version = "0.7", path = "../opentelemetry-http", optional = true } protobuf = { version = "2.18", optional = true } @@ -53,14 +54,14 @@ thiserror = "1.0" [dev-dependencies] tokio-stream = { version = "0.1", features = ["net"] } # need tokio runtime to run smoke tests. -opentelemetry = { features = ["trace", "rt-tokio", "testing"], path = "../opentelemetry" } +opentelemetry_sdk = { features = ["trace", "rt-tokio", "testing"], path = "../opentelemetry-sdk" } time = { version = "0.3", features = ["macros"] } tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } [features] # telemetry pillars and functions -trace = ["opentelemetry/trace", "opentelemetry-proto/traces"] -metrics = ["opentelemetry/metrics", "opentelemetry-proto/metrics", "grpc-tonic"] +trace = ["opentelemetry_api/trace", "opentelemetry_sdk/trace", "opentelemetry-proto/traces"] +metrics = ["opentelemetry_api/metrics", "opentelemetry_sdk/metrics", "opentelemetry-proto/metrics", "grpc-tonic"] # add ons serialize = ["serde"] diff --git a/opentelemetry-otlp/src/lib.rs b/opentelemetry-otlp/src/lib.rs index 97b644e260..4a7de0bba9 100644 --- a/opentelemetry-otlp/src/lib.rs +++ b/opentelemetry-otlp/src/lib.rs @@ -85,10 +85,10 @@ //! on the choice of exporters. //! //! ```no_run -//! use opentelemetry::{KeyValue, trace::Tracer}; -//! use opentelemetry::sdk::{trace::{self, RandomIdGenerator, Sampler}, Resource}; -//! use opentelemetry::sdk::metrics::{selectors, PushController}; -//! use opentelemetry::sdk::util::tokio_interval_stream; +//! use opentelemetry_api::{KeyValue, trace::Tracer}; +//! use opentelemetry_sdk::{trace::{self, RandomIdGenerator, Sampler}, Resource}; +//! use opentelemetry_sdk::metrics::{selectors, PushController}; +//! use opentelemetry_sdk::util::tokio_interval_stream; //! use opentelemetry_otlp::{Protocol, WithExportConfig, ExportConfig}; //! use std::time::Duration; //! use tonic::metadata::*; @@ -207,7 +207,7 @@ pub use crate::exporter::{ OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT, }; -use opentelemetry::sdk::export::ExportError; +use opentelemetry_sdk::export::ExportError; #[cfg(feature = "metrics")] use std::time::{Duration, SystemTime, UNIX_EPOCH}; diff --git a/opentelemetry-otlp/src/metric.rs b/opentelemetry-otlp/src/metric.rs index a55cbfcb9d..5787a6c928 100644 --- a/opentelemetry-otlp/src/metric.rs +++ b/opentelemetry-otlp/src/metric.rs @@ -8,30 +8,31 @@ use crate::exporter::{ tonic::{TonicConfig, TonicExporterBuilder}, ExportConfig, }; -use crate::transform::{record_to_metric, sink, CheckpointedMetrics}; +use crate::transform::sink; use crate::{Error, OtlpPipeline}; +use async_trait::async_trait; use core::fmt; -use opentelemetry::{global, metrics::Result, runtime::Runtime}; -use opentelemetry::{ - sdk::{ - export::metrics::{ - self, - aggregation::{AggregationKind, Temporality, TemporalitySelector}, - AggregatorSelector, InstrumentationLibraryReader, - }, - metrics::{ - controllers::{self, BasicController}, - processors, - sdk_api::Descriptor, - }, - Resource, - }, - Context, +use opentelemetry_api::{ + global, + metrics::{MetricsError, Result}, }; #[cfg(feature = "grpc-tonic")] use opentelemetry_proto::tonic::collector::metrics::v1::{ metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest, }; +use opentelemetry_sdk::{ + metrics::{ + data::{ResourceMetrics, Temporality}, + exporter::PushMetricsExporter, + reader::{ + AggregationSelector, DefaultAggregationSelector, DefaultTemporalitySelector, + TemporalitySelector, + }, + Aggregation, InstrumentKind, MeterProvider, PeriodicReader, + }, + runtime::Runtime, + Resource, +}; use std::fmt::{Debug, Formatter}; #[cfg(feature = "grpc-tonic")] use std::str::FromStr; @@ -53,21 +54,14 @@ pub const OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_METRICS impl OtlpPipeline { /// Create a OTLP metrics pipeline. - pub fn metrics( - self, - aggregator_selector: AS, - temporality_selector: TS, - rt: RT, - ) -> OtlpMetricPipeline + pub fn metrics(self, rt: RT) -> OtlpMetricPipeline where - AS: AggregatorSelector, - TS: TemporalitySelector + Clone, RT: Runtime, { OtlpMetricPipeline { rt, - aggregator_selector, - temporality_selector, + aggregator_selector: None, + temporality_selector: None, exporter_pipeline: None, resource: None, period: None, @@ -89,7 +83,8 @@ impl MetricsExporterBuilder { /// Build a OTLP metrics exporter with given configuration. pub fn build_metrics_exporter( self, - temporality_selector: Box, + temporality_selector: Box, + aggregation_selector: Box, ) -> Result { match self { #[cfg(feature = "grpc-tonic")] @@ -97,6 +92,7 @@ impl MetricsExporterBuilder { builder.exporter_config, builder.tonic_config, temporality_selector, + aggregation_selector, )?), } } @@ -112,20 +108,18 @@ impl From for MetricsExporterBuilder { /// /// Note that currently the OTLP metrics exporter only supports tonic as it's grpc layer and tokio as /// runtime. -pub struct OtlpMetricPipeline { +pub struct OtlpMetricPipeline { rt: RT, - aggregator_selector: AS, - temporality_selector: TS, + aggregator_selector: Option>, + temporality_selector: Option>, exporter_pipeline: Option, resource: Option, period: Option, timeout: Option, } -impl OtlpMetricPipeline +impl OtlpMetricPipeline where - AS: AggregatorSelector + Send + Sync + 'static, - TS: TemporalitySelector + Clone + Send + Sync + 'static, RT: Runtime, { /// Build with resource key value pairs. @@ -160,38 +154,60 @@ where } } + /// Build with the given temporality selector + pub fn with_temporality_selector(self, selector: T) -> Self { + OtlpMetricPipeline { + temporality_selector: Some(Box::new(selector)), + ..self + } + } + + /// Build with the given aggregation selector + pub fn with_aggregation_selector(self, selector: T) -> Self { + OtlpMetricPipeline { + aggregator_selector: Some(Box::new(selector)), + ..self + } + } + /// Build push controller. - pub fn build(self) -> Result { + pub fn build(self) -> Result { let exporter = self .exporter_pipeline .ok_or(Error::NoExporterBuilder)? - .build_metrics_exporter(Box::new(self.temporality_selector.clone()))?; + .build_metrics_exporter( + self.temporality_selector + .unwrap_or_else(|| Box::new(DefaultTemporalitySelector::new())), + self.aggregator_selector + .unwrap_or_else(|| Box::new(DefaultAggregationSelector::new())), + )?; + + let mut builder = PeriodicReader::builder(exporter, self.rt); - let mut builder = controllers::basic(processors::factory( - self.aggregator_selector, - self.temporality_selector, - )) - .with_exporter(exporter); if let Some(period) = self.period { - builder = builder.with_collect_period(period); + builder = builder.with_interval(period); } if let Some(timeout) = self.timeout { - builder = builder.with_collect_timeout(timeout) + builder = builder.with_timeout(timeout) } + + let reader = builder.build(); + + let mut provider = MeterProvider::builder().with_reader(reader); + if let Some(resource) = self.resource { - builder = builder.with_resource(resource); + provider = provider.with_resource(resource); } - let controller = builder.build(); - controller.start(&Context::current(), self.rt)?; + let provider = provider.build(); - global::set_meter_provider(controller.clone()); + global::set_meter_provider(provider.clone()); - Ok(controller) + Ok(provider) } } -impl fmt::Debug for OtlpMetricPipeline { +impl fmt::Debug for OtlpMetricPipeline { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("OtlpMetricPipeline") .field("exporter_pipeline", &self.exporter_pipeline) @@ -212,7 +228,8 @@ enum ExportMsg { pub struct MetricsExporter { #[cfg(feature = "tokio")] sender: Mutex>, - temporality_selector: Box, + temporality_selector: Box, + aggregation_selector: Box, metadata: Option, } @@ -226,8 +243,14 @@ impl Debug for MetricsExporter { } impl TemporalitySelector for MetricsExporter { - fn temporality_for(&self, descriptor: &Descriptor, kind: &AggregationKind) -> Temporality { - self.temporality_selector.temporality_for(descriptor, kind) + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.temporality_selector.temporality(kind) + } +} + +impl AggregationSelector for MetricsExporter { + fn aggregation(&self, kind: InstrumentKind) -> Aggregation { + self.aggregation_selector.aggregation(kind) } } @@ -237,7 +260,8 @@ impl MetricsExporter { pub fn new( config: ExportConfig, mut tonic_config: TonicConfig, - temporality_selector: Box, + temporality_selector: Box, + aggregation_selector: Box, ) -> Result { let endpoint = match std::env::var(OTEL_EXPORTER_OTLP_METRICS_ENDPOINT) { Ok(val) => val, @@ -286,28 +310,16 @@ impl MetricsExporter { Ok(MetricsExporter { sender: Mutex::new(sender), temporality_selector, + aggregation_selector, metadata: tonic_config.metadata.take(), }) } } -impl metrics::MetricsExporter for MetricsExporter { - fn export( - &self, - _cx: &Context, - res: &Resource, - reader: &dyn InstrumentationLibraryReader, - ) -> Result<()> { - let mut resource_metrics: Vec = Vec::default(); - // transform the metrics into proto. Append the resource and instrumentation library information into it. - reader.try_for_each(&mut |library, record| { - record.try_for_each(self, &mut |record| { - let metrics = record_to_metric(record, self.temporality_selector.as_ref())?; - resource_metrics.push((res.clone().into(), library.clone(), metrics)); - Ok(()) - }) - })?; - let mut request = Request::new(sink(resource_metrics)); +#[async_trait] +impl PushMetricsExporter for MetricsExporter { + async fn export(&self, metrics: &mut ResourceMetrics) -> Result<()> { + let mut request = Request::new(sink(metrics)); if let Some(metadata) = &self.metadata { for key_and_value in metadata.iter() { match key_and_value { @@ -328,12 +340,16 @@ impl metrics::MetricsExporter for MetricsExporter { .map_err(|_| Error::PoisonedLock("otlp metric exporter's tonic sender"))?; Ok(()) } -} -impl Drop for MetricsExporter { - fn drop(&mut self) { - let _sender_lock_guard = self.sender.lock().map(|sender| { - let _ = sender.try_send(ExportMsg::Shutdown); - }); + async fn force_flush(&self) -> Result<()> { + // this component is stateless + Ok(()) + } + + async fn shutdown(&self) -> Result<()> { + let sender = self.sender.lock()?; + sender + .try_send(ExportMsg::Shutdown) + .map_err(|e| MetricsError::Other(format!("error shutting down otlp {e}"))) } } diff --git a/opentelemetry-otlp/src/span.rs b/opentelemetry-otlp/src/span.rs index bad484a175..6ad9045cfe 100644 --- a/opentelemetry-otlp/src/span.rs +++ b/opentelemetry-otlp/src/span.rs @@ -53,15 +53,15 @@ use {std::collections::HashMap, std::sync::Arc}; use crate::exporter::ExportConfig; use crate::OtlpPipeline; -use opentelemetry::{ +use opentelemetry_api::{ global, - sdk::{ - self, - export::trace::{ExportResult, SpanData}, - trace::TraceRuntime, - }, trace::{TraceError, TracerProvider}, }; +use opentelemetry_sdk::{ + self as sdk, + export::trace::{ExportResult, SpanData}, + trace::TraceRuntime, +}; use async_trait::async_trait; @@ -122,7 +122,7 @@ impl OtlpTracePipeline { /// /// Returns a [`Tracer`] with the name `opentelemetry-otlp` and current crate version. /// - /// [`Tracer`]: opentelemetry::trace::Tracer + /// [`Tracer`]: opentelemetry_api::trace::Tracer pub fn install_simple(self) -> Result { Ok(build_simple_with_exporter( self.exporter_builder @@ -139,7 +139,7 @@ impl OtlpTracePipeline { /// /// `install_batch` will panic if not called within a tokio runtime /// - /// [`Tracer`]: opentelemetry::trace::Tracer + /// [`Tracer`]: opentelemetry_api::trace::Tracer pub fn install_batch( self, runtime: R, @@ -496,7 +496,7 @@ async fn http_send_request( } #[async_trait] -impl opentelemetry::sdk::export::trace::SpanExporter for SpanExporter { +impl opentelemetry_sdk::export::trace::SpanExporter for SpanExporter { fn export( &mut self, batch: Vec, diff --git a/opentelemetry-otlp/src/transform/metrics.rs b/opentelemetry-otlp/src/transform/metrics.rs index 2f4d8023d7..27bdfedc89 100644 --- a/opentelemetry-otlp/src/transform/metrics.rs +++ b/opentelemetry-otlp/src/transform/metrics.rs @@ -1,735 +1,190 @@ #[cfg(feature = "grpc-tonic")] -// The prost currently will generate a non optional deprecated field for labels. -// We cannot assign value to it otherwise clippy will complain. -// We cannot ignore it as it's not an optional field. -// We can remove this after we removed the labels field from proto. -#[allow(deprecated)] pub(crate) mod tonic { - use opentelemetry::metrics::MetricsError; - use opentelemetry::sdk::export::metrics::{ - aggregation::{ - Count, Histogram as SdkHistogram, LastValue, Max, Min, Sum as SdkSum, - TemporalitySelector, - }, - Record, - }; - use opentelemetry::sdk::metrics::aggregators::{ - HistogramAggregator, LastValueAggregator, SumAggregator, - }; - use opentelemetry::sdk::InstrumentationLibrary; - use opentelemetry_proto::tonic::metrics::v1::DataPointFlags; - use opentelemetry_proto::tonic::FromNumber; + use std::any::Any; + use std::fmt; + + use opentelemetry_api::{global, metrics::MetricsError}; + use opentelemetry_proto::tonic::common::v1::InstrumentationScope as TonicInstrumentationScope; + use opentelemetry_proto::tonic::resource::v1::Resource as TonicResource; use opentelemetry_proto::tonic::{ collector::metrics::v1::ExportMetricsServiceRequest, - common::v1::KeyValue, metrics::v1::{ - metric::Data, number_data_point, AggregationTemporality, Gauge, Histogram, - HistogramDataPoint, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, + exemplar::Value as TonicExemplarValue, metric::Data as TonicMetricData, + number_data_point::Value as TonicDataPointValue, + AggregationTemporality as TonicTemporality, DataPointFlags as TonicDataPointFlags, + Exemplar as TonicExemplar, Gauge as TonicGauge, Histogram as TonicHistogram, + HistogramDataPoint as TonicHistogramDataPoint, Metric as TonicMetric, + NumberDataPoint as TonicNumberDataPoint, ResourceMetrics as TonicResourceMetrics, + ScopeMetrics as TonicScopeMetrics, Sum as TonicSum, }, }; + use opentelemetry_sdk::metrics::data::{ + self, Exemplar as SdkExemplar, Gauge as SdkGauge, Histogram as SdkHistogram, + Metric as SdkMetric, ScopeMetrics as SdkScopeMetrics, Sum as SdkSum, + }; + use opentelemetry_sdk::Resource as SdkResource; use crate::to_nanos; - use crate::transform::{CheckpointedMetrics, ResourceWrapper}; - use std::collections::{BTreeMap, HashMap}; - pub(crate) fn record_to_metric( - record: &Record, - temporality_selector: &dyn TemporalitySelector, - ) -> Result { - let descriptor = record.descriptor(); - let aggregator = record.aggregator().ok_or(MetricsError::NoDataCollected)?; - let attributes = record - .attributes() - .iter() - .map(|kv| kv.into()) - .collect::>(); - let temporality: AggregationTemporality = temporality_selector - .temporality_for(descriptor, aggregator.aggregation().kind()) - .into(); - let kind = descriptor.number_kind(); - Ok(Metric { - name: descriptor.name().to_string(), - description: descriptor.description().cloned().unwrap_or_default(), - unit: descriptor.unit().unwrap_or("").to_string(), - data: { - if let Some(last_value) = aggregator.as_any().downcast_ref::() - { - Some({ - let (val, sample_time) = last_value.last_value()?; - Data::Gauge(Gauge { - data_points: vec![NumberDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes, - start_time_unix_nano: to_nanos(*record.start_time()), - time_unix_nano: to_nanos(sample_time), - value: Some(number_data_point::Value::from_number(val, kind)), - exemplars: Vec::default(), - }], - }) - }) - } else if let Some(sum) = aggregator.as_any().downcast_ref::() { - Some({ - let val = sum.sum()?; - Data::Sum(Sum { - data_points: vec![NumberDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes, - start_time_unix_nano: to_nanos(*record.start_time()), - time_unix_nano: to_nanos(*record.end_time()), - value: Some(number_data_point::Value::from_number(val, kind)), - exemplars: Vec::default(), - }], - aggregation_temporality: temporality as i32, - is_monotonic: descriptor.instrument_kind().monotonic(), - }) - }) - } else if let Some(histogram) = - aggregator.as_any().downcast_ref::() - { - Some({ - let (sum, count, min, max, buckets) = ( - histogram.sum()?, - histogram.count()?, - histogram.min()?, - histogram.max()?, - histogram.histogram()?, - ); - Data::Histogram(Histogram { - data_points: vec![HistogramDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes, - start_time_unix_nano: to_nanos(*record.start_time()), - time_unix_nano: to_nanos(*record.end_time()), - count, - sum: Some(sum.to_f64(kind)), - min: Some(min.to_f64(kind)), - max: Some(max.to_f64(kind)), - bucket_counts: buckets - .counts() - .iter() - .cloned() - .map(|c| c as u64) - .collect(), - explicit_bounds: buckets.boundaries().clone(), - exemplars: Vec::default(), - }], - aggregation_temporality: temporality as i32, - }) - }) - } else { - None - } - }, - }) + pub(crate) fn sink(metrics: &data::ResourceMetrics) -> ExportMetricsServiceRequest { + ExportMetricsServiceRequest { + resource_metrics: vec![TonicResourceMetrics { + resource: transform_resource(&metrics.resource), + scope_metrics: transform_scope_metrics(&metrics.scope_metrics), + schema_url: metrics + .resource + .schema_url() + .map(Into::into) + .unwrap_or_default(), + }], + } } - // Group metrics with resources and instrumentation libraries with resources first, - // then instrumentation libraries. - #[allow(clippy::map_entry)] // caused by https://github.com/rust-lang/rust-clippy/issues/4674 - pub(crate) fn sink(metrics: Vec) -> ExportMetricsServiceRequest { - let mut sink_map = BTreeMap::< - ResourceWrapper, - HashMap>, - >::new(); - for (resource, instrumentation_library, metric) in metrics { - if sink_map.contains_key(&resource) { - // found resource, see if we can find instrumentation library - sink_map.entry(resource).and_modify(|map| { - if map.contains_key(&instrumentation_library) { - map.entry(instrumentation_library).and_modify(|map| { - if map.contains_key(&metric.name) { - map.entry(metric.name.clone()) - .and_modify(|base| merge(base, metric)); - } else { - map.insert(metric.name.clone(), metric); - } - }); - } else { - map.insert(instrumentation_library, { - let mut map = HashMap::new(); - map.insert(metric.name.clone(), metric); - map - }); - } - }); - } else { - // insert resource -> instrumentation library -> metrics - sink_map.insert(resource, { - let mut map = HashMap::new(); - map.insert(instrumentation_library, { - let mut map = HashMap::new(); - map.insert(metric.name.clone(), metric); - map - }); - map - }); - } + fn transform_resource(r: &SdkResource) -> Option { + if r.is_empty() { + return None; } - // convert resource -> instrumentation library -> [metrics] into proto struct ResourceMetric - ExportMetricsServiceRequest { - resource_metrics: sink_map - .into_iter() - .map(|(resource, metric_map)| ResourceMetrics { - schema_url: resource - .schema_url() - .map(|s| s.to_string()) - .unwrap_or_default(), - resource: Some(resource.into()), - scope_metrics: metric_map - .into_iter() - .map(|(instrumentation_library, metrics)| ScopeMetrics { - schema_url: instrumentation_library - .schema_url - .clone() - .unwrap_or_default() - .to_string(), - scope: Some(instrumentation_library.into()), - metrics: metrics.into_values().collect::>(), // collect values - }) - .collect::>(), - }) - .collect::>(), - } + Some(TonicResource { + attributes: r.iter().map(Into::into).collect(), + dropped_attributes_count: 0, + }) } - // if the data points are the compatible, merge, otherwise do nothing - macro_rules! merge_compatible_type { - ($base: ident, $other: ident, - $ ( - $t:path => $($other_t: path),* - ) ; *) => { - match &mut $base.data { - $( - Some($t(base_data)) => { - match $other.data { - $( - Some($other_t(other_data)) => { - if other_data.data_points.len() > 0 { - base_data.data_points.extend(other_data.data_points); - } - }, - )* - _ => {} - } - }, - )* - _ => {} - } - }; + fn transform_scope_metrics(sms: &[SdkScopeMetrics]) -> Vec { + sms.iter() + .map(|sm| TonicScopeMetrics { + scope: Some(TonicInstrumentationScope::from(&sm.scope)), + metrics: transform_metrics(&sm.metrics), + schema_url: sm + .scope + .schema_url + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + }) + .collect() } - // Merge `other` metric proto struct into base by append its data point. - // If two metric proto don't have the same type or name, do nothing - pub(crate) fn merge(base: &mut Metric, other: Metric) { - if base.name != other.name { - return; - } - merge_compatible_type!(base, other, - Data::Sum => Data::Sum; - Data::Gauge => Data::Sum, Data::Gauge; - Data::Histogram => Data::Histogram; - Data::Summary => Data::Summary - ); + fn transform_metrics(metrics: &[SdkMetric]) -> Vec { + metrics + .iter() + .map(|metric| TonicMetric { + name: metric.name.to_string(), + description: metric.description.to_string(), + unit: metric.unit.as_str().to_string(), + data: transform_data(metric.data.as_any()), + }) + .collect() } -} - -#[cfg(test)] -#[allow(deprecated)] -mod tests { - #[cfg(feature = "grpc-tonic")] - mod tonic { - use crate::transform::metrics::tonic::merge; - use crate::transform::{record_to_metric, sink, ResourceWrapper}; - use opentelemetry::attributes::AttributeSet; - use opentelemetry::metrics::MetricsError; - use opentelemetry::sdk::export::metrics::aggregation::cumulative_temporality_selector; - use opentelemetry::sdk::export::metrics::record; - use opentelemetry::sdk::metrics::aggregators::{ - histogram, last_value, Aggregator, SumAggregator, - }; - use opentelemetry::sdk::metrics::sdk_api::{ - Descriptor, InstrumentKind, Number, NumberKind, - }; - use opentelemetry::sdk::{InstrumentationLibrary, Resource}; - use opentelemetry::Context; - use opentelemetry_proto::tonic::metrics::v1::DataPointFlags; - use opentelemetry_proto::tonic::{ - common::v1::{any_value, AnyValue, KeyValue}, - metrics::v1::{ - metric::Data, number_data_point, Gauge, Histogram, HistogramDataPoint, Metric, - NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, - }, - Attributes, FromNumber, - }; - use std::cmp::Ordering; - use std::sync::Arc; - use time::macros::datetime; - - fn key_value(key: &str, value: &str) -> KeyValue { - KeyValue { - key: key.to_string(), - value: Some(AnyValue { - value: Some(any_value::Value::StringValue(value.to_string())), - }), - } - } - - fn i64_to_value(val: i64) -> number_data_point::Value { - number_data_point::Value::AsInt(val) - } - #[allow(clippy::type_complexity)] - fn get_metric_with_name( - name: &'static str, - data_points: Vec<(Vec<(&'static str, &'static str)>, u64, u64, i64)>, - ) -> Metric { - Metric { - name: name.to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Gauge(Gauge { - data_points: data_points - .into_iter() - .map(|(attributes, start_time, end_time, value)| { - get_int_data_point(attributes, start_time, end_time, value) - }) - .collect::>(), - })), - } + fn transform_data(data: &dyn Any) -> Option { + if let Some(hist) = data.downcast_ref::>() { + Some(TonicMetricData::Histogram(transform_histogram(hist))) + } else if let Some(hist) = data.downcast_ref::>() { + Some(TonicMetricData::Histogram(transform_histogram(hist))) + } else if let Some(hist) = data.downcast_ref::>() { + Some(TonicMetricData::Histogram(transform_histogram(hist))) + } else if let Some(sum) = data.downcast_ref::>() { + Some(TonicMetricData::Sum(transform_sum(sum))) + } else if let Some(sum) = data.downcast_ref::>() { + Some(TonicMetricData::Sum(transform_sum(sum))) + } else if let Some(sum) = data.downcast_ref::>() { + Some(TonicMetricData::Sum(transform_sum(sum))) + } else if let Some(gauge) = data.downcast_ref::>() { + Some(TonicMetricData::Gauge(transform_gauge(gauge))) + } else if let Some(gauge) = data.downcast_ref::>() { + Some(TonicMetricData::Gauge(transform_gauge(gauge))) + } else if let Some(gauge) = data.downcast_ref::>() { + Some(TonicMetricData::Gauge(transform_gauge(gauge))) + } else { + global::handle_error(MetricsError::Other("unknown aggregator".into())); + None } + } - fn get_int_data_point( - attributes: Vec<(&'static str, &'static str)>, - start_time: u64, - end_time: u64, - value: i64, - ) -> NumberDataPoint { - NumberDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes: attributes - .into_iter() - .map(|(key, value)| key_value(key, value)) - .collect::>(), - start_time_unix_nano: start_time, - time_unix_nano: end_time, - value: Some(number_data_point::Value::from_number( - value.into(), - &NumberKind::I64, - )), - exemplars: vec![], - } - } - - type InstrumentationLibraryKv = (&'static str, Option<&'static str>); - type ResourceKv = Vec<(&'static str, &'static str)>; - type MetricRaw = (&'static str, Vec); - type DataPointRaw = (Vec<(&'static str, &'static str)>, u64, u64, i64); - - fn convert_to_resource_metrics( - data: (ResourceKv, Vec<(InstrumentationLibraryKv, Vec)>), - ) -> opentelemetry_proto::tonic::metrics::v1::ResourceMetrics { - // convert to proto resource - let attributes: Attributes = data - .0 - .into_iter() - .map(|(k, v)| opentelemetry::KeyValue::new(k.to_string(), v.to_string())) - .collect::>() - .into(); - let resource = opentelemetry_proto::tonic::resource::v1::Resource { - attributes: attributes.0, - dropped_attributes_count: 0, - }; - let mut scope_metrics = vec![]; - for ((instrumentation_name, instrumentation_version), metrics) in data.1 { - scope_metrics.push(ScopeMetrics { - scope: Some( - opentelemetry_proto::tonic::common::v1::InstrumentationScope { - name: instrumentation_name.to_string(), - attributes: Vec::new(), - version: instrumentation_version.unwrap_or("").to_string(), - dropped_attributes_count: 0, - }, - ), - schema_url: "".to_string(), - metrics: metrics - .into_iter() - .map(|(name, data_points)| get_metric_with_name(name, data_points)) - .collect::>(), - }); - } - ResourceMetrics { - resource: Some(resource), - schema_url: "".to_string(), - scope_metrics, - } - } - - // Assert two ResourceMetrics are equal. The challenge here is vectors in ResourceMetrics should - // be compared as unordered list/set. The currently method sort the input stably, and compare the - // instance one by one. - // - // Based on current implementation of sink function. There are two parts where the order is unknown. - // The first one is instrumentation_library_metrics in ResourceMetrics. - // The other is metrics in ScopeMetrics. - // - // If we changed the sink function to process the input in parallel, we will have to sort other vectors - // like data points in Metrics. - fn assert_resource_metrics(mut expect: ResourceMetrics, mut actual: ResourceMetrics) { - assert_eq!( - expect - .resource - .as_mut() - .map(|r| r.attributes.sort_by_key(|kv| kv.key.to_string())), - actual - .resource - .as_mut() - .map(|r| r.attributes.sort_by_key(|kv| kv.key.to_string())) - ); - assert_eq!(expect.scope_metrics.len(), actual.scope_metrics.len()); - let sort_instrumentation_library = - |metric: &ScopeMetrics, other_metric: &ScopeMetrics| match ( - metric.scope.as_ref(), - other_metric.scope.as_ref(), - ) { - (Some(library), Some(other_library)) => library - .name - .cmp(&other_library.name) - .then(library.version.cmp(&other_library.version)), - _ => Ordering::Equal, - }; - let sort_metrics = |metric: &Metric, other_metric: &Metric| { - metric.name.cmp(&other_metric.name).then( - metric - .description - .cmp(&other_metric.description) - .then(metric.unit.cmp(&other_metric.unit)), - ) - }; - expect.scope_metrics.sort_by(sort_instrumentation_library); - actual.scope_metrics.sort_by(sort_instrumentation_library); - - for (mut expect, mut actual) in expect - .scope_metrics - .into_iter() - .zip(actual.scope_metrics.into_iter()) - { - assert_eq!(expect.metrics.len(), actual.metrics.len()); - - expect.metrics.sort_by(sort_metrics); - actual.metrics.sort_by(sort_metrics); - - assert_eq!(expect.metrics, actual.metrics) - } + fn transform_histogram + Into + Copy>( + hist: &SdkHistogram, + ) -> TonicHistogram { + TonicHistogram { + data_points: hist + .data_points + .iter() + .map(|dp| TonicHistogramDataPoint { + attributes: dp.attributes.iter().map(Into::into).collect(), + start_time_unix_nano: to_nanos(dp.start_time), + time_unix_nano: to_nanos(dp.time), + count: dp.count, + sum: Some(dp.sum), + bucket_counts: dp.bucket_counts.clone(), + explicit_bounds: dp.bounds.clone(), + exemplars: dp.exemplars.iter().map(transform_exemplar).collect(), + flags: TonicDataPointFlags::default() as u32, + min: dp.min, + max: dp.max, + }) + .collect(), + aggregation_temporality: TonicTemporality::from(hist.temporality).into(), } + } - #[test] - fn test_record_to_metric() -> Result<(), MetricsError> { - let cx = Context::new(); - let attributes = vec![("test1", "value1"), ("test2", "value2")]; - let str_kv_attributes = attributes + fn transform_sum< + T: fmt::Debug + Into + Into + Copy, + >( + sum: &SdkSum, + ) -> TonicSum { + TonicSum { + data_points: sum + .data_points .iter() - .cloned() - .map(|(key, value)| key_value(key, value)) - .collect::>(); - let attribute_set = AttributeSet::from_attributes( - attributes - .iter() - .cloned() - .map(|(k, v)| opentelemetry::KeyValue::new(k, v)), - ); - let start_time = datetime!(2020-12-25 10:10:0 UTC); // unit nano 1608891000000000000 - let end_time = datetime!(2020-12-25 10:10:30 UTC); // unix nano 1608891030000000000 - - // Sum - { - let descriptor = Descriptor::new( - "test".to_string(), - InstrumentKind::Counter, - NumberKind::I64, - None, - None, - ); - let aggregator = SumAggregator::default(); - let val = Number::from(12_i64); - aggregator.update(&cx, &val, &descriptor)?; - let wrapped_aggregator: Arc = Arc::new(aggregator); - let record = record( - &descriptor, - &attribute_set, - Some(&wrapped_aggregator), - start_time.into(), - end_time.into(), - ); - let metric = record_to_metric(&record, &cumulative_temporality_selector())?; - - let expect = Metric { - name: "test".to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Sum(Sum { - data_points: vec![NumberDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes: str_kv_attributes.clone(), - start_time_unix_nano: 1608891000000000000, - time_unix_nano: 1608891030000000000, - value: Some(i64_to_value(12i64)), - exemplars: vec![], - }], - aggregation_temporality: 2, - is_monotonic: true, - })), - }; - - assert_eq!(expect, metric); - } - - // Last Value - { - let descriptor = Descriptor::new( - "test".to_string(), - InstrumentKind::GaugeObserver, - NumberKind::I64, - None, - None, - ); - let aggregator = last_value(); - let val1 = Number::from(12_i64); - let val2 = Number::from(14_i64); - aggregator.update(&cx, &val1, &descriptor)?; - aggregator.update(&cx, &val2, &descriptor)?; - let wrapped_aggregator: Arc = Arc::new(aggregator); - let record = record( - &descriptor, - &attribute_set, - Some(&wrapped_aggregator), - start_time.into(), - end_time.into(), - ); - let metric = record_to_metric(&record, &cumulative_temporality_selector())?; - - let expect = Metric { - name: "test".to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Gauge(Gauge { - data_points: vec![NumberDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes: str_kv_attributes.clone(), - start_time_unix_nano: 1608891000000000000, - time_unix_nano: if let Data::Gauge(gauge) = metric.data.clone().unwrap() - { - // ignore this field as it is the time the value updated. - // It changes every time the test runs - gauge.data_points[0].time_unix_nano - } else { - 0 - }, - value: Some(i64_to_value(14i64)), - exemplars: vec![], - }], - })), - }; - - assert_eq!(expect, metric); - } - - // Histogram - { - let descriptor = Descriptor::new( - "test".to_string(), - InstrumentKind::Histogram, - NumberKind::I64, - None, - None, - ); - let bound = [0.1, 0.2, 0.3]; - let aggregator = histogram(&bound); - let vals = vec![1i64.into(), 2i64.into(), 3i64.into()]; - for val in vals.iter() { - aggregator.update(&cx, val, &descriptor)?; - } - let wrapped_aggregator: Arc = Arc::new(aggregator); - let record = record( - &descriptor, - &attribute_set, - Some(&wrapped_aggregator), - start_time.into(), - end_time.into(), - ); - let metric = record_to_metric(&record, &cumulative_temporality_selector())?; - - let expect = Metric { - name: "test".to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Histogram(Histogram { - data_points: vec![HistogramDataPoint { - flags: DataPointFlags::FlagNone as u32, - attributes: str_kv_attributes, - start_time_unix_nano: 1608891000000000000, - time_unix_nano: 1608891030000000000, - count: 3, - sum: Some(6f64), - min: Some(1.0), - max: Some(3.0), - bucket_counts: vec![0, 0, 0, 3], - explicit_bounds: vec![0.1, 0.2, 0.3], - exemplars: vec![], - }], - aggregation_temporality: 2, - })), - }; - - assert_eq!(expect, metric); - } - - Ok(()) + .map(|dp| TonicNumberDataPoint { + attributes: dp.attributes.iter().map(Into::into).collect(), + start_time_unix_nano: dp.start_time.map(to_nanos).unwrap_or_default(), + time_unix_nano: dp.time.map(to_nanos).unwrap_or_default(), + exemplars: dp.exemplars.iter().map(transform_exemplar).collect(), + flags: TonicDataPointFlags::default() as u32, + value: Some(dp.value.into()), + }) + .collect(), + aggregation_temporality: TonicTemporality::from(sum.temporality).into(), + is_monotonic: sum.is_monotonic, } + } - #[test] - fn test_sink() { - let test_data: Vec<(ResourceWrapper, InstrumentationLibrary, Metric)> = vec![ - ( - vec![("runtime", "tokio")], - ("otlp", Some("0.1.1")), - "test", - (vec![("attribute1", "attribute2")], 12, 23, 2), - ), - ( - vec![("runtime", "tokio")], - ("otlp", Some("0.1.1")), - "test", - (vec![("attribute2", "attribute2")], 16, 19, 20), - ), - ( - vec![("runtime", "tokio"), ("rustc", "v48.0")], - ("otlp", Some("0.1.1")), - "test", - (vec![("attribute2", "attribute2")], 16, 19, 20), - ), - ( - vec![("runtime", "tokio")], - ("otlp", None), - "test", - (vec![("attribute1", "attribute2")], 15, 16, 88), - ), - ( - vec![("runtime", "tokio")], - ("otlp", None), - "another_test", - (vec![("attribute1", "attribute2")], 15, 16, 99), - ), - ] - .into_iter() - .map( - |(kvs, (name, version), metric_name, (attributes, start_time, end_time, value))| { - ( - ResourceWrapper::from(Resource::new(kvs.into_iter().map(|(k, v)| { - opentelemetry::KeyValue::new(k.to_string(), v.to_string()) - }))), - InstrumentationLibrary::new(name, version, None), - get_metric_with_name( - metric_name, - vec![(attributes, start_time, end_time, value)], - ), - ) - }, - ) - .collect::>(); - - let request = sink(test_data); - let actual = request.resource_metrics; - - let expect = vec![ - ( - vec![("runtime", "tokio")], - vec![ - ( - ("otlp", Some("0.1.1")), - vec![( - "test", - vec![ - (vec![("attribute1", "attribute2")], 12, 23, 2), - (vec![("attribute2", "attribute2")], 16, 19, 20), - ], - )], - ), - ( - ("otlp", None), - vec![ - ( - "test", - vec![(vec![("attribute1", "attribute2")], 15, 16, 88)], - ), - ( - "another_test", - vec![(vec![("attribute1", "attribute2")], 15, 16, 99)], - ), - ], - ), - ], - ), - ( - vec![("runtime", "tokio"), ("rustc", "v48.0")], - vec![( - ("otlp", Some("0.1.1")), - vec![( - "test", - vec![(vec![("attribute2", "attribute2")], 16, 19, 20)], - )], - )], - ), - ] - .into_iter() - .map(convert_to_resource_metrics); - - for (expect, actual) in expect.into_iter().zip(actual.into_iter()) { - assert_resource_metrics(expect, actual); - } + fn transform_gauge< + T: fmt::Debug + Into + Into + Copy, + >( + gauge: &SdkGauge, + ) -> TonicGauge { + TonicGauge { + data_points: gauge + .data_points + .iter() + .map(|dp| TonicNumberDataPoint { + attributes: dp.attributes.iter().map(Into::into).collect(), + start_time_unix_nano: dp.start_time.map(to_nanos).unwrap_or_default(), + time_unix_nano: dp.time.map(to_nanos).unwrap_or_default(), + exemplars: dp.exemplars.iter().map(transform_exemplar).collect(), + flags: TonicDataPointFlags::default() as u32, + value: Some(dp.value.into()), + }) + .collect(), } + } - #[test] - fn test_merge() { - let data_point_base = get_int_data_point(vec![("method", "POST")], 12, 12, 3); - let data_point_addon = get_int_data_point(vec![("method", "PUT")], 12, 12, 3); - - let mut metric1 = Metric { - name: "test".to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Sum(Sum { - data_points: vec![data_point_base.clone()], - aggregation_temporality: 2, - is_monotonic: true, - })), - }; - - let metric2 = Metric { - name: "test".to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Sum(Sum { - data_points: vec![data_point_addon.clone()], - aggregation_temporality: 2, - is_monotonic: true, - })), - }; - - let expect = Metric { - name: "test".to_string(), - description: "".to_string(), - unit: "".to_string(), - data: Some(Data::Sum(Sum { - data_points: vec![data_point_base, data_point_addon], - aggregation_temporality: 2, - is_monotonic: true, - })), - }; - - merge(&mut metric1, metric2); - - assert_eq!(metric1, expect); + fn transform_exemplar + Copy>( + ex: &SdkExemplar, + ) -> TonicExemplar { + TonicExemplar { + filtered_attributes: ex + .filtered_attributes + .iter() + .map(|kv| (&kv.key, &kv.value).into()) + .collect(), + time_unix_nano: to_nanos(ex.time), + span_id: ex.span_id.into(), + trace_id: ex.trace_id.into(), + value: Some(ex.value.into()), } } } diff --git a/opentelemetry-otlp/src/transform/mod.rs b/opentelemetry-otlp/src/transform/mod.rs index 13c11a5b59..1232d4f0f9 100644 --- a/opentelemetry-otlp/src/transform/mod.rs +++ b/opentelemetry-otlp/src/transform/mod.rs @@ -1,22 +1,5 @@ #[cfg(feature = "metrics")] mod metrics; -mod resource; -#[cfg(all(feature = "grpc-tonic", feature = "metrics"))] -pub(crate) use metrics::tonic::record_to_metric; #[cfg(all(feature = "grpc-tonic", feature = "metrics"))] pub(crate) use metrics::tonic::sink; -#[cfg(all(feature = "grpc-tonic", feature = "metrics"))] -pub(crate) use resource::ResourceWrapper; - -#[cfg(all(feature = "grpc-tonic", feature = "metrics"))] -use opentelemetry::sdk::InstrumentationLibrary; - -// Metrics in OTEL proto format checked from checkpoint with information of resource and instrumentation -// library. -#[cfg(all(feature = "grpc-tonic", feature = "metrics"))] -pub(crate) type CheckpointedMetrics = ( - ResourceWrapper, - InstrumentationLibrary, - opentelemetry_proto::tonic::metrics::v1::Metric, -); diff --git a/opentelemetry-otlp/src/transform/resource.rs b/opentelemetry-otlp/src/transform/resource.rs index d83816eb2e..f6a13a0d54 100644 --- a/opentelemetry-otlp/src/transform/resource.rs +++ b/opentelemetry-otlp/src/transform/resource.rs @@ -3,10 +3,10 @@ use opentelemetry_proto::tonic::{common::v1::KeyValue, resource::v1::Resource}; use std::cmp::Ordering; #[derive(PartialEq)] -pub(crate) struct ResourceWrapper(opentelemetry::sdk::Resource); +pub(crate) struct ResourceWrapper(opentelemetry_sdk::Resource); -impl From for ResourceWrapper { - fn from(r: opentelemetry::sdk::Resource) -> Self { +impl From for ResourceWrapper { + fn from(r: opentelemetry_sdk::Resource) -> Self { ResourceWrapper(r) } } diff --git a/opentelemetry-otlp/tests/smoke.rs b/opentelemetry-otlp/tests/smoke.rs index dd1405ed35..d7e6c5a32d 100644 --- a/opentelemetry-otlp/tests/smoke.rs +++ b/opentelemetry-otlp/tests/smoke.rs @@ -1,6 +1,6 @@ use futures::StreamExt; -use opentelemetry::global::shutdown_tracer_provider; -use opentelemetry::trace::{Span, SpanKind, Tracer}; +use opentelemetry_api::global::shutdown_tracer_provider; +use opentelemetry_api::trace::{Span, SpanKind, Tracer}; use opentelemetry_otlp::WithExportConfig; use opentelemetry_proto::tonic::collector::trace::v1::{ trace_service_server::{TraceService, TraceServiceServer}, @@ -85,7 +85,7 @@ async fn smoke_tracer() { .with_endpoint(format!("http://{}", addr)) .with_metadata(metadata), ) - .install_batch(opentelemetry::runtime::Tokio) + .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("failed to install"); println!("Sending span..."); diff --git a/opentelemetry-prometheus/Cargo.toml b/opentelemetry-prometheus/Cargo.toml index b644632fad..164c6af5d2 100644 --- a/opentelemetry-prometheus/Cargo.toml +++ b/opentelemetry-prometheus/Cargo.toml @@ -20,13 +20,14 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] -opentelemetry = { version = "0.18", path = "../opentelemetry", default-features = false, features = ["metrics"] } +once_cell = "1.17" +opentelemetry_api = { version = "0.18", path = "../opentelemetry-api", default-features = false, features = ["metrics"] } +opentelemetry_sdk = { version = "0.18", path = "../opentelemetry-sdk", default-features = false, features = ["metrics"] } prometheus = "0.13" protobuf = "2.14" [dev-dependencies] -opentelemetry = { path = "../opentelemetry", features = ["metrics", "testing"] } -lazy_static = "1.4" +opentelemetry-semantic-conventions = { path = "../opentelemetry-semantic-conventions" } [features] prometheus-encoding = [] diff --git a/opentelemetry-prometheus/src/config.rs b/opentelemetry-prometheus/src/config.rs new file mode 100644 index 0000000000..2a21f0f4df --- /dev/null +++ b/opentelemetry-prometheus/src/config.rs @@ -0,0 +1,108 @@ +use core::fmt; +use std::sync::{Arc, Mutex}; + +use once_cell::sync::OnceCell; +use opentelemetry_api::metrics::{MetricsError, Result}; +use opentelemetry_sdk::metrics::{reader::AggregationSelector, ManualReaderBuilder}; + +use crate::{Collector, PrometheusExporter}; + +/// [PrometheusExporter] configuration options +#[derive(Default)] +pub struct ExporterBuilder { + registry: Option, + disable_target_info: bool, + without_units: bool, + aggregation: Option>, + disable_scope_info: bool, +} + +impl fmt::Debug for ExporterBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExporterBuilder") + .field("registry", &self.registry) + .field("disable_target_info", &self.disable_target_info) + .field("without_units", &self.without_units) + .field("aggregation", &self.aggregation.is_some()) + .field("disable_scope_info", &self.disable_scope_info) + .finish() + } +} + +impl ExporterBuilder { + /// Disables exporter's addition of unit suffixes to metric names. + /// + /// By default, metric names include a unit suffix to follow Prometheus naming + /// conventions. For example, the counter metric `request.duration`, with unit + /// `ms` would become `request_duration_milliseconds_total`. + /// + /// With this option set, the name would instead be `request_duration_total`. + pub fn without_units(mut self) -> Self { + self.without_units = true; + self + } + + /// Configures the exporter to not export the resource `target_info` metric. + /// + /// If not specified, the exporter will create a `target_info` metric containing + /// the metrics' [Resource] attributes. + /// + /// [Resource]: opentelemetry_sdk::Resource + pub fn without_target_info(mut self) -> Self { + self.disable_target_info = true; + self + } + + /// Configures the exporter to not export the `otel_scope_info` metric. + /// + /// If not specified, the exporter will create a `otel_scope_info` metric + /// containing the metrics' Instrumentation Scope, and also add labels about + /// Instrumentation Scope to all metric points. + pub fn without_scope_info(mut self) -> Self { + self.disable_scope_info = true; + self + } + + /// Configures which [prometheus::Registry] the exporter will use. + /// + /// If no registry is specified, the prometheus default is used. + pub fn with_registry(mut self, registry: prometheus::Registry) -> Self { + self.registry = Some(registry); + self + } + + /// Configure the [AggregationSelector] the exporter will use. + /// + /// If no selector is provided, the [DefaultAggregationSelector] is used. + /// + /// [DefaultAggregationSelector]: opentelemetry_sdk::metrics::reader::DefaultAggregationSelector + pub fn with_aggregation_selector(mut self, agg: impl AggregationSelector + 'static) -> Self { + self.aggregation = Some(Box::new(agg)); + self + } + + /// Creates a new [PrometheusExporter] from this configuration. + pub fn build(self) -> Result { + let mut reader = ManualReaderBuilder::new(); + if let Some(selector) = self.aggregation { + reader = reader.with_aggregation_selector(selector) + } + let reader = Arc::new(reader.build()); + + let collector = Collector { + reader: Arc::clone(&reader), + disable_target_info: self.disable_target_info, + without_units: self.without_units, + disable_scope_info: self.disable_scope_info, + create_target_info_once: OnceCell::new(), + inner: Mutex::new(Default::default()), + }; + + let registry = self.registry.unwrap_or_else(prometheus::Registry::new); + registry + .register(Box::new(collector)) + .map_err(|e| MetricsError::Other(e.to_string()))?; + + Ok(PrometheusExporter { reader }) + } +} diff --git a/opentelemetry-prometheus/src/lib.rs b/opentelemetry-prometheus/src/lib.rs index 14f5803481..e46dd11065 100644 --- a/opentelemetry-prometheus/src/lib.rs +++ b/opentelemetry-prometheus/src/lib.rs @@ -1,65 +1,79 @@ -//! # OpenTelemetry Prometheus Exporter +//! An OpenTelemetry exporter for [Prometheus] metrics. //! -//! ### Prometheus Exporter Example +//! [Prometheus]: https://prometheus.io //! -//! ```rust -//! use opentelemetry::{global, Context, KeyValue, sdk::Resource}; -//! use opentelemetry::sdk::export::metrics::aggregation; -//! use opentelemetry::sdk::metrics::{controllers, processors, selectors}; -//! use opentelemetry_prometheus::PrometheusExporter; -//! use prometheus::{TextEncoder, Encoder}; +//! ``` +//! use opentelemetry_api::{metrics::MeterProvider as _, Context, KeyValue}; +//! use opentelemetry_sdk::metrics::MeterProvider; +//! use prometheus::{Encoder, TextEncoder}; +//! +//! # fn main() -> Result<(), Box> { +//! let cx = Context::current(); //! -//! fn init_meter() -> PrometheusExporter { -//! let controller = controllers::basic( -//! processors::factory( -//! selectors::simple::histogram([1.0, 2.0, 5.0, 10.0, 20.0, 50.0]), -//! aggregation::cumulative_temporality_selector(), -//! ) -//! ) -//! .build(); +//! // create a new prometheus registry +//! let registry = prometheus::Registry::new(); //! -//! opentelemetry_prometheus::exporter(controller).init() -//! } +//! // configure OpenTelemetry to use this registry +//! let exporter = opentelemetry_prometheus::exporter() +//! .with_registry(registry.clone()) +//! .build()?; //! -//! let cx = Context::current(); -//! let exporter = init_meter(); -//! let meter = global::meter("my-app"); +//! // set up a meter meter to create instruments +//! let provider = MeterProvider::builder().with_reader(exporter).build(); +//! let meter = provider.meter("my-app"); //! //! // Use two instruments //! let counter = meter //! .u64_counter("a.counter") //! .with_description("Counts things") //! .init(); -//! let recorder = meter +//! let histogram = meter //! .i64_histogram("a.histogram") //! .with_description("Records values") //! .init(); //! //! counter.add(&cx, 100, &[KeyValue::new("key", "value")]); -//! recorder.record(&cx, 100, &[KeyValue::new("key", "value")]); +//! histogram.record(&cx, 100, &[KeyValue::new("key", "value")]); //! //! // Encode data as text or protobuf //! let encoder = TextEncoder::new(); -//! let metric_families = exporter.registry().gather(); +//! let metric_families = registry.gather(); //! let mut result = Vec::new(); -//! encoder.encode(&metric_families, &mut result); +//! encoder.encode(&metric_families, &mut result)?; //! //! // result now contains encoded metrics: //! // -//! // # HELP a_counter Counts things -//! // # TYPE a_counter counter -//! // a_counter{R="V",key="value",otel_scope_name="my-app",otel_scope_version=""} 100 +//! // # HELP a_counter_total Counts things +//! // # TYPE a_counter_total counter +//! // a_counter_total{key="value",otel_scope_name="my-app"} 100 //! // # HELP a_histogram Records values //! // # TYPE a_histogram histogram -//! // a_histogram_bucket{R="V",key="value",le="0.5",otel_scope_name="my-app",otel_scope_version=""} 0 -//! // a_histogram_bucket{R="V",key="value",le="0.9",otel_scope_name="my-app",otel_scope_version=""} 0 -//! // a_histogram_bucket{R="V",key="value",le="0.99",otel_scope_name="my-app",otel_scope_version=""} 0 -//! // a_histogram_bucket{R="V",key="value",le="+Inf",otel_scope_name="my-app",otel_scope_version=""} 1 -//! // a_histogram_sum{R="V",key="value",otel_scope_name="my-app",otel_scope_version=""} 100 -//! // a_histogram_count{R="V",key="value",otel_scope_name="my-app",otel_scope_version=""} 1 -//! // HELP otel_scope_info Instrumentation Scope metadata -//! // TYPE otel_scope_info gauge -//! // otel_scope_info{otel_scope_name="ex.com/B",otel_scope_version=""} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="0"} 0 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="5"} 0 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="10"} 0 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="25"} 0 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="50"} 0 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="75"} 0 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="100"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="250"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="500"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="750"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="1000"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="2500"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="5000"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="7500"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="10000"} 1 +//! // a_histogram_bucket{key="value",otel_scope_name="my-app",le="+Inf"} 1 +//! // a_histogram_sum{key="value",otel_scope_name="my-app"} 100 +//! // a_histogram_count{key="value",otel_scope_name="my-app"} 1 +//! // # HELP otel_scope_info Instrumentation Scope metadata +//! // # TYPE otel_scope_info gauge +//! // otel_scope_info{otel_scope_name="my-app"} 1 +//! // # HELP target_info Target metadata +//! // # TYPE target_info gauge +//! // target_info{service_name="unknown_service"} 1 +//! # Ok(()) +//! # } //! ``` #![warn( future_incompatible, @@ -80,447 +94,557 @@ )] #![cfg_attr(test, deny(warnings))] -use opentelemetry::metrics::MeterProvider; -use opentelemetry::sdk::export::metrics::aggregation::{ - self, AggregationKind, Temporality, TemporalitySelector, +use once_cell::sync::OnceCell; +use opentelemetry_api::{ + global, + metrics::{MetricsError, Result, Unit}, + Context, Key, Value, }; -use opentelemetry::sdk::export::metrics::InstrumentationLibraryReader; -use opentelemetry::sdk::metrics::sdk_api::Descriptor; -#[cfg(feature = "prometheus-encoding")] -pub use prometheus::{Encoder, TextEncoder}; - -use opentelemetry::sdk::{ - export::metrics::{ - aggregation::{Histogram, LastValue, Sum}, - Record, - }, +use opentelemetry_sdk::{ metrics::{ - aggregators::{HistogramAggregator, LastValueAggregator, SumAggregator}, - controllers::BasicController, - sdk_api::NumberKind, + data::{self, ResourceMetrics, Temporality}, + reader::{AggregationSelector, MetricProducer, MetricReader, TemporalitySelector}, + Aggregation, InstrumentKind, ManualReader, Pipeline, }, - Resource, + Resource, Scope, +}; +use prometheus::{ + core::Desc, + proto::{LabelPair, MetricFamily, MetricType}, }; -use opentelemetry::{attributes, metrics::MetricsError, Context, Key, Value}; -use opentelemetry::{global, InstrumentationLibrary, StringValue}; -use std::sync::{Arc, Mutex}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, + sync::{Arc, Mutex}, +}; +use std::{fmt, sync::Weak}; -mod sanitize; +const TARGET_INFO_NAME: &str = "target_info"; +const TARGET_INFO_DESCRIPTION: &str = "Target metadata"; -use sanitize::sanitize; +const SCOPE_INFO_METRIC_NAME: &str = "otel_scope_info"; +const SCOPE_INFO_DESCRIPTION: &str = "Instrumentation Scope metadata"; -/// Monotonic Sum metric points MUST have _total added as a suffix to the metric name -/// https://github.com/open-telemetry/opentelemetry-specification/blob/v1.14.0/specification/metrics/data-model.md#sums-1 -const MONOTONIC_COUNTER_SUFFIX: &str = "_total"; +const SCOPE_INFO_KEYS: [&str; 2] = ["otel_scope_name", "otel_scope_version"]; -/// Instrumentation Scope name MUST added as otel_scope_name label. -const OTEL_SCOPE_NAME: &str = "otel_scope_name"; +// prometheus counters MUST have a _total suffix: +// https://github.com/open-telemetry/opentelemetry-specification/blob/v1.19.0/specification/compatibility/prometheus_and_openmetrics.md?plain=1#L282 +const COUNTER_SUFFIX: &str = "_total"; -/// Instrumentation Scope version MUST added as otel_scope_name label. -const OTEL_SCOPE_VERSION: &str = "otel_scope_version"; +mod config; -/// otel_scope_name metric name. -const SCOPE_INFO_METRIC_NAME: &str = "otel_scope_info"; +pub use config::ExporterBuilder; -/// otel_scope_name metric help. -const SCOPE_INFO_DESCRIPTION: &str = "Instrumentation Scope metadata"; -/// Create a new prometheus exporter builder. -pub fn exporter(controller: BasicController) -> ExporterBuilder { - ExporterBuilder::new(controller) +/// Creates a builder to configure a [PrometheusExporter] +pub fn exporter() -> ExporterBuilder { + ExporterBuilder::default() } -/// Configuration for the prometheus exporter. +/// Prometheus metrics exporter #[derive(Debug)] -pub struct ExporterBuilder { - /// The prometheus registry that will be used to register instruments. - /// - /// If not set a new empty `Registry` is created. - registry: Option, - - /// The metrics controller - controller: BasicController, - - /// config for exporter - config: Option, +pub struct PrometheusExporter { + reader: Arc, } -impl ExporterBuilder { - /// Create a new exporter builder with a given controller - pub fn new(controller: BasicController) -> Self { - ExporterBuilder { - registry: None, - controller, - config: Some(Default::default()), - } - } - - /// Set the prometheus registry to be used by this exporter - pub fn with_registry(self, registry: prometheus::Registry) -> Self { - ExporterBuilder { - registry: Some(registry), - ..self - } - } - - /// Set config to be used by this exporter - pub fn with_config(self, config: ExporterConfig) -> Self { - ExporterBuilder { - config: Some(config), - ..self - } - } - - /// Sets up a complete export pipeline with the recommended setup, using the - /// recommended selector and standard processor. - pub fn try_init(self) -> Result { - let config = self.config.unwrap_or_default(); - - let registry = self.registry.unwrap_or_else(prometheus::Registry::new); - - let controller = Arc::new(Mutex::new(self.controller)); - let collector = - Collector::with_controller(controller.clone()).with_scope_info(config.with_scope_info); - registry - .register(Box::new(collector)) - .map_err(|e| MetricsError::Other(e.to_string()))?; - - let exporter = PrometheusExporter { - registry, - controller, - }; - global::set_meter_provider(exporter.meter_provider()?); - - Ok(exporter) - } - - /// Sets up a complete export pipeline with the recommended setup, using the - /// recommended selector and standard processor. - /// - /// # Panics - /// - /// This panics if the exporter cannot be registered in the prometheus registry. - pub fn init(self) -> PrometheusExporter { - self.try_init().unwrap() +impl TemporalitySelector for PrometheusExporter { + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.reader.temporality(kind) } } -/// Config for prometheus exporter -#[derive(Debug)] -pub struct ExporterConfig { - /// Add the otel_scope_info metric and otel_scope_ labels when with_scope_info is true, and the default value is true. - with_scope_info: bool, +impl AggregationSelector for PrometheusExporter { + fn aggregation(&self, kind: InstrumentKind) -> Aggregation { + self.reader.aggregation(kind) + } } -impl Default for ExporterConfig { - fn default() -> Self { - ExporterConfig { - with_scope_info: true, - } +impl MetricReader for PrometheusExporter { + fn register_pipeline(&self, pipeline: Weak) { + self.reader.register_pipeline(pipeline) } -} -impl ExporterConfig { - /// Set with_scope_info for [`ExporterConfig`]. - /// It's the flag to add the otel_scope_info metric and otel_scope_ labels. - pub fn with_scope_info(mut self, enabled: bool) -> Self { - self.with_scope_info = enabled; - self + fn register_producer(&self, producer: Box) { + self.reader.register_producer(producer) } -} -/// An implementation of `metrics::Exporter` that sends metrics to Prometheus. -/// -/// This exporter supports Prometheus pulls, as such it does not -/// implement the export.Exporter interface. -#[derive(Clone, Debug)] -pub struct PrometheusExporter { - registry: prometheus::Registry, - controller: Arc>, -} + fn collect(&self, cx: &Context, rm: &mut ResourceMetrics) -> Result<()> { + self.reader.collect(cx, rm) + } -impl PrometheusExporter { - /// Returns a reference to the current prometheus registry. - pub fn registry(&self) -> &prometheus::Registry { - &self.registry + fn force_flush(&self, cx: &Context) -> Result<()> { + self.reader.force_flush(cx) } - /// Get this exporter's provider. - pub fn meter_provider(&self) -> Result { - self.controller - .lock() - .map_err(Into::into) - .map(|locked| locked.clone()) + fn shutdown(&self) -> Result<()> { + self.reader.shutdown() } } -#[derive(Debug)] struct Collector { - controller: Arc>, - with_scope_info: bool, + reader: Arc, + disable_target_info: bool, + without_units: bool, + disable_scope_info: bool, + create_target_info_once: OnceCell, + inner: Mutex, } -impl TemporalitySelector for Collector { - fn temporality_for(&self, descriptor: &Descriptor, kind: &AggregationKind) -> Temporality { - aggregation::cumulative_temporality_selector().temporality_for(descriptor, kind) - } +#[derive(Default)] +struct CollectorInner { + scope_infos: HashMap, + metric_families: HashMap, } impl Collector { - fn with_controller(controller: Arc>) -> Self { - Collector { - controller, - with_scope_info: true, + fn get_name(&self, m: &data::Metric) -> Cow<'static, str> { + let name = sanitize_name(&m.name); + match get_unit_suffixes(&m.unit) { + Some(suffix) if !self.without_units => Cow::Owned(format!("{name}{suffix}")), + _ => name, } } - fn with_scope_info(mut self, with_scope_info: bool) -> Self { - self.with_scope_info = with_scope_info; - self - } } impl prometheus::core::Collector for Collector { - /// Unused as descriptors are dynamically registered. - fn desc(&self) -> Vec<&prometheus::core::Desc> { + fn desc(&self) -> Vec<&Desc> { Vec::new() } - /// Collect all otel metrics and convert to prometheus metrics. - fn collect(&self) -> Vec { - if let Ok(controller) = self.controller.lock() { - let mut metrics = Vec::new(); - - if let Err(err) = controller.collect(&Context::current()) { + fn collect(&self) -> Vec { + let mut inner = match self.inner.lock() { + Ok(guard) => guard, + Err(err) => { global::handle_error(err); - return metrics; + return Vec::new(); } + }; + + let mut metrics = ResourceMetrics { + resource: Resource::empty(), + scope_metrics: vec![], + }; + if let Err(err) = self.reader.collect(&Context::new(), &mut metrics) { + global::handle_error(err); + return vec![]; + } + let mut res = Vec::with_capacity(metrics.scope_metrics.len() + 1); + + let target_info = self.create_target_info_once.get_or_init(|| { + // Resource should be immutable, we don't need to compute again + create_info_metric(TARGET_INFO_NAME, TARGET_INFO_DESCRIPTION, &metrics.resource) + }); + if !self.disable_target_info { + res.push(target_info.clone()) + } - if let Err(err) = controller.try_for_each(&mut |library, reader| { - let mut scope_labels: Vec = Vec::new(); - if self.with_scope_info { - scope_labels = get_scope_labels(library); - metrics.push(build_scope_metric(scope_labels.clone())); + for scope_metrics in metrics.scope_metrics { + let scope_labels = if !self.disable_scope_info { + let scope_info = inner + .scope_infos + .entry(scope_metrics.scope.clone()) + .or_insert_with_key(create_scope_info_metric); + res.push(scope_info.clone()); + + let mut labels = + Vec::with_capacity(1 + scope_metrics.scope.version.is_some() as usize); + let mut name = LabelPair::new(); + name.set_name(SCOPE_INFO_KEYS[0].into()); + name.set_value(scope_metrics.scope.name.to_string()); + labels.push(name); + if let Some(version) = &scope_metrics.scope.version { + let mut l_version = LabelPair::new(); + l_version.set_name(SCOPE_INFO_KEYS[1].into()); + l_version.set_value(version.to_string()); + labels.push(l_version); } - reader.try_for_each(self, &mut |record| { - let agg = record.aggregator().ok_or(MetricsError::NoDataCollected)?; - let number_kind = record.descriptor().number_kind(); - let instrument_kind = record.descriptor().instrument_kind(); - - let desc = get_metric_desc(record); - let labels = - get_metric_labels(record, controller.resource(), &mut scope_labels.clone()); - - if let Some(hist) = agg.as_any().downcast_ref::() { - metrics.push(build_histogram(hist, number_kind, desc, labels)?); - } else if let Some(sum) = agg.as_any().downcast_ref::() { - let counter = if instrument_kind.monotonic() { - build_monotonic_counter(sum, number_kind, desc, labels)? - } else { - build_non_monotonic_counter(sum, number_kind, desc, labels)? - }; - - metrics.push(counter); - } else if let Some(last) = agg.as_any().downcast_ref::() { - metrics.push(build_last_value(last, number_kind, desc, labels)?); - } - Ok(()) - }) - }) { - global::handle_error(err); + labels + } else { + Vec::new() + }; + + for metrics in scope_metrics.metrics { + let name = self.get_name(&metrics); + let description = metrics.description; + let data = metrics.data.as_any(); + let mfs = &mut inner.metric_families; + if let Some(hist) = data.downcast_ref::>() { + add_histogram_metric(&mut res, hist, description, &scope_labels, name, mfs); + } else if let Some(hist) = data.downcast_ref::>() { + add_histogram_metric(&mut res, hist, description, &scope_labels, name, mfs); + } else if let Some(hist) = data.downcast_ref::>() { + add_histogram_metric(&mut res, hist, description, &scope_labels, name, mfs); + } else if let Some(sum) = data.downcast_ref::>() { + add_sum_metric(&mut res, sum, description, &scope_labels, name, mfs); + } else if let Some(sum) = data.downcast_ref::>() { + add_sum_metric(&mut res, sum, description, &scope_labels, name, mfs); + } else if let Some(sum) = data.downcast_ref::>() { + add_sum_metric(&mut res, sum, description, &scope_labels, name, mfs); + } else if let Some(g) = data.downcast_ref::>() { + add_gauge_metric(&mut res, g, description, &scope_labels, name, mfs); + } else if let Some(g) = data.downcast_ref::>() { + add_gauge_metric(&mut res, g, description, &scope_labels, name, mfs); + } else if let Some(g) = data.downcast_ref::>() { + add_gauge_metric(&mut res, g, description, &scope_labels, name, mfs); + } } - - metrics - } else { - Vec::new() } + + res } } -fn build_last_value( - lv: &LastValueAggregator, - kind: &NumberKind, - desc: PrometheusMetricDesc, - labels: Vec, -) -> Result { - let (last_value, _) = lv.last_value()?; +/// Maps attributes into Prometheus-style label pairs. +/// +/// It sanitizes invalid characters and handles duplicate keys (due to +/// sanitization) by sorting and concatenating the values following the spec. +fn get_attrs(kvs: &mut dyn Iterator, extra: &[LabelPair]) -> Vec { + let mut keys_map = BTreeMap::>::new(); + for (key, value) in kvs { + let key = sanitize_prom_kv(key.as_str()); + keys_map + .entry(key) + .and_modify(|v| v.push(value.to_string())) + .or_insert_with(|| vec![value.to_string()]); + } - let mut g = prometheus::proto::Gauge::default(); - g.set_value(last_value.to_f64(kind)); + let mut res = Vec::with_capacity(keys_map.len() + extra.len()); - let mut m = prometheus::proto::Metric::default(); - m.set_label(protobuf::RepeatedField::from_vec(labels)); - m.set_gauge(g); + for (key, mut values) in keys_map.into_iter() { + values.sort_unstable(); - let mut mf = prometheus::proto::MetricFamily::default(); - mf.set_name(desc.name); - mf.set_help(desc.help); - mf.set_field_type(prometheus::proto::MetricType::GAUGE); - mf.set_metric(protobuf::RepeatedField::from_vec(vec![m])); + let mut lp = LabelPair::new(); + lp.set_name(key); + lp.set_value(values.join(";")); + res.push(lp); + } - Ok(mf) -} + if !extra.is_empty() { + res.extend(&mut extra.iter().cloned()); + } -fn build_non_monotonic_counter( - sum: &SumAggregator, - kind: &NumberKind, - desc: PrometheusMetricDesc, - labels: Vec, -) -> Result { - let sum = sum.sum()?; + res +} - let mut g = prometheus::proto::Gauge::default(); - g.set_value(sum.to_f64(kind)); +fn validate_metrics( + name: &str, + description: &str, + metric_type: MetricType, + mfs: &mut HashMap, +) -> (bool, Option) { + if let Some(existing) = mfs.get(name) { + if existing.get_field_type() != metric_type { + global::handle_error(MetricsError::Other(format!("Instrument type conflict, using existing type definition. Instrument {name}, Existing: {:?}, dropped: {:?}", existing.get_field_type(), metric_type))); + return (true, None); + } + if existing.get_help() != description { + global::handle_error(MetricsError::Other(format!("Instrument description conflict, using existing. Instrument {name}, Existing: {:?}, dropped: {:?}", existing.get_help(), description))); + return (false, Some(existing.get_help().to_string())); + } + (false, None) + } else { + let mut mf = MetricFamily::default(); + mf.set_name(name.into()); + mf.set_help(description.to_string()); + mf.set_field_type(metric_type); + mfs.insert(name.to_string(), mf); - let mut m = prometheus::proto::Metric::default(); - m.set_label(protobuf::RepeatedField::from_vec(labels)); - m.set_gauge(g); + (false, None) + } +} - let mut mf = prometheus::proto::MetricFamily::default(); - mf.set_name(desc.name); - mf.set_help(desc.help); - mf.set_field_type(prometheus::proto::MetricType::GAUGE); - mf.set_metric(protobuf::RepeatedField::from_vec(vec![m])); +fn add_histogram_metric( + res: &mut Vec, + histogram: &data::Histogram, + mut description: Cow<'static, str>, + extra: &[LabelPair], + name: Cow<'static, str>, + mfs: &mut HashMap, +) { + // Consider supporting exemplars when `prometheus` crate has the feature + // See: https://github.com/tikv/rust-prometheus/issues/393 + let (drop, help) = validate_metrics(&name, &description, MetricType::HISTOGRAM, mfs); + if drop { + return; + } + if let Some(help) = help { + description = help.into(); + } - Ok(mf) + for dp in &histogram.data_points { + let kvs = get_attrs(&mut dp.attributes.iter(), extra); + let bounds_len = dp.bounds.len(); + let (bucket, _) = dp.bounds.iter().enumerate().fold( + (Vec::with_capacity(bounds_len), 0), + |(mut acc, mut count), (i, bound)| { + count += dp.bucket_counts[i]; + + let mut b = prometheus::proto::Bucket::default(); + b.set_upper_bound(*bound); + b.set_cumulative_count(count); + acc.push(b); + (acc, count) + }, + ); + + let mut h = prometheus::proto::Histogram::default(); + h.set_sample_sum(dp.sum); + h.set_sample_count(dp.count); + h.set_bucket(protobuf::RepeatedField::from_vec(bucket)); + let mut pm = prometheus::proto::Metric::default(); + pm.set_label(protobuf::RepeatedField::from_vec(kvs)); + pm.set_histogram(h); + + let mut mf = prometheus::proto::MetricFamily::default(); + mf.set_name(name.to_string()); + mf.set_help(description.to_string()); + mf.set_field_type(prometheus::proto::MetricType::HISTOGRAM); + mf.set_metric(protobuf::RepeatedField::from_vec(vec![pm])); + res.push(mf); + } } -fn build_monotonic_counter( - sum: &SumAggregator, - kind: &NumberKind, - desc: PrometheusMetricDesc, - labels: Vec, -) -> Result { - let sum = sum.sum()?; +fn add_sum_metric( + res: &mut Vec, + sum: &data::Sum, + mut description: Cow<'static, str>, + extra: &[LabelPair], + mut name: Cow<'static, str>, + mfs: &mut HashMap, +) { + let metric_type; + if sum.is_monotonic { + name = format!("{name}{COUNTER_SUFFIX}").into(); + metric_type = MetricType::COUNTER; + } else { + metric_type = MetricType::GAUGE; + } - let mut c = prometheus::proto::Counter::default(); - c.set_value(sum.to_f64(kind)); + let (drop, help) = validate_metrics(&name, &description, metric_type, mfs); + if drop { + return; + } + if let Some(help) = help { + description = help.into(); + } - let mut m = prometheus::proto::Metric::default(); - m.set_label(protobuf::RepeatedField::from_vec(labels)); - m.set_counter(c); + for dp in &sum.data_points { + let kvs = get_attrs(&mut dp.attributes.iter(), extra); - let mut mf = prometheus::proto::MetricFamily::default(); - mf.set_name(desc.name + MONOTONIC_COUNTER_SUFFIX); - mf.set_help(desc.help); - mf.set_field_type(prometheus::proto::MetricType::COUNTER); - mf.set_metric(protobuf::RepeatedField::from_vec(vec![m])); + let mut pm = prometheus::proto::Metric::default(); + pm.set_label(protobuf::RepeatedField::from_vec(kvs)); + + if sum.is_monotonic { + let mut c = prometheus::proto::Counter::default(); + c.set_value(dp.value.as_f64()); + pm.set_counter(c); + } else { + let mut g = prometheus::proto::Gauge::default(); + g.set_value(dp.value.as_f64()); + pm.set_gauge(g); + } - Ok(mf) + let mut mf = prometheus::proto::MetricFamily::default(); + mf.set_name(name.to_string()); + mf.set_help(description.to_string()); + mf.set_field_type(metric_type); + mf.set_metric(protobuf::RepeatedField::from_vec(vec![pm])); + res.push(mf); + } } -fn build_histogram( - hist: &HistogramAggregator, - kind: &NumberKind, - desc: PrometheusMetricDesc, - labels: Vec, -) -> Result { - let raw_buckets = hist.histogram()?; - let sum = hist.sum()?; - - let mut h = prometheus::proto::Histogram::default(); - h.set_sample_sum(sum.to_f64(kind)); - - let mut count = 0; - let mut buckets = Vec::with_capacity(raw_buckets.boundaries().len()); - for (i, upper_bound) in raw_buckets.boundaries().iter().enumerate() { - count += raw_buckets.counts()[i] as u64; - let mut b = prometheus::proto::Bucket::default(); - b.set_cumulative_count(count); - b.set_upper_bound(*upper_bound); - buckets.push(b); +fn add_gauge_metric( + res: &mut Vec, + gauge: &data::Gauge, + mut description: Cow<'static, str>, + extra: &[LabelPair], + name: Cow<'static, str>, + mfs: &mut HashMap, +) { + let (drop, help) = validate_metrics(&name, &description, MetricType::GAUGE, mfs); + if drop { + return; + } + if let Some(help) = help { + description = help.into(); + } + + for dp in &gauge.data_points { + let kvs = get_attrs(&mut dp.attributes.iter(), extra); + + let mut g = prometheus::proto::Gauge::default(); + g.set_value(dp.value.as_f64()); + let mut pm = prometheus::proto::Metric::default(); + pm.set_label(protobuf::RepeatedField::from_vec(kvs)); + pm.set_gauge(g); + + let mut mf = prometheus::proto::MetricFamily::default(); + mf.set_name(name.to_string()); + mf.set_help(description.to_string()); + mf.set_field_type(MetricType::GAUGE); + mf.set_metric(protobuf::RepeatedField::from_vec(vec![pm])); + res.push(mf); } - // Include the +inf bucket in the total count. - count += raw_buckets.counts()[raw_buckets.counts().len() - 1] as u64; - h.set_bucket(protobuf::RepeatedField::from_vec(buckets)); - h.set_sample_count(count); +} + +fn create_info_metric( + target_info_name: &str, + target_info_description: &str, + resource: &Resource, +) -> MetricFamily { + let mut g = prometheus::proto::Gauge::default(); + g.set_value(1.0); let mut m = prometheus::proto::Metric::default(); - m.set_label(protobuf::RepeatedField::from_vec(labels)); - m.set_histogram(h); + m.set_label(protobuf::RepeatedField::from_vec(get_attrs( + &mut resource.iter(), + &[], + ))); + m.set_gauge(g); - let mut mf = prometheus::proto::MetricFamily::default(); - mf.set_name(desc.name); - mf.set_help(desc.help); - mf.set_field_type(prometheus::proto::MetricType::HISTOGRAM); + let mut mf = MetricFamily::default(); + mf.set_name(target_info_name.into()); + mf.set_help(target_info_description.into()); + mf.set_field_type(MetricType::GAUGE); mf.set_metric(protobuf::RepeatedField::from_vec(vec![m])); - - Ok(mf) + mf } -fn build_scope_metric( - labels: Vec, -) -> prometheus::proto::MetricFamily { - let mut g = prometheus::proto::Gauge::new(); +fn create_scope_info_metric(scope: &Scope) -> MetricFamily { + let mut g = prometheus::proto::Gauge::default(); g.set_value(1.0); + let mut labels = Vec::with_capacity(1 + scope.version.is_some() as usize); + let mut name = LabelPair::new(); + name.set_name(SCOPE_INFO_KEYS[0].into()); + name.set_value(scope.name.to_string()); + labels.push(name); + if let Some(version) = &scope.version { + let mut v_label = LabelPair::new(); + v_label.set_name(SCOPE_INFO_KEYS[1].into()); + v_label.set_value(version.to_string()); + labels.push(v_label); + } + let mut m = prometheus::proto::Metric::default(); m.set_label(protobuf::RepeatedField::from_vec(labels)); m.set_gauge(g); - let mut mf = prometheus::proto::MetricFamily::default(); - mf.set_name(String::from(SCOPE_INFO_METRIC_NAME)); - mf.set_help(String::from(SCOPE_INFO_DESCRIPTION)); - mf.set_field_type(prometheus::proto::MetricType::GAUGE); + let mut mf = MetricFamily::default(); + mf.set_name(SCOPE_INFO_METRIC_NAME.into()); + mf.set_help(SCOPE_INFO_DESCRIPTION.into()); + mf.set_field_type(MetricType::GAUGE); mf.set_metric(protobuf::RepeatedField::from_vec(vec![m])); - mf } -fn get_scope_labels(library: &InstrumentationLibrary) -> Vec { - let mut labels = Vec::new(); - labels.push(build_label_pair( - &Key::new(OTEL_SCOPE_NAME), - &Value::String(StringValue::from(library.name.clone().to_string())), - )); - if let Some(version) = library.version.to_owned() { - labels.push(build_label_pair( - &Key::new(OTEL_SCOPE_VERSION), - &Value::String(StringValue::from(version.to_string())), - )); +fn get_unit_suffixes(unit: &Unit) -> Option<&'static str> { + match unit.as_str() { + "1" => Some("_ratio"), + "By" => Some("_bytes"), + "ms" => Some("_milliseconds"), + _ => None, + } +} + +#[allow(clippy::ptr_arg)] +fn sanitize_name(s: &Cow<'static, str>) -> Cow<'static, str> { + // prefix chars to add in case name starts with number + let mut prefix = ""; + + // Find first invalid char + if let Some((replace_idx, _)) = s.char_indices().find(|(i, c)| { + if *i == 0 && c.is_ascii_digit() { + // first char is number, add prefix and replace reset of chars + prefix = "_"; + true + } else { + // keep checking + !c.is_alphanumeric() && *c != '_' && *c != ':' + } + }) { + // up to `replace_idx` have been validated, convert the rest + let (valid, rest) = s.split_at(replace_idx); + Cow::Owned( + prefix + .chars() + .chain(valid.chars()) + .chain(rest.chars().map(|c| { + if c.is_ascii_alphanumeric() || c == '_' || c == ':' { + c + } else { + '_' + } + })) + .collect(), + ) } else { - labels.push(build_label_pair( - &Key::new(OTEL_SCOPE_VERSION), - &Value::String(StringValue::from("")), - )); + s.clone() // no invalid chars found, return existing } - labels } -fn build_label_pair(key: &Key, value: &Value) -> prometheus::proto::LabelPair { - let mut lp = prometheus::proto::LabelPair::new(); - lp.set_name(sanitize(key.as_str())); - lp.set_value(value.to_string()); +fn sanitize_prom_kv(s: &str) -> String { + s.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == ':' { + c + } else { + '_' + } + }) + .collect() +} - lp +trait Numeric: fmt::Debug { + // lossy at large values for u64 and i64 but prometheus only handles floats + fn as_f64(&self) -> f64; } -fn get_metric_labels( - record: &Record<'_>, - resource: &Resource, - scope_labels: &mut Vec, -) -> Vec { - // Duplicate keys are resolved by taking the record label value over - // the resource value. - let iter = attributes::merge_iters(record.attributes().iter(), resource.iter()); - let mut labels: Vec = iter - .map(|(key, value)| build_label_pair(key, value)) - .collect(); - - labels.append(scope_labels); - labels +impl Numeric for u64 { + fn as_f64(&self) -> f64 { + *self as f64 + } } -struct PrometheusMetricDesc { - name: String, - help: String, +impl Numeric for i64 { + fn as_f64(&self) -> f64 { + *self as f64 + } } -fn get_metric_desc(record: &Record<'_>) -> PrometheusMetricDesc { - let desc = record.descriptor(); - let name = sanitize(desc.name()); - let help = desc - .description() - .cloned() - .unwrap_or_else(|| desc.name().to_string()); - PrometheusMetricDesc { name, help } +impl Numeric for f64 { + fn as_f64(&self) -> f64 { + *self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn name_sanitization() { + let tests = vec![ + ("nam€_with_3_width_rune", "nam__with_3_width_rune"), + ("`", "_"), + ( + r##"! "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWKYZ[]\^_abcdefghijklmnopqrstuvwkyz{|}~"##, + "________________0123456789:______ABCDEFGHIJKLMNOPQRSTUVWKYZ_____abcdefghijklmnopqrstuvwkyz____", + ), + + ("Avalid_23name", "Avalid_23name"), + ("_Avalid_23name", "_Avalid_23name"), + ("1valid_23name", "_1valid_23name"), + ("avalid_23name", "avalid_23name"), + ("Ava:lid_23name", "Ava:lid_23name"), + ("a lid_23name", "a_lid_23name"), + (":leading_colon", ":leading_colon"), + ("colon:in:the:middle", "colon:in:the:middle"), + ("", ""), + ]; + + for (input, want) in tests { + assert_eq!(want, sanitize_name(&input.into()), "input: {input}") + } + } } diff --git a/opentelemetry-prometheus/src/sanitize.rs b/opentelemetry-prometheus/src/sanitize.rs deleted file mode 100644 index 8a304ff2ad..0000000000 --- a/opentelemetry-prometheus/src/sanitize.rs +++ /dev/null @@ -1,67 +0,0 @@ -/// sanitize returns a string that is truncated to 100 characters if it's too -/// long, and replaces non-alphanumeric characters to underscores. -pub(crate) fn sanitize>(raw: T) -> String { - let mut escaped = raw - .as_ref() - .chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .peekable(); - - let prefix = if escaped.peek().map_or(false, |c| c.is_ascii_digit()) { - "key_" - } else if escaped.peek().map_or(false, |&c| c == '_') { - "key" - } else { - "" - }; - - prefix.chars().chain(escaped).take(100).collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::borrow::Cow; - - fn key_data() -> Vec<(&'static str, Cow<'static, str>, Cow<'static, str>)> { - vec![ - ( - "replace character", - "test/key-1".into(), - "test_key_1".into(), - ), - ( - "add prefix if starting with digit", - "0123456789".into(), - "key_0123456789".into(), - ), - ( - "add prefix if starting with _", - "_0123456789".into(), - "key_0123456789".into(), - ), - ( - "starts with _ after sanitization", - "/0123456789".into(), - "key_0123456789".into(), - ), - ( - "limits to 100", - "a".repeat(101).into(), - "a".repeat(100).into(), - ), - ( - "valid input", - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0123456789".into(), - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0123456789".into(), - ), - ] - } - - #[test] - fn sanitize_key_names() { - for (name, raw, sanitized) in key_data() { - assert_eq!(sanitize(raw), sanitized, "{} doesn't match", name) - } - } -} diff --git a/opentelemetry-prometheus/tests/data/conflict_help_two_counters_1.txt b/opentelemetry-prometheus/tests/data/conflict_help_two_counters_1.txt new file mode 100644 index 0000000000..843e43ba53 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_help_two_counters_1.txt @@ -0,0 +1,11 @@ +# HELP bar_bytes_total meter a bar +# TYPE bar_bytes_total counter +bar_bytes_total{type="bar",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_bytes_total{type="bar",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_help_two_counters_2.txt b/opentelemetry-prometheus/tests/data/conflict_help_two_counters_2.txt new file mode 100644 index 0000000000..61c55238dc --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_help_two_counters_2.txt @@ -0,0 +1,11 @@ +# HELP bar_bytes_total meter b bar +# TYPE bar_bytes_total counter +bar_bytes_total{type="bar",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_bytes_total{type="bar",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_help_two_histograms_1.txt b/opentelemetry-prometheus/tests/data/conflict_help_two_histograms_1.txt new file mode 100644 index 0000000000..ca3780ecdc --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_help_two_histograms_1.txt @@ -0,0 +1,45 @@ +# HELP bar_bytes meter a bar +# TYPE bar_bytes histogram +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="0"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="25"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="50"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="75"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="100"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="250"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="750"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="1000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="2500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="7500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="+Inf"} 1 +bar_bytes_sum{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_bytes_count{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="0"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="25"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="50"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="75"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="100"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="250"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="750"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="1000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="2500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="7500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="+Inf"} 1 +bar_bytes_sum{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +bar_bytes_count{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_help_two_histograms_2.txt b/opentelemetry-prometheus/tests/data/conflict_help_two_histograms_2.txt new file mode 100644 index 0000000000..939d7da091 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_help_two_histograms_2.txt @@ -0,0 +1,45 @@ +# HELP bar_bytes meter b bar +# TYPE bar_bytes histogram +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="0"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="25"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="50"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="75"} 0 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="100"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="250"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="750"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="1000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="2500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="7500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="+Inf"} 1 +bar_bytes_sum{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_bytes_count{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="0"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="25"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="50"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="75"} 0 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="100"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="250"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="750"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="1000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="2500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="7500"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10000"} 1 +bar_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="+Inf"} 1 +bar_bytes_sum{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +bar_bytes_count{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_1.txt b/opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_1.txt new file mode 100644 index 0000000000..e70f86b90e --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_1.txt @@ -0,0 +1,11 @@ +# HELP bar_bytes meter a bar +# TYPE bar_bytes gauge +bar_bytes{type="bar",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_bytes{type="bar",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_2.txt b/opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_2.txt new file mode 100644 index 0000000000..f5f332bffe --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_help_two_updowncounters_2.txt @@ -0,0 +1,11 @@ +# HELP bar_bytes meter b bar +# TYPE bar_bytes gauge +bar_bytes{type="bar",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_bytes{type="bar",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_1.txt b/opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_1.txt new file mode 100644 index 0000000000..a9e57c8e95 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_1.txt @@ -0,0 +1,9 @@ +# HELP foo_total meter foo +# TYPE foo_total counter +foo_total{type="foo",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_2.txt b/opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_2.txt new file mode 100644 index 0000000000..f52fd05ca2 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_type_counter_and_updowncounter_2.txt @@ -0,0 +1,9 @@ +# HELP foo_total meter foo +# TYPE foo_total gauge +foo_total{type="foo",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_1.txt b/opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_1.txt new file mode 100644 index 0000000000..6ec9f665bf --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_1.txt @@ -0,0 +1,9 @@ +# HELP foo_bytes meter gauge foo +# TYPE foo_bytes gauge +foo_bytes{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_2.txt b/opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_2.txt new file mode 100644 index 0000000000..1722eb2ec6 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_type_histogram_and_updowncounter_2.txt @@ -0,0 +1,26 @@ +# HELP foo_bytes meter histogram foo +# TYPE foo_bytes histogram +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="0"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="25"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="50"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="75"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="100"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="250"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="750"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="1000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="2500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="7500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="+Inf"} 1 +foo_bytes_sum{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +foo_bytes_count{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_unit_two_counters.txt b/opentelemetry-prometheus/tests/data/conflict_unit_two_counters.txt new file mode 100644 index 0000000000..57546f0d79 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_unit_two_counters.txt @@ -0,0 +1,11 @@ +# HELP bar_total meter bar +# TYPE bar_total counter +bar_total{type="bar",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_total{type="bar",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_unit_two_histograms.txt b/opentelemetry-prometheus/tests/data/conflict_unit_two_histograms.txt new file mode 100644 index 0000000000..8774aba323 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_unit_two_histograms.txt @@ -0,0 +1,45 @@ +# HELP bar meter histogram bar +# TYPE bar histogram +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="0"} 0 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5"} 0 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10"} 0 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="25"} 0 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="50"} 0 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="75"} 0 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="100"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="250"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="500"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="750"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="1000"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="2500"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5000"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="7500"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10000"} 1 +bar_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="+Inf"} 1 +bar_sum{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar_count{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="0"} 0 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5"} 0 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10"} 0 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="25"} 0 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="50"} 0 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="75"} 0 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="100"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="250"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="500"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="750"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="1000"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="2500"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5000"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="7500"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10000"} 1 +bar_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="+Inf"} 1 +bar_sum{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +bar_count{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/conflict_unit_two_updowncounters.txt b/opentelemetry-prometheus/tests/data/conflict_unit_two_updowncounters.txt new file mode 100644 index 0000000000..5365cff9ce --- /dev/null +++ b/opentelemetry-prometheus/tests/data/conflict_unit_two_updowncounters.txt @@ -0,0 +1,11 @@ +# HELP bar meter gauge bar +# TYPE bar gauge +bar{type="bar",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +bar{type="bar",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/counter.txt b/opentelemetry-prometheus/tests/data/counter.txt new file mode 100644 index 0000000000..053aba6e80 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/counter.txt @@ -0,0 +1,10 @@ +# HELP foo_milliseconds_total a simple counter +# TYPE foo_milliseconds_total counter +foo_milliseconds_total{A="B",C="D",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 24.3 +foo_milliseconds_total{A="D",C="B",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 5 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/custom_resource.txt b/opentelemetry-prometheus/tests/data/custom_resource.txt new file mode 100644 index 0000000000..200c8a1bc8 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/custom_resource.txt @@ -0,0 +1,9 @@ +# HELP foo_total a simple counter +# TYPE foo_total counter +foo_total{A="B",C="D",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 24.3 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{A="B",C="D",service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/empty_resource.txt b/opentelemetry-prometheus/tests/data/empty_resource.txt new file mode 100644 index 0000000000..e313006e34 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/empty_resource.txt @@ -0,0 +1,9 @@ +# HELP foo_total a simple counter +# TYPE foo_total counter +foo_total{A="B",C="D",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 24.3 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info 1 diff --git a/opentelemetry-prometheus/tests/data/gauge.txt b/opentelemetry-prometheus/tests/data/gauge.txt new file mode 100644 index 0000000000..c37ca8d33d --- /dev/null +++ b/opentelemetry-prometheus/tests/data/gauge.txt @@ -0,0 +1,9 @@ +# HELP bar_ratio a fun little gauge +# TYPE bar_ratio gauge +bar_ratio{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 0.75 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/histogram.txt b/opentelemetry-prometheus/tests/data/histogram.txt new file mode 100644 index 0000000000..490e49c7a5 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/histogram.txt @@ -0,0 +1,21 @@ +# HELP histogram_baz_bytes a very nice histogram +# TYPE histogram_baz_bytes histogram +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="0"} 0 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="5"} 0 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="10"} 1 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="25"} 2 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="50"} 2 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="75"} 2 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="100"} 2 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="250"} 4 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="500"} 4 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="1000"} 4 +histogram_baz_bytes_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="+Inf"} 4 +histogram_baz_bytes_sum{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 236 +histogram_baz_bytes_count{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 4 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/multi_scopes.txt b/opentelemetry-prometheus/tests/data/multi_scopes.txt new file mode 100644 index 0000000000..e910457c2a --- /dev/null +++ b/opentelemetry-prometheus/tests/data/multi_scopes.txt @@ -0,0 +1,13 @@ +# HELP bar_milliseconds_total meter bar counter +# TYPE bar_milliseconds_total counter +bar_milliseconds_total{type="bar",otel_scope_name="meterbar",otel_scope_version="v0.1.0"} 200 +# HELP foo_milliseconds_total meter foo counter +# TYPE foo_milliseconds_total counter +foo_milliseconds_total{type="foo",otel_scope_name="meterfoo",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="meterbar",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="meterfoo",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/no_conflict_two_counters.txt b/opentelemetry-prometheus/tests/data/no_conflict_two_counters.txt new file mode 100644 index 0000000000..55f6c4965f --- /dev/null +++ b/opentelemetry-prometheus/tests/data/no_conflict_two_counters.txt @@ -0,0 +1,11 @@ +# HELP foo_bytes_total meter counter foo +# TYPE foo_bytes_total counter +foo_bytes_total{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +foo_bytes_total{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/no_conflict_two_histograms.txt b/opentelemetry-prometheus/tests/data/no_conflict_two_histograms.txt new file mode 100644 index 0000000000..634ad5e2de --- /dev/null +++ b/opentelemetry-prometheus/tests/data/no_conflict_two_histograms.txt @@ -0,0 +1,45 @@ +# HELP foo_bytes meter histogram foo +# TYPE foo_bytes histogram +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="0"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="25"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="50"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="75"} 0 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="100"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="250"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="750"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="1000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="2500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="5000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="7500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="10000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0",le="+Inf"} 1 +foo_bytes_sum{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +foo_bytes_count{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="0"} 0 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5"} 0 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10"} 0 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="25"} 0 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="50"} 0 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="75"} 0 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="100"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="250"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="750"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="1000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="2500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="5000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="7500"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="10000"} 1 +foo_bytes_bucket{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0",le="+Inf"} 1 +foo_bytes_sum{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +foo_bytes_count{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/no_conflict_two_updowncounters.txt b/opentelemetry-prometheus/tests/data/no_conflict_two_updowncounters.txt new file mode 100644 index 0000000000..17b7262096 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/no_conflict_two_updowncounters.txt @@ -0,0 +1,11 @@ +# HELP foo_bytes meter gauge foo +# TYPE foo_bytes gauge +foo_bytes{A="B",otel_scope_name="ma",otel_scope_version="v0.1.0"} 100 +foo_bytes{A="B",otel_scope_name="mb",otel_scope_version="v0.1.0"} 100 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="ma",otel_scope_version="v0.1.0"} 1 +otel_scope_info{otel_scope_name="mb",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/sanitized_labels.txt b/opentelemetry-prometheus/tests/data/sanitized_labels.txt new file mode 100644 index 0000000000..79ec70e61f --- /dev/null +++ b/opentelemetry-prometheus/tests/data/sanitized_labels.txt @@ -0,0 +1,9 @@ +# HELP foo_total a sanitary counter +# TYPE foo_total counter +foo_total{A_B="Q",C_D="Y;Z",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 24.3 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/sanitized_names.txt b/opentelemetry-prometheus/tests/data/sanitized_names.txt new file mode 100644 index 0000000000..5a2f337b1f --- /dev/null +++ b/opentelemetry-prometheus/tests/data/sanitized_names.txt @@ -0,0 +1,35 @@ +# HELP _0invalid_counter_name_total a counter with an invalid name +# TYPE _0invalid_counter_name_total counter +_0invalid_counter_name_total{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 100 +# HELP bar a fun little gauge +# TYPE bar gauge +bar{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 75 +# HELP invalid_gauge_name a gauge with an invalid name +# TYPE invalid_gauge_name gauge +invalid_gauge_name{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 100 +# HELP invalid_hist_name a histogram with an invalid name +# TYPE invalid_hist_name histogram +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="0"} 0 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="5"} 0 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="10"} 0 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="25"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="50"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="75"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="100"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="250"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="500"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="750"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="1000"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="2500"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="5000"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="7500"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="10000"} 1 +invalid_hist_name_bucket{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0",le="+Inf"} 1 +invalid_hist_name_sum{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 23 +invalid_hist_name_count{A="B",C="D",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/without_scope_and_target_info.txt b/opentelemetry-prometheus/tests/data/without_scope_and_target_info.txt new file mode 100644 index 0000000000..9a551fd8ef --- /dev/null +++ b/opentelemetry-prometheus/tests/data/without_scope_and_target_info.txt @@ -0,0 +1,3 @@ +# HELP bar_bytes_total a fun little counter +# TYPE bar_bytes_total counter +bar_bytes_total{A="B",C="D"} 3 diff --git a/opentelemetry-prometheus/tests/data/without_scope_info.txt b/opentelemetry-prometheus/tests/data/without_scope_info.txt new file mode 100644 index 0000000000..090520bd10 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/without_scope_info.txt @@ -0,0 +1,6 @@ +# HELP bar_ratio a fun little gauge +# TYPE bar_ratio gauge +bar_ratio{A="B",C="D"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="rust",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/opentelemetry-prometheus/tests/data/without_target_info.txt b/opentelemetry-prometheus/tests/data/without_target_info.txt new file mode 100644 index 0000000000..69f0e83668 --- /dev/null +++ b/opentelemetry-prometheus/tests/data/without_target_info.txt @@ -0,0 +1,6 @@ +# HELP foo_total a simple counter +# TYPE foo_total counter +foo_total{A="B",C="D",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 24.3 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 diff --git a/opentelemetry-prometheus/tests/integration_test.rs b/opentelemetry-prometheus/tests/integration_test.rs index 0cd3fbe5c1..c97434eb02 100644 --- a/opentelemetry-prometheus/tests/integration_test.rs +++ b/opentelemetry-prometheus/tests/integration_test.rs @@ -1,218 +1,727 @@ -use opentelemetry::sdk::export::metrics::aggregation; -use opentelemetry::sdk::metrics::{controllers, processors, selectors}; -use opentelemetry::sdk::Resource; -use opentelemetry::Context; -use opentelemetry::{metrics::MeterProvider, KeyValue}; -use opentelemetry_prometheus::{ExporterConfig, PrometheusExporter}; +use std::fs; +use std::path::Path; +use std::time::Duration; + +use opentelemetry_api::metrics::{Meter, MeterProvider as _, Unit}; +use opentelemetry_api::KeyValue; +use opentelemetry_api::{Context, Key}; +use opentelemetry_prometheus::ExporterBuilder; +use opentelemetry_sdk::metrics::{new_view, Aggregation, Instrument, MeterProvider, Stream}; +use opentelemetry_sdk::resource::{ + EnvResourceDetector, SdkProvidedResourceDetector, TelemetryResourceDetector, +}; +use opentelemetry_sdk::Resource; +use opentelemetry_semantic_conventions::resource::{SERVICE_NAME, TELEMETRY_SDK_VERSION}; use prometheus::{Encoder, TextEncoder}; #[test] -fn free_unused_instruments() { - let cx = Context::new(); - let controller = controllers::basic(processors::factory( - selectors::simple::histogram(vec![-0.5, 1.0]), - aggregation::cumulative_temporality_selector(), - )) - .with_resource(Resource::new(vec![KeyValue::new("R", "V")])) - .build(); - let exporter = opentelemetry_prometheus::exporter(controller).init(); - let mut expected = Vec::new(); - - { - let meter = - exporter - .meter_provider() - .unwrap() - .versioned_meter("test", Some("v0.1.0"), None); - let counter = meter.f64_counter("counter").init(); - - let attributes = vec![KeyValue::new("A", "B"), KeyValue::new("C", "D")]; - - counter.add(&cx, 10.0, &attributes); - counter.add(&cx, 5.3, &attributes); - - expected.push(r#"counter_total{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0"} 15.3"#); - expected.push(r#"otel_scope_info{otel_scope_name="test",otel_scope_version="v0.1.0"} 1"#); +fn prometheus_exporter_integration() { + struct TestCase { + name: &'static str, + empty_resource: bool, + custom_resource_attrs: Vec, + record_metrics: Box, + builder: ExporterBuilder, + expected_file: &'static str, } - // Standard export - compare_export(&exporter, expected.clone()); - // Final export before instrument dropped - compare_export(&exporter, expected.clone()); - // Instrument dropped, but last value kept by prom exporter - compare_export(&exporter, expected); -} - -#[test] -fn test_add() { - let cx = Context::new(); - let controller = controllers::basic(processors::factory( - selectors::simple::histogram(vec![-0.5, 1.0]), - aggregation::cumulative_temporality_selector(), - )) - .with_resource(Resource::new(vec![KeyValue::new("R", "V")])) - .build(); - let exporter = opentelemetry_prometheus::exporter(controller) - .with_config(ExporterConfig::default().with_scope_info(false)) - .init(); - - let meter = exporter - .meter_provider() - .unwrap() - .versioned_meter("test", None, None); - - let up_down_counter = meter.f64_up_down_counter("updowncounter").init(); - let counter = meter.f64_counter("counter").init(); - let histogram = meter.f64_histogram("my.histogram").init(); - - let attributes = vec![KeyValue::new("A", "B"), KeyValue::new("C", "D")]; - - let mut expected = Vec::new(); - - counter.add(&cx, 10.0, &attributes); - counter.add(&cx, 5.3, &attributes); - - expected.push(r#"counter_total{A="B",C="D",R="V"} 15.3"#); - let cb_attributes = attributes.clone(); - let gauge = meter.i64_observable_gauge("intgauge").init(); - meter - .register_callback(move |cx| gauge.observe(cx, 1, cb_attributes.as_ref())) - .unwrap(); - - expected.push(r#"intgauge{A="B",C="D",R="V"} 1"#); - - histogram.record(&cx, -0.6, &attributes); - histogram.record(&cx, -0.4, &attributes); - histogram.record(&cx, 0.6, &attributes); - histogram.record(&cx, 20.0, &attributes); - - expected.push(r#"my_histogram_bucket{A="B",C="D",R="V",le="+Inf"} 4"#); - expected.push(r#"my_histogram_bucket{A="B",C="D",R="V",le="-0.5"} 1"#); - expected.push(r#"my_histogram_bucket{A="B",C="D",R="V",le="1"} 3"#); - expected.push(r#"my_histogram_count{A="B",C="D",R="V"} 4"#); - expected.push(r#"my_histogram_sum{A="B",C="D",R="V"} 19.6"#); - - up_down_counter.add(&cx, 10.0, &attributes); - up_down_counter.add(&cx, -3.2, &attributes); + impl Default for TestCase { + fn default() -> Self { + TestCase { + name: "", + empty_resource: false, + custom_resource_attrs: Vec::new(), + record_metrics: Box::new(|_, _| {}), + builder: ExporterBuilder::default(), + expected_file: "", + } + } + } - expected.push(r#"updowncounter{A="B",C="D",R="V"} 6.8"#); + let test_cases = vec![ + TestCase { + name: "counter", + expected_file: "counter.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![ + Key::new("A").string("B"), + Key::new("C").string("D"), + Key::new("E").bool(true), + Key::new("F").i64(42), + ]; + let counter = meter + .f64_counter("foo") + .with_description("a simple counter") + .with_unit(Unit::new("ms")) + .init(); + counter.add(cx, 5.0, &attrs); + counter.add(cx, 10.3, &attrs); + counter.add(cx, 9.0, &attrs); + let attrs2 = vec![ + Key::new("A").string("D"), + Key::new("C").string("B"), + Key::new("E").bool(true), + Key::new("F").i64(42), + ]; + counter.add(cx, 5.0, &attrs2); + }), + ..Default::default() + }, + TestCase { + name: "gauge", + expected_file: "gauge.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![Key::new("A").string("B"), Key::new("C").string("D")]; + let gauge = meter + .f64_up_down_counter("bar") + .with_description("a fun little gauge") + .with_unit(Unit::new("1")) + .init(); + gauge.add(cx, 1.0, &attrs); + gauge.add(cx, -0.25, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "histogram", + expected_file: "histogram.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![Key::new("A").string("B"), Key::new("C").string("D")]; + let histogram = meter + .f64_histogram("histogram_baz") + .with_description("a very nice histogram") + .with_unit(Unit::new("By")) + .init(); + histogram.record(cx, 23.0, &attrs); + histogram.record(cx, 7.0, &attrs); + histogram.record(cx, 101.0, &attrs); + histogram.record(cx, 105.0, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "sanitized attributes to labels", + expected_file: "sanitized_labels.txt", + builder: ExporterBuilder::default().without_units(), + record_metrics: Box::new(|cx, meter| { + let attrs = vec![ + // exact match, value should be overwritten + Key::new("A.B").string("X"), + Key::new("A.B").string("Q"), + // unintended match due to sanitization, values should be concatenated + Key::new("C.D").string("Y"), + Key::new("C/D").string("Z"), + ]; + let counter = meter + .f64_counter("foo") + .with_description("a sanitary counter") + // This unit is not added to + .with_unit(Unit::new("By")) + .init(); + counter.add(cx, 5.0, &attrs); + counter.add(cx, 10.3, &attrs); + counter.add(cx, 9.0, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "invalid instruments are renamed", + expected_file: "sanitized_names.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![Key::new("A").string("B"), Key::new("C").string("D")]; + // Valid. + let mut gauge = meter + .f64_up_down_counter("bar") + .with_description("a fun little gauge") + .init(); + gauge.add(cx, 100., &attrs); + gauge.add(cx, -25.0, &attrs); + + // Invalid, will be renamed. + gauge = meter + .f64_up_down_counter("invalid.gauge.name") + .with_description("a gauge with an invalid name") + .init(); + gauge.add(cx, 100.0, &attrs); + + let counter = meter + .f64_counter("0invalid.counter.name") + .with_description("a counter with an invalid name") + .init(); + counter.add(cx, 100.0, &attrs); + + let histogram = meter + .f64_histogram("invalid.hist.name") + .with_description("a histogram with an invalid name") + .init(); + histogram.record(cx, 23.0, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "empty resource", + empty_resource: true, + expected_file: "empty_resource.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![ + Key::new("A").string("B"), + Key::new("C").string("D"), + Key::new("E").bool(true), + Key::new("F").i64(42), + ]; + let counter = meter + .f64_counter("foo") + .with_description("a simple counter") + .init(); + counter.add(cx, 5.0, &attrs); + counter.add(cx, 10.3, &attrs); + counter.add(cx, 9.0, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "custom resource", + custom_resource_attrs: vec![Key::new("A").string("B"), Key::new("C").string("D")], + expected_file: "custom_resource.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![ + Key::new("A").string("B"), + Key::new("C").string("D"), + Key::new("E").bool(true), + Key::new("F").i64(42), + ]; + let counter = meter + .f64_counter("foo") + .with_description("a simple counter") + .init(); + counter.add(cx, 5., &attrs); + counter.add(cx, 10.3, &attrs); + counter.add(cx, 9.0, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "without target_info", + builder: ExporterBuilder::default().without_target_info(), + expected_file: "without_target_info.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![ + Key::new("A").string("B"), + Key::new("C").string("D"), + Key::new("E").bool(true), + Key::new("F").i64(42), + ]; + let counter = meter + .f64_counter("foo") + .with_description("a simple counter") + .init(); + counter.add(cx, 5.0, &attrs); + counter.add(cx, 10.3, &attrs); + counter.add(cx, 9.0, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "without scope_info", + builder: ExporterBuilder::default().without_scope_info(), + expected_file: "without_scope_info.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![Key::new("A").string("B"), Key::new("C").string("D")]; + let gauge = meter + .i64_up_down_counter("bar") + .with_description("a fun little gauge") + .with_unit(Unit::new("1")) + .init(); + gauge.add(cx, 2, &attrs); + gauge.add(cx, -1, &attrs); + }), + ..Default::default() + }, + TestCase { + name: "without scope_info and target_info", + builder: ExporterBuilder::default() + .without_scope_info() + .without_target_info(), + expected_file: "without_scope_and_target_info.txt", + record_metrics: Box::new(|cx, meter| { + let attrs = vec![Key::new("A").string("B"), Key::new("C").string("D")]; + let counter = meter + .u64_counter("bar") + .with_description("a fun little counter") + .with_unit(Unit::new("By")) + .init(); + counter.add(cx, 2, &attrs); + counter.add(cx, 1, &attrs); + }), + ..Default::default() + }, + ]; - compare_export(&exporter, expected) + for tc in test_cases { + let cx = Context::default(); + let registry = prometheus::Registry::new(); + let exporter = tc + .builder + .with_registry(registry.clone()) + .build() + .expect(&format!("exporter init for {}", tc.name)); + + let res = if tc.empty_resource { + Resource::empty() + } else { + Resource::from_detectors( + Duration::from_secs(0), + vec![ + Box::new(SdkProvidedResourceDetector), + Box::new(EnvResourceDetector::new()), + Box::new(TelemetryResourceDetector), + ], + ) + .merge(&mut Resource::new( + vec![ + // always specify service.name because the default depends on the running OS + SERVICE_NAME.string("prometheus_test"), + // Overwrite the semconv.TelemetrySDKVersionKey value so we don't need to update every version + TELEMETRY_SDK_VERSION.string("latest"), + ] + .into_iter() + .chain(tc.custom_resource_attrs.into_iter()), + )) + }; + + let provider = MeterProvider::builder() + .with_resource(res) + .with_reader(exporter) + .with_view( + new_view( + Instrument::new().name("histogram_*"), + Stream::new().aggregation(Aggregation::ExplicitBucketHistogram { + boundaries: vec![ + 0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 1000.0, + ], + no_min_max: false, + }), + ) + .unwrap(), + ) + .build(); + let meter = provider.versioned_meter("testmeter", Some("v0.1.0"), None); + (tc.record_metrics)(&cx, meter); + + let content = fs::read_to_string(Path::new("./tests/data").join(tc.expected_file)) + .expect(tc.expected_file); + gather_and_compare(registry, content, tc.name); + } } -#[test] -fn test_sanitization() { - let cx = Context::new(); - let controller = controllers::basic(processors::factory( - selectors::simple::histogram(vec![-0.5, 1.0]), - aggregation::cumulative_temporality_selector(), - )) - .with_resource(Resource::new(vec![KeyValue::new( - "service.name", - "Test Service", - )])) - .build(); - let exporter = opentelemetry_prometheus::exporter(controller) - .with_config(ExporterConfig::default().with_scope_info(false)) - .init(); - let meter = exporter - .meter_provider() - .unwrap() - .versioned_meter("test", None, None); - - let histogram = meter.f64_histogram("http.server.duration").init(); - let attributes = vec![ - KeyValue::new("http.method", "GET"), - KeyValue::new("http.host", "server"), - ]; - histogram.record(&cx, -0.6, &attributes); - histogram.record(&cx, -0.4, &attributes); - histogram.record(&cx, 0.6, &attributes); - histogram.record(&cx, 20.0, &attributes); - - let expected = vec![ - r#"http_server_duration_bucket{http_host="server",http_method="GET",service_name="Test Service",le="+Inf"} 4"#, - r#"http_server_duration_bucket{http_host="server",http_method="GET",service_name="Test Service",le="-0.5"} 1"#, - r#"http_server_duration_bucket{http_host="server",http_method="GET",service_name="Test Service",le="1"} 3"#, - r#"http_server_duration_count{http_host="server",http_method="GET",service_name="Test Service"} 4"#, - r#"http_server_duration_sum{http_host="server",http_method="GET",service_name="Test Service"} 19.6"#, - ]; - compare_export(&exporter, expected) +fn gather_and_compare(registry: prometheus::Registry, expected: String, name: &'static str) { + let mut output = Vec::new(); + let encoder = TextEncoder::new(); + let metric_families = registry.gather(); + encoder.encode(&metric_families, &mut output).unwrap(); + let output_string = String::from_utf8(output).unwrap(); + + assert_eq!(output_string, expected, "{name}"); } #[test] -fn test_scope_info() { +fn multiple_scopes() { let cx = Context::new(); - let controller = controllers::basic(processors::factory( - selectors::simple::histogram(vec![-0.5, 1.0]), - aggregation::cumulative_temporality_selector(), - )) - .with_resource(Resource::new(vec![KeyValue::new("R", "V")])) - .build(); - let exporter = opentelemetry_prometheus::exporter(controller).init(); - - let meter = exporter - .meter_provider() - .unwrap() - .versioned_meter("test", Some("v0.1.0"), None); - - let up_down_counter = meter.f64_up_down_counter("updowncounter").init(); - let counter = meter.f64_counter("counter").init(); - let histogram = meter.f64_histogram("my.histogram").init(); - - let attributes = vec![KeyValue::new("A", "B"), KeyValue::new("C", "D")]; - - let mut expected = Vec::new(); - - counter.add(&cx, 10.0, &attributes); - counter.add(&cx, 5.3, &attributes); - - expected.push(r#"counter_total{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0"} 15.3"#); - - let cb_attributes = attributes.clone(); - let gauge = meter.i64_observable_gauge("intgauge").init(); - meter - .register_callback(move |cx| gauge.observe(cx, 1, cb_attributes.as_ref())) + let registry = prometheus::Registry::new(); + let exporter = ExporterBuilder::default() + .with_registry(registry.clone()) + .build() .unwrap(); - expected.push( - r#"intgauge{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0"} 1"#, - ); + let resource = Resource::from_detectors( + Duration::from_secs(0), + vec![ + Box::new(SdkProvidedResourceDetector), + Box::new(EnvResourceDetector::new()), + Box::new(TelemetryResourceDetector), + ], + ) + .merge(&mut Resource::new( + vec![ + // always specify service.name because the default depends on the running OS + SERVICE_NAME.string("prometheus_test"), + // Overwrite the semconv.TelemetrySDKVersionKey value so we don't need to update every version + TELEMETRY_SDK_VERSION.string("latest"), + ] + .into_iter(), + )); + + let provider = MeterProvider::builder() + .with_reader(exporter) + .with_resource(resource) + .build(); + + let foo_counter = provider + .versioned_meter("meterfoo", Some("v0.1.0"), None) + .u64_counter("foo") + .with_unit(Unit::new("ms")) + .with_description("meter foo counter") + .init(); + foo_counter.add(&cx, 100, &[KeyValue::new("type", "foo")]); + + let bar_counter = provider + .versioned_meter("meterbar", Some("v0.1.0"), None) + .u64_counter("bar") + .with_unit(Unit::new("ms")) + .with_description("meter bar counter") + .init(); + bar_counter.add(&cx, 200, &[KeyValue::new("type", "bar")]); - histogram.record(&cx, -0.6, &attributes); - histogram.record(&cx, -0.4, &attributes); - histogram.record(&cx, 0.6, &attributes); - histogram.record(&cx, 20.0, &attributes); + let content = fs::read_to_string("./tests/data/multi_scopes.txt").unwrap(); + gather_and_compare(registry, content, "multi_scope"); +} - expected.push(r#"my_histogram_bucket{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0",le="+Inf"} 4"#); - expected.push(r#"my_histogram_bucket{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0",le="-0.5"} 1"#); - expected.push(r#"my_histogram_bucket{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0",le="1"} 3"#); - expected.push(r#"my_histogram_count{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0"} 4"#); - expected.push(r#"my_histogram_sum{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0"} 19.6"#); +#[test] +fn duplicate_metrics() { + struct TestCase { + name: &'static str, + custom_resource_attrs: Vec, + record_metrics: Box, + builder: ExporterBuilder, + expected_files: Vec<&'static str>, + } - up_down_counter.add(&cx, 10.0, &attributes); - up_down_counter.add(&cx, -3.2, &attributes); + impl Default for TestCase { + fn default() -> Self { + TestCase { + name: "", + custom_resource_attrs: Vec::new(), + record_metrics: Box::new(|_, _, _| {}), + builder: ExporterBuilder::default(), + expected_files: Vec::new(), + } + } + } - expected.push(r#"updowncounter{A="B",C="D",R="V",otel_scope_name="test",otel_scope_version="v0.1.0"} 6.8"#); - expected.push(r#"otel_scope_info{otel_scope_name="test",otel_scope_version="v0.1.0"} 1"#); + let test_cases = vec![ + TestCase { + name: "no_conflict_two_counters", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let foo_a = meter_a + .u64_counter("foo") + .with_unit(Unit::new("By")) + .with_description("meter counter foo") + .init(); + + foo_a.add(cx, 100, &[KeyValue::new("A", "B")]); + + let foo_b = meter_b + .u64_counter("foo") + .with_unit(Unit::new("By")) + .with_description("meter counter foo") + .init(); + + foo_b.add(cx, 100, &[KeyValue::new("A", "B")]); + }), + expected_files: vec!["no_conflict_two_counters.txt"], + ..Default::default() + }, + TestCase { + name: "no_conflict_two_updowncounters", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let foo_a = meter_a + .i64_up_down_counter("foo") + .with_unit(Unit::new("By")) + .with_description("meter gauge foo") + .init(); + + foo_a.add(cx, 100, &[KeyValue::new("A", "B")]); + + let foo_b = meter_b + .i64_up_down_counter("foo") + .with_unit(Unit::new("By")) + .with_description("meter gauge foo") + .init(); + + foo_b.add(cx, 100, &[KeyValue::new("A", "B")]); + }), + expected_files: vec!["no_conflict_two_updowncounters.txt"], + ..Default::default() + }, + TestCase { + name: "no_conflict_two_histograms", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let foo_a = meter_a + .i64_histogram("foo") + .with_unit(Unit::new("By")) + .with_description("meter histogram foo") + .init(); + + foo_a.record(cx, 100, &[KeyValue::new("A", "B")]); + + let foo_b = meter_b + .i64_histogram("foo") + .with_unit(Unit::new("By")) + .with_description("meter histogram foo") + .init(); + + foo_b.record(cx, 100, &[KeyValue::new("A", "B")]); + }), + expected_files: vec!["no_conflict_two_histograms.txt"], + ..Default::default() + }, + TestCase { + name: "conflict_help_two_counters", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let bar_a = meter_a + .u64_counter("bar") + .with_unit(Unit::new("By")) + .with_description("meter a bar") + .init(); + + bar_a.add(cx, 100, &[KeyValue::new("type", "bar")]); + + let bar_b = meter_b + .u64_counter("bar") + .with_unit(Unit::new("By")) + .with_description("meter b bar") + .init(); + + bar_b.add(cx, 100, &[KeyValue::new("type", "bar")]); + }), + expected_files: vec![ + "conflict_help_two_counters_1.txt", + "conflict_help_two_counters_2.txt", + ], + ..Default::default() + }, + TestCase { + name: "conflict_help_two_updowncounters", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let bar_a = meter_a + .i64_up_down_counter("bar") + .with_unit(Unit::new("By")) + .with_description("meter a bar") + .init(); + + bar_a.add(cx, 100, &[KeyValue::new("type", "bar")]); + + let bar_b = meter_b + .i64_up_down_counter("bar") + .with_unit(Unit::new("By")) + .with_description("meter b bar") + .init(); + + bar_b.add(cx, 100, &[KeyValue::new("type", "bar")]); + }), + expected_files: vec![ + "conflict_help_two_updowncounters_1.txt", + "conflict_help_two_updowncounters_2.txt", + ], + ..Default::default() + }, + TestCase { + name: "conflict_help_two_histograms", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let bar_a = meter_a + .i64_histogram("bar") + .with_unit(Unit::new("By")) + .with_description("meter a bar") + .init(); + + bar_a.record(cx, 100, &[KeyValue::new("A", "B")]); + + let bar_b = meter_b + .i64_histogram("bar") + .with_unit(Unit::new("By")) + .with_description("meter b bar") + .init(); + + bar_b.record(cx, 100, &[KeyValue::new("A", "B")]); + }), + expected_files: vec![ + "conflict_help_two_histograms_1.txt", + "conflict_help_two_histograms_2.txt", + ], + ..Default::default() + }, + TestCase { + name: "conflict_unit_two_counters", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let baz_a = meter_a + .u64_counter("bar") + .with_unit(Unit::new("By")) + .with_description("meter bar") + .init(); + + baz_a.add(cx, 100, &[KeyValue::new("type", "bar")]); + + let baz_b = meter_b + .u64_counter("bar") + .with_unit(Unit::new("ms")) + .with_description("meter bar") + .init(); + + baz_b.add(cx, 100, &[KeyValue::new("type", "bar")]); + }), + builder: ExporterBuilder::default().without_units(), + expected_files: vec!["conflict_unit_two_counters.txt"], + ..Default::default() + }, + TestCase { + name: "conflict_unit_two_updowncounters", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let bar_a = meter_a + .i64_up_down_counter("bar") + .with_unit(Unit::new("By")) + .with_description("meter gauge bar") + .init(); + + bar_a.add(cx, 100, &[KeyValue::new("type", "bar")]); + + let bar_b = meter_b + .i64_up_down_counter("bar") + .with_unit(Unit::new("ms")) + .with_description("meter gauge bar") + .init(); + + bar_b.add(cx, 100, &[KeyValue::new("type", "bar")]); + }), + builder: ExporterBuilder::default().without_units(), + expected_files: vec!["conflict_unit_two_updowncounters.txt"], + ..Default::default() + }, + TestCase { + name: "conflict_unit_two_histograms", + record_metrics: Box::new(|cx, meter_a, meter_b| { + let bar_a = meter_a + .i64_histogram("bar") + .with_unit(Unit::new("By")) + .with_description("meter histogram bar") + .init(); + + bar_a.record(cx, 100, &[KeyValue::new("A", "B")]); + + let bar_b = meter_b + .i64_histogram("bar") + .with_unit(Unit::new("ms")) + .with_description("meter histogram bar") + .init(); + + bar_b.record(cx, 100, &[KeyValue::new("A", "B")]); + }), + builder: ExporterBuilder::default().without_units(), + expected_files: vec!["conflict_unit_two_histograms.txt"], + ..Default::default() + }, + TestCase { + name: "conflict_type_counter_and_updowncounter", + record_metrics: Box::new(|cx, meter_a, _meter_b| { + let counter = meter_a + .u64_counter("foo") + .with_unit(Unit::new("By")) + .with_description("meter foo") + .init(); + + counter.add(cx, 100, &[KeyValue::new("type", "foo")]); + + let gauge = meter_a + .i64_up_down_counter("foo_total") + .with_unit(Unit::new("By")) + .with_description("meter foo") + .init(); + + gauge.add(cx, 200, &[KeyValue::new("type", "foo")]); + }), + builder: ExporterBuilder::default().without_units(), + expected_files: vec![ + "conflict_type_counter_and_updowncounter_1.txt", + "conflict_type_counter_and_updowncounter_2.txt", + ], + ..Default::default() + }, + TestCase { + name: "conflict_type_histogram_and_updowncounter", + record_metrics: Box::new(|cx, meter_a, _meter_b| { + let foo_a = meter_a + .i64_up_down_counter("foo") + .with_unit(Unit::new("By")) + .with_description("meter gauge foo") + .init(); + + foo_a.add(cx, 100, &[KeyValue::new("A", "B")]); + + let foo_histogram_a = meter_a + .i64_histogram("foo") + .with_unit(Unit::new("By")) + .with_description("meter histogram foo") + .init(); + + foo_histogram_a.record(cx, 100, &[KeyValue::new("A", "B")]); + }), + expected_files: vec![ + "conflict_type_histogram_and_updowncounter_1.txt", + "conflict_type_histogram_and_updowncounter_2.txt", + ], + ..Default::default() + }, + ]; - compare_export(&exporter, expected) + for tc in test_cases { + let cx = Context::default(); + let registry = prometheus::Registry::new(); + let exporter = tc + .builder + .with_registry(registry.clone()) + .build() + .expect(&format!("exporter init for {}", tc.name)); + + let resource = Resource::from_detectors( + Duration::from_secs(0), + vec![ + Box::new(SdkProvidedResourceDetector), + Box::new(EnvResourceDetector::new()), + Box::new(TelemetryResourceDetector), + ], + ) + .merge(&mut Resource::new( + vec![ + // always specify service.name because the default depends on the running OS + SERVICE_NAME.string("prometheus_test"), + // Overwrite the semconv.TelemetrySDKVersionKey value so we don't need to update every version + TELEMETRY_SDK_VERSION.string("latest"), + ] + .into_iter() + .chain(tc.custom_resource_attrs.into_iter()), + )); + + let provider = MeterProvider::builder() + .with_resource(resource) + .with_reader(exporter) + .build(); + + let meter_a = provider.versioned_meter("ma", Some("v0.1.0"), None); + let meter_b = provider.versioned_meter("mb", Some("v0.1.0"), None); + + (tc.record_metrics)(&cx, meter_a, meter_b); + + let possible_matches = tc + .expected_files + .into_iter() + .map(|f| fs::read_to_string(Path::new("./tests/data").join(f)).expect(f)) + .collect(); + gather_and_compare_multi(registry, possible_matches, tc.name); + } } -fn compare_export(exporter: &PrometheusExporter, mut expected: Vec<&'static str>) { +fn gather_and_compare_multi( + registry: prometheus::Registry, + expected: Vec, + name: &'static str, +) { let mut output = Vec::new(); let encoder = TextEncoder::new(); - let metric_families = exporter.registry().gather(); + let metric_families = registry.gather(); encoder.encode(&metric_families, &mut output).unwrap(); let output_string = String::from_utf8(output).unwrap(); - let mut metrics_only = output_string - .split_terminator('\n') - .filter(|line| !line.starts_with('#') && !line.is_empty()) - .collect::>(); - - metrics_only.sort_unstable(); - expected.sort_unstable(); - - assert_eq!(expected.join("\n"), metrics_only.join("\n")) + assert!( + expected.contains(&output_string), + "mismatched output in {name}" + ) } diff --git a/opentelemetry-proto/src/proto.rs b/opentelemetry-proto/src/proto.rs index 4c3c9bd7a4..5a16c8298b 100644 --- a/opentelemetry-proto/src/proto.rs +++ b/opentelemetry-proto/src/proto.rs @@ -66,9 +66,6 @@ pub mod tonic { } pub use crate::transform::common::tonic::Attributes; - - #[cfg(feature = "metrics")] - pub use crate::transform::metrics::tonic::FromNumber; } #[cfg(feature = "gen-protoc")] diff --git a/opentelemetry-proto/src/transform/common.rs b/opentelemetry-proto/src/transform/common.rs index 88a85f89ed..013cd68d78 100644 --- a/opentelemetry-proto/src/transform/common.rs +++ b/opentelemetry-proto/src/transform/common.rs @@ -19,7 +19,7 @@ pub mod tonic { impl From for InstrumentationScope { fn from(library: opentelemetry::sdk::InstrumentationLibrary) -> Self { InstrumentationScope { - name: library.name.to_string(), + name: library.name.into_owned(), attributes: Vec::new(), version: library.version.unwrap_or(Cow::Borrowed("")).to_string(), dropped_attributes_count: 0, @@ -27,6 +27,21 @@ pub mod tonic { } } + impl From<&opentelemetry::sdk::InstrumentationLibrary> for InstrumentationScope { + fn from(library: &opentelemetry::sdk::InstrumentationLibrary) -> Self { + InstrumentationScope { + name: library.name.to_string(), + attributes: Vec::new(), + version: library + .version + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + dropped_attributes_count: 0, + } + } + } + /// Wrapper type for Vec<[`KeyValue`](crate::proto::tonic::common::v1::KeyValue)> pub struct Attributes(pub ::std::vec::Vec); diff --git a/opentelemetry-proto/src/transform/metrics.rs b/opentelemetry-proto/src/transform/metrics.rs index b31e08e80c..db2fafff6f 100644 --- a/opentelemetry-proto/src/transform/metrics.rs +++ b/opentelemetry-proto/src/transform/metrics.rs @@ -5,46 +5,66 @@ #[allow(deprecated)] #[cfg(feature = "gen-tonic")] pub mod tonic { - use crate::proto::tonic::{ - common::v1::KeyValue, - metrics::v1::{number_data_point, AggregationTemporality}, - }; - use opentelemetry::{ - metrics::MetricsError, - sdk::{ - export::metrics::aggregation::Temporality, - metrics::sdk_api::{Number, NumberKind}, - }, - }; + use crate::proto::tonic::{common::v1::KeyValue, metrics::v1::AggregationTemporality}; + use crate::tonic::metrics::v1::{exemplar, number_data_point}; + use opentelemetry::{metrics::MetricsError, sdk::metrics::data::Temporality}; use opentelemetry::{Key, Value}; - /// Convert [`Number`](opentelemetry::sdk::metrics::sdk_api::Number) to target type based - /// on it's [`NumberKind`](opentelemetry::sdk::metrics::sdk_api::NumberKind). - pub trait FromNumber { - fn from_number(number: Number, number_kind: &NumberKind) -> Self; + impl From for exemplar::Value { + fn from(value: u64) -> Self { + exemplar::Value::AsInt(i64::try_from(value).unwrap_or_default()) + } } - impl FromNumber for number_data_point::Value { - fn from_number(number: Number, number_kind: &NumberKind) -> Self { - match &number_kind { - NumberKind::I64 | NumberKind::U64 => { - number_data_point::Value::AsInt(number.to_i64(number_kind)) - } - NumberKind::F64 => number_data_point::Value::AsDouble(number.to_f64(number_kind)), - } + impl From for exemplar::Value { + fn from(value: i64) -> Self { + exemplar::Value::AsInt(value) + } + } + + impl From for exemplar::Value { + fn from(value: f64) -> Self { + exemplar::Value::AsDouble(value) + } + } + + impl From for number_data_point::Value { + fn from(value: u64) -> Self { + number_data_point::Value::AsInt(i64::try_from(value).unwrap_or_default()) + } + } + + impl From for number_data_point::Value { + fn from(value: i64) -> Self { + number_data_point::Value::AsInt(value) + } + } + + impl From for number_data_point::Value { + fn from(value: f64) -> Self { + number_data_point::Value::AsDouble(value) } } impl From<(&Key, &Value)> for KeyValue { fn from(kv: (&Key, &Value)) -> Self { KeyValue { - key: kv.0.clone().into(), + key: kv.0.to_string(), value: Some(kv.1.clone().into()), } } } + impl From<&opentelemetry::KeyValue> for KeyValue { + fn from(kv: &opentelemetry::KeyValue) -> Self { + KeyValue { + key: kv.key.to_string(), + value: Some(kv.value.clone().into()), + } + } + } + impl From for AggregationTemporality { fn from(temporality: Temporality) -> Self { match temporality { diff --git a/opentelemetry-sdk/Cargo.toml b/opentelemetry-sdk/Cargo.toml index afef20f933..a97285591d 100644 --- a/opentelemetry-sdk/Cargo.toml +++ b/opentelemetry-sdk/Cargo.toml @@ -15,14 +15,14 @@ opentelemetry-http = { version = "0.7.0", path = "../opentelemetry-http", option async-std = { version = "1.6", features = ["unstable"], optional = true } async-trait = { version = "0.1", optional = true } crossbeam-channel = { version = "0.5", optional = true } -dashmap = { version = "5.1.0", optional = true } -fnv = { version = "1.0", optional = true } futures-channel = "0.3" futures-executor = "0.3" futures-util = { version = "0.3.17", default-features = false, features = ["std", "sink", "async-await-macro"] } once_cell = "1.10" +ordered-float = {version = "3.4.0", optional = true } percent-encoding = { version = "2.0", optional = true } rand = { version = "0.8", default-features = false, features = ["std", "std_rng"], optional = true } +regex = { version = "1.0", optional = true } serde = { version = "1.0", features = ["derive", "rc"], optional = true } serde_json = { version = "1", optional = true } thiserror = "1" @@ -45,7 +45,7 @@ crossbeam-queue = "0.3.1" default = ["trace"] trace = ["opentelemetry_api/trace", "crossbeam-channel", "rand", "async-trait", "percent-encoding"] jaeger_remote_sampler = ["trace", "opentelemetry-http", "http", "serde", "serde_json", "url"] -metrics = ["opentelemetry_api/metrics", "dashmap", "fnv"] +metrics = ["opentelemetry_api/metrics", "regex", "ordered-float"] testing = ["opentelemetry_api/testing", "trace", "metrics", "rt-async-std", "rt-tokio", "rt-tokio-current-thread", "tokio/macros", "tokio/rt-multi-thread"] rt-tokio = ["tokio", "tokio-stream"] rt-tokio-current-thread = ["tokio", "tokio-stream"] diff --git a/opentelemetry-sdk/src/export/metrics/aggregation/temporality.rs b/opentelemetry-sdk/src/export/metrics/aggregation/temporality.rs deleted file mode 100644 index afcc058fda..0000000000 --- a/opentelemetry-sdk/src/export/metrics/aggregation/temporality.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::export::metrics::aggregation::AggregationKind; -use crate::metrics::sdk_api::{Descriptor, InstrumentKind}; - -#[derive(Clone)] -struct ConstantTemporalitySelector(Temporality); - -impl TemporalitySelector for ConstantTemporalitySelector { - fn temporality_for(&self, _descriptor: &Descriptor, _kind: &AggregationKind) -> Temporality { - self.0 - } -} - -/// Returns an [`TemporalitySelector`] that returns a constant [`Temporality`]. -pub fn constant_temporality_selector(temporality: Temporality) -> impl TemporalitySelector + Clone { - ConstantTemporalitySelector(temporality) -} - -/// Returns an [`TemporalitySelector`] that always returns [`Temporality::Cumulative`]. -pub fn cumulative_temporality_selector() -> impl TemporalitySelector + Clone { - constant_temporality_selector(Temporality::Cumulative) -} - -/// Returns an [`TemporalitySelector`] that always returns [`Temporality::Delta`]. -pub fn delta_temporality_selector() -> impl TemporalitySelector + Clone { - constant_temporality_selector(Temporality::Delta) -} - -/// Returns a [`TemporalitySelector`] that always returns the cumulative [`Temporality`] to avoid -/// long-term memory requirements. -pub fn stateless_temporality_selector() -> impl TemporalitySelector + Clone { - constant_temporality_selector(Temporality::Cumulative) -} - -#[derive(Clone)] -struct StatelessTemporalitySelector; - -impl TemporalitySelector for StatelessTemporalitySelector { - fn temporality_for(&self, descriptor: &Descriptor, kind: &AggregationKind) -> Temporality { - if kind == &AggregationKind::SUM && descriptor.instrument_kind().precomputed_sum() { - Temporality::Cumulative - } else { - Temporality::Delta - } - } -} - -/// Temporality indicates the temporal aggregation exported by an exporter. -/// These bits may be OR-d together when multiple exporters are in use. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[non_exhaustive] -pub enum Temporality { - /// Indicates that an Exporter expects a Cumulative Aggregation. - Cumulative = 1, - - /// Indicates that an Exporter expects a Delta Aggregation. - Delta = 2, -} - -impl Temporality { - /// Tests whether `kind` includes a specific kind of exporter. - pub fn includes(&self, other: &Self) -> bool { - (*self as u32) & (*other as u32) != 0 - } - - /// Returns whether a temporality of this kind requires memory to export correctly. - pub fn memory_required(&self, kind: &InstrumentKind) -> bool { - match kind { - InstrumentKind::Histogram - | InstrumentKind::GaugeObserver - | InstrumentKind::Counter - | InstrumentKind::UpDownCounter => { - // Cumulative-oriented instruments: - self.includes(&Temporality::Cumulative) - } - - InstrumentKind::CounterObserver | InstrumentKind::UpDownCounterObserver => { - // Delta-oriented instruments: - self.includes(&Temporality::Delta) - } - } - } -} - -/// TemporalitySelector is a sub-interface of Exporter used to indicate -/// whether the Processor should compute Delta or Cumulative -/// Aggregations. -pub trait TemporalitySelector { - /// TemporalityFor should return the correct Temporality that - /// should be used when exporting data for the given metric - /// instrument and Aggregator kind. - fn temporality_for(&self, descriptor: &Descriptor, kind: &AggregationKind) -> Temporality; -} diff --git a/opentelemetry-sdk/src/export/metrics/mod.rs b/opentelemetry-sdk/src/export/metrics/mod.rs deleted file mode 100644 index d90a2e302b..0000000000 --- a/opentelemetry-sdk/src/export/metrics/mod.rs +++ /dev/null @@ -1,355 +0,0 @@ -//! Metrics Export - -use core::fmt; -use std::{sync::Arc, time::SystemTime}; - -use opentelemetry_api::{attributes, metrics::Result, Context, InstrumentationLibrary}; - -use crate::{ - metrics::{aggregators::Aggregator, sdk_api::Descriptor}, - Resource, -}; - -use self::aggregation::TemporalitySelector; - -pub mod aggregation; -mod stdout; - -pub use stdout::{stdout, ExportLine, ExportNumeric, StdoutExporter, StdoutExporterBuilder}; - -/// AggregatorSelector supports selecting the kind of `Aggregator` to use at -/// runtime for a specific metric instrument. -pub trait AggregatorSelector { - /// This allocates a variable number of aggregators of a kind suitable for - /// the requested export. - /// - /// When the call returns `None`, the metric instrument is explicitly disabled. - /// - /// This must return a consistent type to avoid confusion in later stages of - /// the metrics export process, e.g., when merging or checkpointing - /// aggregators for a specific instrument. - /// - /// This call should not block. - fn aggregator_for(&self, descriptor: &Descriptor) -> Option>; -} - -/// A container for the common elements for exported metric data that are shared -/// by the `Accumulator`->`Processor` and `Processor`->`Exporter` steps. -#[derive(Debug)] -pub struct Metadata<'a> { - descriptor: &'a Descriptor, - attributes: &'a attributes::AttributeSet, -} - -impl<'a> Metadata<'a> { - /// Create a new `Metadata` instance. - pub fn new(descriptor: &'a Descriptor, attributes: &'a attributes::AttributeSet) -> Self { - { - Metadata { - descriptor, - attributes, - } - } - } - - /// A description of the metric instrument being exported. - pub fn descriptor(&self) -> &Descriptor { - self.descriptor - } - - /// The attributes associated with the instrument and the aggregated data. - pub fn attributes(&self) -> &attributes::AttributeSet { - self.attributes - } -} - -/// Allows `Accumulator` implementations to construct new `Accumulation`s to -/// send to `Processor`s. The `Descriptor`, `Attributes`, `Resource`, and -/// `Aggregator` represent aggregate metric events received over a single -/// collection period. -pub fn accumulation<'a>( - descriptor: &'a Descriptor, - attributes: &'a attributes::AttributeSet, - aggregator: &'a Arc, -) -> Accumulation<'a> { - Accumulation::new(descriptor, attributes, aggregator) -} - -/// A container for the exported data for a single metric instrument and attribute -/// set, as prepared by an `Accumulator` for the `Processor`. -pub struct Accumulation<'a> { - metadata: Metadata<'a>, - aggregator: &'a Arc, -} - -impl<'a> Accumulation<'a> { - /// Create a new `Record` instance. - pub fn new( - descriptor: &'a Descriptor, - attributes: &'a attributes::AttributeSet, - aggregator: &'a Arc, - ) -> Self { - Accumulation { - metadata: Metadata::new(descriptor, attributes), - aggregator, - } - } - - /// A description of the metric instrument being exported. - pub fn descriptor(&self) -> &Descriptor { - self.metadata.descriptor - } - - /// The attributes associated with the instrument and the aggregated data. - pub fn attributes(&self) -> &attributes::AttributeSet { - self.metadata.attributes - } - - /// The checkpointed aggregator for this metric. - pub fn aggregator(&self) -> &Arc { - self.aggregator - } -} - -impl<'a> fmt::Debug for Accumulation<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Accumulation") - .field("metadata", &self.metadata) - .finish() - } -} - -/// Metric data processor. -/// -/// Locked processors are responsible gathering exported results from the SDK during -/// collection, and deciding over which dimensions to group the exported data. -/// -/// The `process` method is called during collection in a single-threaded -/// context from the SDK, after the aggregator is checkpointed, allowing the -/// processor to build the set of metrics currently being exported. -pub trait LockedProcessor { - /// Process is called by the SDK once per internal record, passing the export - /// [`Accumulation`] (a Descriptor, the corresponding attributes, and the - /// checkpointed aggregator). - /// - /// This call has no [`Context`] argument because it is expected to perform only - /// computation. An SDK is not expected to call exporters from with Process, use - /// a controller for that. - fn process(&mut self, accumulation: Accumulation<'_>) -> Result<()>; -} - -/// A container for the exported data for a single metric instrument and attribute -/// set, as prepared by the `Processor` for the `Exporter`. This includes the -/// effective start and end time for the aggregation. -pub struct Record<'a> { - metadata: Metadata<'a>, - aggregator: Option<&'a Arc>, - start: SystemTime, - end: SystemTime, -} - -impl Record<'_> { - /// A description of the metric instrument being exported. - pub fn descriptor(&self) -> &Descriptor { - self.metadata.descriptor - } - - /// The attributes associated with the instrument and the aggregated data. - pub fn attributes(&self) -> &attributes::AttributeSet { - self.metadata.attributes - } - - /// The aggregator for this metric - pub fn aggregator(&self) -> Option<&Arc> { - self.aggregator - } - - /// The start time of the interval covered by this aggregation. - pub fn start_time(&self) -> &SystemTime { - &self.start - } - - /// The end time of the interval covered by this aggregation. - pub fn end_time(&self) -> &SystemTime { - &self.end - } -} - -/// Exporter handles presentation of the checkpoint of aggregate -/// metrics. This is the final stage of a metrics export pipeline, -/// where metric data are formatted for a specific system. -pub trait MetricsExporter: TemporalitySelector { - /// Export is called immediately after completing a collection - /// pass in the SDK. - /// - /// The Context comes from the controller that initiated - /// collection. - /// - /// The InstrumentationLibraryReader interface refers to the - /// Processor that just completed collection. - fn export( - &self, - cx: &Context, - res: &Resource, - reader: &dyn InstrumentationLibraryReader, - ) -> Result<()>; -} - -/// InstrumentationLibraryReader is an interface for exporters to iterate -/// over one instrumentation library of metric data at a time. -pub trait InstrumentationLibraryReader { - /// ForEach calls the passed function once per instrumentation library, - /// allowing the caller to emit metrics grouped by the library that - /// produced them. - fn try_for_each( - &self, - f: &mut dyn FnMut(&InstrumentationLibrary, &mut dyn Reader) -> Result<()>, - ) -> Result<()>; -} - -/// Reader allows a controller to access a complete checkpoint of -/// aggregated metrics from the Processor for a single library of -/// metric data. This is passed to the Exporter which may then use -/// ForEach to iterate over the collection of aggregated metrics. -pub trait Reader { - /// ForEach iterates over aggregated checkpoints for all - /// metrics that were updated during the last collection - /// period. Each aggregated checkpoint returned by the - /// function parameter may return an error. - /// - /// The TemporalitySelector argument is used to determine - /// whether the Record is computed using Delta or Cumulative - /// aggregation. - /// - /// ForEach tolerates ErrNoData silently, as this is - /// expected from the Meter implementation. Any other kind - /// of error will immediately halt ForEach and return - /// the error to the caller. - fn try_for_each( - &mut self, - temp_selector: &dyn TemporalitySelector, - f: &mut dyn FnMut(&Record<'_>) -> Result<()>, - ) -> Result<()>; -} - -impl fmt::Debug for Record<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Record") - .field("metadata", &self.metadata) - .field("start", &self.start) - .field("end", &self.end) - .finish() - } -} - -/// The interface used to create checkpoints. -pub trait Checkpointer: Processor { - /// Synchronizes the checkpoint process and allows a single locked - /// checkpoint to be accessed at a time. - fn checkpoint( - &self, - f: &mut dyn FnMut(&mut dyn LockedCheckpointer) -> Result<()>, - ) -> Result<()>; -} - -/// The interface used by a controller to coordinate the processor with -/// accumulator(s) and exporter(s). -/// -/// The StartCollection() and FinishCollection() methods start and finish a -/// collection interval. Controllers call the Accumulator(s) during collection -/// to process Accumulations. -pub trait LockedCheckpointer { - /// Processes metric data for export. - /// - /// The `process` method is bracketed by `start_collection` and - /// `finish_collection` calls. - fn processor(&mut self) -> &mut dyn LockedProcessor; - - /// Reader returns the current data set. - /// - /// This may be called before and after collection. The implementation is - /// required to return the same value throughout its lifetime. - fn reader(&mut self) -> &mut dyn Reader; - - /// begins a collection interval. - fn start_collection(&mut self); - - /// ends a collection interval. - fn finish_collection(&mut self) -> Result<()>; -} - -/// An interface for producing configured [`Checkpointer`] instances. -pub trait CheckpointerFactory { - /// Creates a new configured checkpointer. - fn checkpointer(&self) -> Arc; -} - -/// Allows `Processor` implementations to construct export records. The -/// `Descriptor`, `Attributes`, and `Aggregator` represent aggregate metric events -/// received over a single collection period. -pub fn record<'a>( - descriptor: &'a Descriptor, - attributes: &'a attributes::AttributeSet, - aggregator: Option<&'a Arc>, - start: SystemTime, - end: SystemTime, -) -> Record<'a> { - Record { - metadata: Metadata::new(descriptor, attributes), - aggregator, - start, - end, - } -} - -/// A utility extension to allow upcasting. -/// -/// Can be removed once [trait_upcasting] is stablized. -/// -/// [trait_upcasting]: https://doc.rust-lang.org/unstable-book/language-features/trait-upcasting.html -pub trait AsDynProcessor { - /// Create an `Arc` from an impl of [`Processor`]. - fn as_dyn_processor<'a>(self: Arc) -> Arc - where - Self: 'a; -} - -impl AsDynProcessor for T { - fn as_dyn_processor<'a>(self: Arc) -> Arc - where - Self: 'a, - { - self - } -} - -/// Processor is responsible for deciding which kind of aggregation to use (via -/// `aggregation_selector`), gathering exported results from the SDK during -/// collection, and deciding over which dimensions to group the exported data. -/// -/// The SDK supports binding only one of these interfaces, as it has the sole -/// responsibility of determining which Aggregator to use for each record. -/// -/// The embedded AggregatorSelector interface is called (concurrently) in -/// instrumentation context to select the appropriate Aggregator for an -/// instrument. -pub trait Processor: AsDynProcessor { - /// AggregatorSelector is responsible for selecting the - /// concrete type of Aggregator used for a metric in the SDK. - /// - /// This may be a static decision based on fields of the - /// Descriptor, or it could use an external configuration - /// source to customize the treatment of each metric - /// instrument. - /// - /// The result from AggregatorSelector.AggregatorFor should be - /// the same type for a given Descriptor or else nil. The same - /// type should be returned for a given descriptor, because - /// Aggregators only know how to Merge with their own type. If - /// the result is nil, the metric instrument will be disabled. - /// - /// Note that the SDK only calls AggregatorFor when new records - /// require an Aggregator. This does not provide a way to - /// disable metrics with active records. - fn aggregator_selector(&self) -> &dyn AggregatorSelector; -} diff --git a/opentelemetry-sdk/src/export/metrics/stdout.rs b/opentelemetry-sdk/src/export/metrics/stdout.rs deleted file mode 100644 index 1263ea289e..0000000000 --- a/opentelemetry-sdk/src/export/metrics/stdout.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Stdout Metrics Exporter -use crate::{ - export::metrics::{ - aggregation::{stateless_temporality_selector, LastValue, Sum, TemporalitySelector}, - InstrumentationLibraryReader, MetricsExporter, - }, - metrics::aggregators::{LastValueAggregator, SumAggregator}, - Resource, -}; -use opentelemetry_api::{ - attributes::{default_encoder, AttributeSet, Encoder}, - metrics::{MetricsError, Result}, - Context, KeyValue, -}; -use std::fmt; -use std::io; -use std::sync::Mutex; -use std::time::SystemTime; - -/// Create a new stdout exporter builder with the configuration for a stdout exporter. -pub fn stdout() -> StdoutExporterBuilder { - StdoutExporterBuilder::::builder() -} - -/// An OpenTelemetry metric exporter that transmits telemetry to -/// the local STDOUT or via the registered implementation of `Write`. -#[derive(Debug)] -pub struct StdoutExporter { - /// Writer is the destination. If not set, `Stdout` is used. - writer: Mutex, - - /// Specifies if timestamps should be printed - timestamps: bool, - - /// Encodes the attributes. - attribute_encoder: Box, - - /// An optional user-defined function to format a given export batch. - formatter: Option, -} - -/// Individually exported metric -/// -/// Can be formatted using [`StdoutExporterBuilder::with_formatter`]. -#[derive(Default, Debug)] -pub struct ExportLine { - /// metric name - pub name: String, - - /// populated if using sum aggregator - pub sum: Option, - - /// populated if using last value aggregator - pub last_value: Option, - - /// metric timestamp - pub timestamp: Option, -} - -/// A number exported as debug for serialization -pub struct ExportNumeric(Box); - -impl fmt::Debug for ExportNumeric { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl StdoutExporter { - /// The temporality selector for this exporter - pub fn temporality_selector(&self) -> impl TemporalitySelector { - stateless_temporality_selector() - } -} - -impl TemporalitySelector for StdoutExporter { - fn temporality_for( - &self, - descriptor: &crate::metrics::sdk_api::Descriptor, - kind: &super::aggregation::AggregationKind, - ) -> super::aggregation::Temporality { - stateless_temporality_selector().temporality_for(descriptor, kind) - } -} - -impl MetricsExporter for StdoutExporter -where - W: fmt::Debug + io::Write, -{ - fn export( - &self, - _cx: &Context, - res: &Resource, - reader: &dyn InstrumentationLibraryReader, - ) -> Result<()> { - let mut batch = Vec::new(); - reader.try_for_each(&mut |library, reader| { - let mut attributes = Vec::new(); - if !library.name.is_empty() { - attributes.push(KeyValue::new("instrumentation.name", library.name.clone())); - } - if let Some(version) = &library.version { - attributes.push(KeyValue::new("instrumentation.version", version.clone())); - } - if let Some(schema) = &library.schema_url { - attributes.push(KeyValue::new("instrumentation.schema_url", schema.clone())); - } - let inst_attributes = AttributeSet::from_attributes(attributes.into_iter()); - let encoded_inst_attributes = - inst_attributes.encoded(Some(self.attribute_encoder.as_ref())); - - reader.try_for_each(self, &mut |record| { - let desc = record.descriptor(); - let agg = record.aggregator().ok_or(MetricsError::NoDataCollected)?; - let kind = desc.number_kind(); - - let encoded_resource = res.encoded(self.attribute_encoder.as_ref()); - - let mut expose = ExportLine::default(); - if let Some(sum) = agg.as_any().downcast_ref::() { - expose.sum = Some(ExportNumeric(sum.sum()?.to_debug(kind))); - } else if let Some(last_value) = agg.as_any().downcast_ref::() - { - let (value, timestamp) = last_value.last_value()?; - expose.last_value = Some(ExportNumeric(value.to_debug(kind))); - - if self.timestamps { - expose.timestamp = Some(timestamp); - } - } - - let mut encoded_attributes = String::new(); - let iter = record.attributes().iter(); - if let (0, _) = iter.size_hint() { - encoded_attributes = record - .attributes() - .encoded(Some(self.attribute_encoder.as_ref())); - } - - let mut sb = String::new(); - - sb.push_str(desc.name()); - - if !encoded_attributes.is_empty() - || !encoded_resource.is_empty() - || !encoded_inst_attributes.is_empty() - { - sb.push('{'); - sb.push_str(&encoded_resource); - if !encoded_inst_attributes.is_empty() && !encoded_resource.is_empty() { - sb.push(','); - } - sb.push_str(&encoded_inst_attributes); - if !encoded_attributes.is_empty() - && (!encoded_inst_attributes.is_empty() || !encoded_resource.is_empty()) - { - sb.push(','); - } - sb.push_str(&encoded_attributes); - sb.push('}'); - } - - expose.name = sb; - - batch.push(expose); - Ok(()) - }) - })?; - - self.writer.lock().map_err(From::from).and_then(|mut w| { - let formatted = match &self.formatter { - Some(formatter) => formatter.0(batch)?, - None => format!("{:?}\n", batch), - }; - w.write_all(formatted.as_bytes()) - .map_err(|e| MetricsError::Other(e.to_string())) - }) - } -} - -/// A formatter for user-defined batch serialization. -struct Formatter(Box) -> Result + Send + Sync>); -impl fmt::Debug for Formatter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Formatter(closure)") - } -} - -/// Configuration for a given stdout exporter. -#[derive(Debug)] -pub struct StdoutExporterBuilder { - writer: Mutex, - timestamps: bool, - attribute_encoder: Option>, - formatter: Option, -} - -impl StdoutExporterBuilder -where - W: io::Write + fmt::Debug + Send + Sync + 'static, -{ - fn builder() -> StdoutExporterBuilder { - StdoutExporterBuilder { - writer: Mutex::new(io::stdout()), - timestamps: true, - attribute_encoder: None, - formatter: None, - } - } - /// Set the writer that this exporter will use. - pub fn with_writer(self, writer: W2) -> StdoutExporterBuilder { - StdoutExporterBuilder { - writer: Mutex::new(writer), - timestamps: self.timestamps, - attribute_encoder: self.attribute_encoder, - formatter: self.formatter, - } - } - - /// Hide the timestamps from exported results - pub fn with_do_not_print_time(self, do_not_print_time: bool) -> Self { - StdoutExporterBuilder { - timestamps: do_not_print_time, - ..self - } - } - - /// Set the attribute encoder that this exporter will use. - pub fn with_attribute_encoder(self, attribute_encoder: E) -> Self - where - E: Encoder + Send + Sync + 'static, - { - StdoutExporterBuilder { - attribute_encoder: Some(Box::new(attribute_encoder)), - ..self - } - } - - /// Set a formatter for serializing export batch data - pub fn with_formatter(self, formatter: T) -> Self - where - T: Fn(Vec) -> Result + Send + Sync + 'static, - { - StdoutExporterBuilder { - formatter: Some(Formatter(Box::new(formatter))), - ..self - } - } - - /// Build a new push controller, returning errors if they arise. - pub fn build(self) -> Result> { - Ok(StdoutExporter { - writer: self.writer, - timestamps: self.timestamps, - attribute_encoder: self.attribute_encoder.unwrap_or_else(default_encoder), - formatter: self.formatter, - }) - } -} diff --git a/opentelemetry-sdk/src/export/mod.rs b/opentelemetry-sdk/src/export/mod.rs index 4f749ee7d3..d32d3cf018 100644 --- a/opentelemetry-sdk/src/export/mod.rs +++ b/opentelemetry-sdk/src/export/mod.rs @@ -1,8 +1,5 @@ -//! Metrics Export +//! Telemetry Export -#[cfg(feature = "metrics")] -#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))] -pub mod metrics; #[cfg(feature = "trace")] #[cfg_attr(docsrs, doc(cfg(feature = "trace")))] pub mod trace; diff --git a/opentelemetry-sdk/src/instrumentation.rs b/opentelemetry-sdk/src/instrumentation.rs index dd6385022b..ca0947e62f 100644 --- a/opentelemetry-sdk/src/instrumentation.rs +++ b/opentelemetry-sdk/src/instrumentation.rs @@ -1,6 +1,5 @@ -//! Provides instrumentation information for both tracing and metric. -//! See `OTEPS-0083` for details. -//! -//! [OTEPS-0083](https://github.com/open-telemetry/oteps/blob/master/text/0083-component.md) - pub use opentelemetry_api::InstrumentationLibrary; + +/// A logical unit of the application code with which the emitted telemetry can +/// be associated. +pub type Scope = InstrumentationLibrary; diff --git a/opentelemetry-sdk/src/lib.rs b/opentelemetry-sdk/src/lib.rs index 01e01cca53..ee1d0d1e20 100644 --- a/opentelemetry-sdk/src/lib.rs +++ b/opentelemetry-sdk/src/lib.rs @@ -1,11 +1,86 @@ -//! # OpenTelemetry SDK -//! -//! This SDK provides an opinionated reference implementation of -//! the OpenTelemetry API. The SDK implements the specifics of -//! deciding which data to collect through `Sampler`s, and -//! facilitates the delivery of telemetry data to storage systems -//! through `Exporter`s. These can be configured on `Tracer` and -//! `Meter` creation. +//! OpenTelemetry is a collection of tools, APIs, and SDKs. Use it to +//! instrument, generate, collect, and export telemetry data (metrics, logs, and +//! traces) to help you analyze your software's performance and behavior. +//! +//! # Getting Started +//! +//! ```no_run +//! # #[cfg(feature = "trace")] +//! # { +//! use opentelemetry_api::{global, trace::Tracer}; +//! use opentelemetry_sdk::export::trace::stdout; +//! +//! fn main() { +//! // Create a new trace pipeline that prints to stdout +//! let tracer = stdout::new_pipeline().install_simple(); +//! +//! tracer.in_span("doing_work", |cx| { +//! // Traced app logic here... +//! }); +//! +//! // Shutdown trace pipeline +//! global::shutdown_tracer_provider(); +//! } +//! # } +//! ``` +//! +//! See the [examples] directory for different integration patterns. +//! +//! See the API [`trace`] module docs for more information on creating and managing +//! spans. +//! +//! [examples]: https://github.com/open-telemetry/opentelemetry-rust/tree/main/examples +//! [`trace`]: https://docs.rs/opentelemetry_api/latest/opentelemetry_api/trace/index.html +//! +//! # Metrics (Beta) +//! +//! Note: the metrics implementation is **still in progress** and **subject to major +//! changes**. +//! +//! ### Creating instruments and recording measurements +//! +//! ``` +//! # #[cfg(feature = "metrics")] +//! # { +//! use opentelemetry_api::{global, Context, KeyValue}; +//! +//! let cx = Context::current(); +//! +//! // get a meter from a provider +//! let meter = global::meter("my_service"); +//! +//! // create an instrument +//! let counter = meter.u64_counter("my_counter").init(); +//! +//! // record a measurement +//! counter.add(&cx, 1, &[KeyValue::new("http.client_ip", "83.164.160.102")]); +//! # } +//! ``` +//! +//! See the [examples] directory for different integration patterns. +//! +//! See the API [`metrics`] module docs for more information on creating and +//! managing instruments. +//! +//! [examples]: https://github.com/open-telemetry/opentelemetry-rust/tree/main/examples +//! [`metrics`]: https://docs.rs/opentelemetry_api/latest/opentelemetry_api/metrics/index.html +//! +//! ## Crate Feature Flags +//! +//! The following core crate feature flags are available: +//! +//! * `trace`: Includes the trace SDK (enabled by default). +//! * `metrics`: Includes the unstable metrics SDK. +//! +//! Support for recording and exporting telemetry asynchronously can be added +//! via the following flags: +//! +//! * `rt-tokio`: Spawn telemetry tasks using [tokio]'s multi-thread runtime. +//! * `rt-tokio-current-thread`: Spawn telemetry tasks on a separate runtime so that the main runtime won't be blocked. +//! * `rt-async-std`: Spawn telemetry tasks using [async-std]'s runtime. +//! +//! [tokio]: https://crates.io/crates/tokio +//! [async-std]: https://crates.io/crates/async-std #![warn( future_incompatible, missing_debug_implementations, @@ -45,6 +120,6 @@ pub mod trace; #[doc(hidden)] pub mod util; -pub use instrumentation::InstrumentationLibrary; +pub use instrumentation::{InstrumentationLibrary, Scope}; #[doc(inline)] pub use resource::Resource; diff --git a/opentelemetry-sdk/src/metrics/aggregation.rs b/opentelemetry-sdk/src/metrics/aggregation.rs new file mode 100644 index 0000000000..fccb57f921 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/aggregation.rs @@ -0,0 +1,100 @@ +use std::fmt; + +use opentelemetry_api::metrics::{MetricsError, Result}; + +/// The way recorded measurements are summarized. +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub enum Aggregation { + /// An aggregation that drops all recorded data. + Drop, + + /// An aggregation that uses the default instrument kind selection mapping to + /// select another aggregation. + /// + /// A metric reader can be configured to make an aggregation selection based on + /// instrument kind that differs from the default. This aggregation ensures the + /// default is used. + /// + /// See the [DefaultAggregationSelector] for information about the default + /// instrument kind selection mapping. + /// + /// [DefaultAggregationSelector]: crate::metrics::reader::DefaultAggregationSelector + Default, + + /// An aggregation that summarizes a set of measurements as their arithmetic + /// sum. + Sum, + + /// An aggregation that summarizes a set of measurements as the last one made. + LastValue, + + /// An aggregation that summarizes a set of measurements as an histogram with + /// explicitly defined buckets. + ExplicitBucketHistogram { + /// The increasing bucket boundary values. + /// + /// Boundary values define bucket upper bounds. Buckets are exclusive of their + /// lower boundary and inclusive of their upper bound (except at positive + /// infinity). A measurement is defined to fall into the greatest-numbered + /// bucket with a boundary that is greater than or equal to the measurement. As + /// an example, boundaries defined as: + /// + /// vec![0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 1000.0]; + /// + /// Will define these buckets: + /// + /// (-∞, 0], (0, 5.0], (5.0, 10.0], (10.0, 25.0], (25.0, 50.0], (50.0, 75.0], + /// (75.0, 100.0], (100.0, 250.0], (250.0, 500.0], (500.0, 1000.0], (1000.0, +∞) + boundaries: Vec, + + /// Indicates whether to not record the min and max of the distribution. + /// + /// By default, these extrema are recorded. + /// + /// Recording these extrema for cumulative data is expected to have little + /// value, they will represent the entire life of the instrument instead of just + /// the current collection cycle. It is recommended to set this to true for that + /// type of data to avoid computing the low-value extrema. + no_min_max: bool, + }, +} + +impl fmt::Display for Aggregation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // used for stream id comparisons + let name = match self { + Aggregation::Drop => "Drop", + Aggregation::Default => "Default", + Aggregation::Sum => "Sum", + Aggregation::LastValue => "LastValue", + Aggregation::ExplicitBucketHistogram { .. } => "ExplicitBucketHistogram", + }; + + f.write_str(name) + } +} + +impl Aggregation { + /// Validate that this aggregation has correct configuration + pub fn validate(&self) -> Result<()> { + match self { + Aggregation::Drop => Ok(()), + Aggregation::Default => Ok(()), + Aggregation::Sum => Ok(()), + Aggregation::LastValue => Ok(()), + Aggregation::ExplicitBucketHistogram { boundaries, .. } => { + for x in boundaries.windows(2) { + if x[0] >= x[1] { + return Err(MetricsError::Config(format!( + "aggregation: explicit bucket histogram: non-monotonic boundaries: {:?}", + boundaries, + ))); + } + } + + Ok(()) + } + } + } +} diff --git a/opentelemetry-sdk/src/metrics/aggregators/ddsketch.rs b/opentelemetry-sdk/src/metrics/aggregators/ddsketch.rs deleted file mode 100644 index e45aac2843..0000000000 --- a/opentelemetry-sdk/src/metrics/aggregators/ddsketch.rs +++ /dev/null @@ -1,877 +0,0 @@ -//! DDSketch quantile sketch with relative-error guarantees. -//! DDSketch is a fast and fully-mergeable quantile sketch with relative-error guarantees. -//! -//! The main difference between this approach and previous art is DDSKetch employ a new method to -//! compute the error. Traditionally, the error rate of one sketch is evaluated by rank accuracy, -//! which can still generate a relative large variance if the dataset has long tail. -//! -//! DDSKetch, on the contrary, employs relative error rate that could work well on long tail dataset. -//! -//! The detail of this algorithm can be found in - -use std::{ - any::Any, - cmp::Ordering, - mem, - ops::AddAssign, - sync::{Arc, RwLock}, -}; - -use crate::export::metrics::{Aggregator, Count, Max, Min, MinMaxSumCount, Sum}; -use opentelemetry_api::metrics::{Descriptor, MetricsError, Number, NumberKind, Result}; - -const INITIAL_NUM_BINS: usize = 128; -const GROW_LEFT_BY: i64 = 128; - -const DEFAULT_MAX_NUM_BINS: i64 = 2048; -const DEFAULT_ALPHA: f64 = 0.01; -const DEFAULT_MIN_BOUNDARY: f64 = 1.0e-9; - -/// An aggregator to calculate quantile -pub fn ddsketch(config: &DdSketchConfig, kind: NumberKind) -> DdSketchAggregator { - DdSketchAggregator::new(config, kind) -} - -/// DDSKetch quantile sketch algorithm -/// -/// It can give q-quantiles with α-accurate for any 0<=q<=1. -/// -/// Here the accurate is calculated based on relative-error rate. Thus, the error guarantee adapts the scale of the output data. With relative error guarantee, the histogram can be more accurate in the area of low data density. For example, the long tail of response time data. -/// -/// For example, if the actual percentile is 1 second, and relative-error guarantee -/// is 2%, then the value should within the range of 0.98 to 1.02 -/// second. But if the actual percentile is 1 millisecond, with the same relative-error -/// guarantee, the value returned should within the range of 0.98 to 1.02 millisecond. -/// -/// In order to support both negative and positive inputs, DDSketchAggregator has two DDSketch store within itself to store the negative and positive inputs. -#[derive(Debug)] -pub struct DdSketchAggregator { - inner: RwLock, -} - -impl DdSketchAggregator { - /// Create a new DDSKetchAggregator that would yield a quantile with relative error rate less - /// than `alpha` - /// - /// The input should have a granularity larger than `key_epsilon` - pub fn new(config: &DdSketchConfig, kind: NumberKind) -> DdSketchAggregator { - DdSketchAggregator { - inner: RwLock::new(Inner::new(config, kind)), - } - } -} - -impl Default for DdSketchAggregator { - fn default() -> Self { - DdSketchAggregator::new( - &DdSketchConfig::new(DEFAULT_ALPHA, DEFAULT_MAX_NUM_BINS, DEFAULT_MIN_BOUNDARY), - NumberKind::F64, - ) - } -} - -impl Sum for DdSketchAggregator { - fn sum(&self) -> Result { - self.inner - .read() - .map_err(From::from) - .map(|inner| inner.sum.clone()) - } -} - -impl Min for DdSketchAggregator { - fn min(&self) -> Result { - self.inner - .read() - .map_err(From::from) - .map(|inner| inner.min_value.clone()) - } -} - -impl Max for DdSketchAggregator { - fn max(&self) -> Result { - self.inner - .read() - .map_err(From::from) - .map(|inner| inner.max_value.clone()) - } -} - -impl Count for DdSketchAggregator { - fn count(&self) -> Result { - self.inner - .read() - .map_err(From::from) - .map(|inner| inner.count()) - } -} - -impl MinMaxSumCount for DdSketchAggregator {} - -impl Aggregator for DdSketchAggregator { - fn update(&self, number: &Number, descriptor: &Descriptor) -> Result<()> { - self.inner - .write() - .map_err(From::from) - .map(|mut inner| inner.add(number, descriptor.number_kind())) - } - - fn synchronized_move( - &self, - destination: &Arc<(dyn Aggregator + Send + Sync)>, - descriptor: &Descriptor, - ) -> Result<()> { - if let Some(other) = destination.as_any().downcast_ref::() { - other - .inner - .write() - .map_err(From::from) - .and_then(|mut other| { - self.inner.write().map_err(From::from).map(|mut inner| { - let kind = descriptor.number_kind(); - other.max_value = mem::replace(&mut inner.max_value, kind.zero()); - other.min_value = mem::replace(&mut inner.min_value, kind.zero()); - other.key_epsilon = mem::take(&mut inner.key_epsilon); - other.offset = mem::take(&mut inner.offset); - other.gamma = mem::take(&mut inner.gamma); - other.gamma_ln = mem::take(&mut inner.gamma_ln); - other.positive_store = mem::take(&mut inner.positive_store); - other.negative_store = mem::take(&mut inner.negative_store); - other.sum = mem::replace(&mut inner.sum, kind.zero()); - }) - }) - } else { - Err(MetricsError::InconsistentAggregator(format!( - "Expected {:?}, got: {:?}", - self, destination - ))) - } - } - - fn merge( - &self, - other: &(dyn Aggregator + Send + Sync), - _descriptor: &Descriptor, - ) -> Result<()> { - if let Some(other) = other.as_any().downcast_ref::() { - self.inner.write() - .map_err(From::from) - .and_then(|mut inner| { - other.inner.read() - .map_err(From::from) - .and_then(|other| { - // assert that it can merge - if inner.positive_store.max_num_bins != other.positive_store.max_num_bins { - return Err(MetricsError::InconsistentAggregator(format!( - "When merging two DDSKetchAggregators, their max number of bins must be the same. Expect max number of bins to be {:?}, but get {:?}", inner.positive_store.max_num_bins, other.positive_store.max_num_bins - ))); - } - if inner.negative_store.max_num_bins != other.negative_store.max_num_bins { - return Err(MetricsError::InconsistentAggregator(format!( - "When merging two DDSKetchAggregators, their max number of bins must be the same. Expect max number of bins to be {:?}, but get {:?}", inner.negative_store.max_num_bins, other.negative_store.max_num_bins - ))); - } - - - if (inner.gamma - other.gamma).abs() > std::f64::EPSILON { - return Err(MetricsError::InconsistentAggregator(format!( - "When merging two DDSKetchAggregators, their gamma must be the same. Expect max number of bins to be {:?}, but get {:?}", inner.gamma, other.gamma - ))); - } - - if other.count() == 0 { - return Ok(()); - } - - if inner.count() == 0 { - inner.positive_store.merge(&other.positive_store); - inner.negative_store.merge(&other.negative_store); - inner.sum = other.sum.clone(); - inner.min_value = other.min_value.clone(); - inner.max_value = other.max_value.clone(); - return Ok(()); - } - - inner.positive_store.merge(&other.positive_store); - inner.negative_store.merge(&other.negative_store); - - inner.sum = match inner.kind { - NumberKind::F64 => - Number::from(inner.sum.to_f64(&inner.kind) + other.sum.to_f64(&other.kind)), - NumberKind::U64 => Number::from(inner.sum.to_u64(&inner.kind) + other.sum.to_u64(&other.kind)), - NumberKind::I64 => Number::from(inner.sum.to_i64(&inner.kind) + other.sum.to_i64(&other.kind)) - }; - - if inner.min_value.partial_cmp(&inner.kind, &other.min_value) == Some(Ordering::Greater) { - inner.min_value = other.min_value.clone(); - }; - - if inner.max_value.partial_cmp(&inner.kind, &other.max_value) == Some(Ordering::Less) { - inner.max_value = other.max_value.clone(); - } - - Ok(()) - }) - }) - } else { - Err(MetricsError::InconsistentAggregator(format!( - "Expected {:?}, got: {:?}", - self, other - ))) - } - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -/// DDSKetch Configuration. -#[derive(Debug)] -pub struct DdSketchConfig { - alpha: f64, - max_num_bins: i64, - key_epsilon: f64, -} - -impl DdSketchConfig { - /// Create a new DDSKetch config - pub fn new(alpha: f64, max_num_bins: i64, key_epsilon: f64) -> Self { - DdSketchConfig { - alpha, - max_num_bins, - key_epsilon, - } - } -} - -/// DDSKetch implementation. -/// -/// Note that Inner is not thread-safe. All operation should be protected by a lock or other -/// synchronization. -/// -/// Inner will also convert all Number into actual primitive type and back. -/// -/// According to the paper, the DDSKetch only support positive number. Inner support -/// either positive or negative number. But cannot yield actual result when input has -/// both positive and negative number. -#[derive(Debug)] -struct Inner { - positive_store: Store, - negative_store: Store, - kind: NumberKind, - // sum of all value within store - sum: Number, - // γ = (1 + α)/(1 - α) - gamma: f64, - // ln(γ) - gamma_ln: f64, - // The epsilon when map value to bin key. Any value between [-key_epsilon, key_epsilon] will - // be mapped to bin key 0. Must be a positive number. - key_epsilon: f64, - // offset is here to ensure that keys for positive numbers that are larger than min_value are - // greater than or equal to 1 while the keys for negative numbers are less than or equal to -1. - offset: i64, - - // minimum number that in store. - min_value: Number, - // maximum number that in store. - max_value: Number, -} - -impl Inner { - fn new(config: &DdSketchConfig, kind: NumberKind) -> Inner { - let gamma: f64 = 1.0 + 2.0 * config.alpha / (1.0 - config.alpha); - let mut inner = Inner { - positive_store: Store::new(config.max_num_bins / 2), - negative_store: Store::new(config.max_num_bins / 2), - min_value: kind.max(), - max_value: kind.min(), - sum: kind.zero(), - gamma, - gamma_ln: gamma.ln(), - key_epsilon: config.key_epsilon, - offset: 0, - kind, - }; - // reset offset based on key_epsilon - inner.offset = -(inner.log_gamma(inner.key_epsilon)).ceil() as i64 + 1i64; - inner - } - - fn add(&mut self, v: &Number, kind: &NumberKind) { - let key = self.key(v, kind); - match v.partial_cmp(kind, &Number::from(0.0)) { - Some(Ordering::Greater) | Some(Ordering::Equal) => { - self.positive_store.add(key); - } - Some(Ordering::Less) => { - self.negative_store.add(key); - } - _ => { - // if return none. Do nothing and return - return; - } - } - - // update min and max - if self.min_value.partial_cmp(&self.kind, v) == Some(Ordering::Greater) { - self.min_value = v.clone(); - } - - if self.max_value.partial_cmp(&self.kind, v) == Some(Ordering::Less) { - self.max_value = v.clone(); - } - - match &self.kind { - NumberKind::I64 => { - self.sum = Number::from(self.sum.to_i64(&self.kind) + v.to_i64(kind)); - } - NumberKind::U64 => { - self.sum = Number::from(self.sum.to_u64(&self.kind) + v.to_u64(kind)); - } - NumberKind::F64 => { - self.sum = Number::from(self.sum.to_f64(&self.kind) + v.to_f64(kind)); - } - } - } - - fn key(&self, num: &Number, kind: &NumberKind) -> i64 { - if num.to_f64(kind) < -self.key_epsilon { - let positive_num = match kind { - NumberKind::F64 => Number::from(-num.to_f64(kind)), - NumberKind::U64 => Number::from(num.to_u64(kind)), - NumberKind::I64 => Number::from(-num.to_i64(kind)), - }; - (-self.log_gamma(positive_num.to_f64(kind)).ceil()) as i64 - self.offset - } else if num.to_f64(kind) > self.key_epsilon { - self.log_gamma(num.to_f64(kind)).ceil() as i64 + self.offset - } else { - 0i64 - } - } - - /// get the index of the bucket based on num - fn log_gamma(&self, num: f64) -> f64 { - num.ln() / self.gamma_ln - } - - fn count(&self) -> u64 { - self.negative_store.count + self.positive_store.count - } -} - -#[derive(Debug)] -struct Store { - bins: Vec, - count: u64, - min_key: i64, - max_key: i64, - // maximum number of bins Store can have. - // In the worst case, the bucket can grow as large as the number of the elements inserted into. - // max_num_bins helps control the number of bins. - max_num_bins: i64, -} - -impl Default for Store { - fn default() -> Self { - Store { - bins: vec![0; INITIAL_NUM_BINS], - count: 0, - min_key: 0, - max_key: 0, - max_num_bins: DEFAULT_MAX_NUM_BINS, - } - } -} - -/// DDSKetchInner stores the data -impl Store { - fn new(max_num_bins: i64) -> Store { - Store { - bins: vec![ - 0; - if max_num_bins as usize > INITIAL_NUM_BINS { - INITIAL_NUM_BINS - } else { - max_num_bins as usize - } - ], - count: 0u64, - min_key: 0i64, - max_key: 0i64, - max_num_bins, - } - } - - /// Add count based on key. - /// - /// If key is not in [min_key, max_key], we will expand to left or right - /// - /// - /// The bins are essentially working in a round-robin fashion where we can use all space in bins - /// to represent any continuous space within length. That's why we need to offset the key - /// with `min_key` so that we get the actual bin index. - fn add(&mut self, key: i64) { - if self.count == 0 { - self.max_key = key; - self.min_key = key - self.bins.len() as i64 + 1 - } - - if key < self.min_key { - self.grow_left(key) - } else if key > self.max_key { - self.grow_right(key) - } - let idx = if key - self.min_key < 0 { - 0 - } else { - key - self.min_key - }; - // we unwrap here because grow_left or grow_right will make sure the idx is less than vector size - let bin_count = self.bins.get_mut(idx as usize).unwrap(); - *bin_count += 1; - self.count += 1; - } - - fn grow_left(&mut self, key: i64) { - if self.min_key < key || self.bins.len() >= self.max_num_bins as usize { - return; - } - - let min_key = if self.max_key - key >= self.max_num_bins { - self.max_key - self.max_num_bins + 1 - } else { - let mut min_key = self.min_key; - while min_key > key { - min_key -= GROW_LEFT_BY; - } - min_key - }; - - // The new vector will contain three parts. - // First part is all 0, which is the part expended - // Second part is from existing bins. - // Third part is what's left. - let expected_len = (self.max_key - min_key + 1) as usize; - let mut new_bins = vec![0u64; expected_len]; - let old_bin_slice = &mut new_bins[(self.min_key - min_key) as usize..]; - old_bin_slice.copy_from_slice(&self.bins); - - self.bins = new_bins; - self.min_key = min_key; - } - - fn grow_right(&mut self, key: i64) { - if self.max_key > key { - return; - } - - if key - self.max_key >= self.max_num_bins { - // if currently key minus currently max key is larger than maximum number of bins. - // Move all elements in current bins into the first bin - self.bins = vec![0; self.max_num_bins as usize]; - self.max_key = key; - self.min_key = key - self.max_num_bins + 1; - self.bins.get_mut(0).unwrap().add_assign(self.count); - } else if key - self.min_key >= self.max_num_bins { - let min_key = key - self.max_num_bins + 1; - let upper_bound = if min_key < self.max_key + 1 { - min_key - } else { - self.max_key + 1 - } - self.min_key; - let n = self.bins.iter().take(upper_bound as usize).sum::(); - - if self.bins.len() < self.max_num_bins as usize { - let mut new_bins = vec![0; self.max_num_bins as usize]; - new_bins[0..self.bins.len() - (min_key - self.min_key) as usize] - .as_mut() - .copy_from_slice(&self.bins[(min_key - self.min_key) as usize..]); - self.bins = new_bins; - } else { - // bins length is equal to max number of bins - self.bins.drain(0..(min_key - self.min_key) as usize); - if self.max_num_bins > self.max_key - min_key + 1 { - self.bins.resize( - self.bins.len() - + (self.max_num_bins - (self.max_key - min_key + 1)) as usize, - 0, - ) - } - } - self.max_key = key; - self.min_key = min_key; - self.bins.get_mut(0).unwrap().add_assign(n); - } else { - let mut new_bin = vec![0; (key - self.min_key + 1) as usize]; - new_bin[0..self.bins.len()] - .as_mut() - .copy_from_slice(&self.bins); - self.bins = new_bin; - self.max_key = key; - } - } - - /// Merge two stores - fn merge(&mut self, other: &Store) { - if self.count == 0 { - return; - } - if other.count == 0 { - self.bins = other.bins.clone(); - self.min_key = other.min_key; - self.max_key = other.max_key; - self.count = other.count; - } - - if self.max_key > other.max_key { - if other.min_key < self.min_key { - self.grow_left(other.min_key); - } - let start = if other.min_key > self.min_key { - other.min_key - } else { - self.min_key - } as usize; - for i in start..other.max_key as usize { - self.bins[i - self.min_key as usize] = other.bins[i - other.min_key as usize]; - } - let mut n = 0; - for i in other.min_key as usize..self.min_key as usize { - n += other.bins[i - other.min_key as usize] - } - self.bins[0] += n; - } else if other.min_key < self.min_key { - let mut tmp_bins = vec![0u64; other.bins.len()]; - tmp_bins.as_mut_slice().copy_from_slice(&other.bins); - - for i in self.min_key as usize..self.max_key as usize { - tmp_bins[i - other.min_key as usize] += self.bins[i - self.min_key as usize]; - } - - self.bins = tmp_bins; - self.max_key = other.max_key; - self.min_key = other.min_key; - } else { - self.grow_right(other.max_key); - for i in other.min_key as usize..(other.max_key + 1) as usize { - self.bins[i - self.min_key as usize] += other.bins[i - other.min_key as usize]; - } - } - - self.count += other.count; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::export::metrics::{Aggregator, Count, Max, Min, Sum}; - use opentelemetry_api::metrics::{Descriptor, InstrumentKind, Number, NumberKind}; - use rand_distr::{Distribution, Exp, LogNormal, Normal}; - use std::cmp::Ordering; - use std::sync::Arc; - - const TEST_MAX_BINS: i64 = 1024; - const TEST_ALPHA: f64 = 0.01; - const TEST_KEY_EPSILON: f64 = 1.0e-9; - - // Test utils - - struct Dataset { - data: Vec, - kind: NumberKind, - } - - impl Dataset { - fn from_f64_vec(data: Vec) -> Dataset { - Dataset { - data: data.into_iter().map(Number::from).collect::>(), - kind: NumberKind::F64, - } - } - - fn from_u64_vec(data: Vec) -> Dataset { - Dataset { - data: data.into_iter().map(Number::from).collect::>(), - kind: NumberKind::U64, - } - } - - fn from_i64_vec(data: Vec) -> Dataset { - Dataset { - data: data.into_iter().map(Number::from).collect::>(), - kind: NumberKind::I64, - } - } - - fn sum(&self) -> Number { - match self.kind { - NumberKind::F64 => { - Number::from(self.data.iter().map(|e| e.to_f64(&self.kind)).sum::()) - } - NumberKind::U64 => { - Number::from(self.data.iter().map(|e| e.to_u64(&self.kind)).sum::()) - } - NumberKind::I64 => { - Number::from(self.data.iter().map(|e| e.to_i64(&self.kind)).sum::()) - } - } - } - } - - fn generate_linear_dataset_f64(start: f64, step: f64, num: usize) -> Vec { - let mut vec = Vec::with_capacity(num); - for i in 0..num { - vec.push((start + i as f64 * step) as f64); - } - vec - } - - fn generate_linear_dataset_u64(start: u64, step: u64, num: usize) -> Vec { - let mut vec = Vec::with_capacity(num); - for i in 0..num { - vec.push(start + i as u64 * step); - } - vec - } - - fn generate_linear_dataset_i64(start: i64, step: i64, num: usize) -> Vec { - let mut vec = Vec::with_capacity(num); - for i in 0..num { - vec.push(start + i as i64 * step); - } - vec - } - - /// generate a dataset with normal distribution. Return sorted dataset. - fn generate_normal_dataset(mean: f64, stddev: f64, num: usize) -> Vec { - let normal = Normal::new(mean, stddev).unwrap(); - let mut data = Vec::with_capacity(num); - for _ in 0..num { - data.push(normal.sample(&mut rand::thread_rng())); - } - data.as_mut_slice() - .sort_by(|a, b| a.partial_cmp(b).unwrap()); - data - } - - /// generate a dataset with log normal distribution. Return sorted dataset. - fn generate_log_normal_dataset(mean: f64, stddev: f64, num: usize) -> Vec { - let normal = LogNormal::new(mean, stddev).unwrap(); - let mut data = Vec::with_capacity(num); - for _ in 0..num { - data.push(normal.sample(&mut rand::thread_rng())); - } - data.as_mut_slice() - .sort_by(|a, b| a.partial_cmp(b).unwrap()); - data - } - - fn generate_exponential_dataset(rate: f64, num: usize) -> Vec { - let exponential = Exp::new(rate).unwrap(); - let mut data = Vec::with_capacity(num); - for _ in 0..num { - data.push(exponential.sample(&mut rand::thread_rng())); - } - data.as_mut_slice() - .sort_by(|a, b| a.partial_cmp(b).unwrap()); - data - } - - /// Insert all element of data into ddsketch and assert the quantile result is within the error range. - /// Note that data must be sorted. - fn evaluate_sketch(dataset: Dataset) { - let kind = &dataset.kind; - let ddsketch = DdSketchAggregator::new( - &DdSketchConfig::new(TEST_ALPHA, TEST_MAX_BINS, TEST_KEY_EPSILON), - kind.clone(), - ); - let descriptor = Descriptor::new( - "test".to_string(), - "test", - None, - None, - InstrumentKind::Histogram, - kind.clone(), - ); - - for i in &dataset.data { - let _ = ddsketch.update(i, &descriptor); - } - - assert_eq!( - ddsketch - .min() - .unwrap() - .partial_cmp(kind, dataset.data.get(0).unwrap()), - Some(Ordering::Equal) - ); - assert_eq!( - ddsketch - .max() - .unwrap() - .partial_cmp(kind, dataset.data.last().unwrap()), - Some(Ordering::Equal) - ); - assert_eq!( - ddsketch.sum().unwrap().partial_cmp(kind, &dataset.sum()), - Some(Ordering::Equal) - ); - assert_eq!(ddsketch.count().unwrap(), dataset.data.len() as u64); - } - - // Test basic operation of Store - - /// First set max_num_bins < number of keys, test to see if the store will collapse to left - /// most bin instead of expending beyond the max_num_bins - #[test] - fn test_insert_into_store() { - let mut store = Store::new(200); - for i in -100..1300 { - store.add(i) - } - assert_eq!(store.count, 1400); - assert_eq!(store.bins.len(), 200); - } - - /// Test to see if copy_from_slice will panic because the range size is different in left and right - #[test] - fn test_grow_right() { - let mut store = Store::new(150); - for i in &[-100, -50, 150, -20, 10] { - store.add(*i) - } - assert_eq!(store.count, 5); - } - - /// Test to see if copy_from_slice will panic because the range size is different in left and right - #[test] - fn test_grow_left() { - let mut store = Store::new(150); - for i in &[500, 150, 10] { - store.add(*i) - } - assert_eq!(store.count, 3); - } - - /// Before merge, store1 should hold 300 bins that looks like [201,1,1,1,...], - /// store 2 should hold 200 bins looks like [301,1,1,...] - /// After merge, store 1 should still hold 300 bins with following distribution - /// - /// index [0,0] -> 201 - /// - /// index [1,99] -> 1 - /// - /// index [100, 100] -> 302 - /// - /// index [101, 299] -> 2 - #[test] - fn test_merge_stores() { - let mut store1 = Store::new(300); - let mut store2 = Store::new(200); - for i in 500..1000 { - store1.add(i); - store2.add(i); - } - store1.merge(&store2); - assert_eq!(store1.bins.get(0), Some(&201)); - assert_eq!(&store1.bins[1..100], vec![1u64; 99].as_slice()); - assert_eq!(store1.bins[100], 302); - assert_eq!(&store1.bins[101..], vec![2u64; 199].as_slice()); - assert_eq!(store1.count, 1000); - } - - // Test ddsketch with different distribution - - #[test] - fn test_linear_distribution() { - // test u64 - let mut dataset = Dataset::from_u64_vec(generate_linear_dataset_u64(12, 3, 5000)); - evaluate_sketch(dataset); - - // test i64 - dataset = Dataset::from_i64_vec(generate_linear_dataset_i64(-12, 3, 5000)); - evaluate_sketch(dataset); - - // test f64 - dataset = Dataset::from_f64_vec(generate_linear_dataset_f64(-12.0, 3.0, 5000)); - evaluate_sketch(dataset); - } - - #[test] - fn test_normal_distribution() { - let mut dataset = Dataset::from_f64_vec(generate_normal_dataset(150.0, 1.2, 100)); - evaluate_sketch(dataset); - - dataset = Dataset::from_f64_vec(generate_normal_dataset(-30.0, 4.4, 100)); - evaluate_sketch(dataset); - } - - #[test] - fn test_log_normal_distribution() { - let dataset = Dataset::from_f64_vec(generate_log_normal_dataset(120.0, 0.5, 100)); - evaluate_sketch(dataset); - } - - #[test] - fn test_exponential_distribution() { - let dataset = Dataset::from_f64_vec(generate_exponential_dataset(2.0, 500)); - evaluate_sketch(dataset); - } - - // Test Aggregator operation of DDSketch - #[test] - fn test_synchronized_move() { - let dataset = Dataset::from_f64_vec(generate_normal_dataset(1.0, 3.5, 100)); - let kind = &dataset.kind; - let ddsketch = DdSketchAggregator::new( - &DdSketchConfig::new(TEST_ALPHA, TEST_MAX_BINS, TEST_KEY_EPSILON), - kind.clone(), - ); - let descriptor = Descriptor::new( - "test".to_string(), - "test", - None, - None, - InstrumentKind::Histogram, - kind.clone(), - ); - for i in &dataset.data { - let _ = ddsketch.update(i, &descriptor); - } - let expected_sum = ddsketch.sum().unwrap().to_f64(&NumberKind::F64); - let expected_count = ddsketch.count().unwrap(); - let expected_min = ddsketch.min().unwrap().to_f64(&NumberKind::F64); - let expected_max = ddsketch.max().unwrap().to_f64(&NumberKind::F64); - - let moved_ddsketch: Arc<(dyn Aggregator + Send + Sync)> = - Arc::new(DdSketchAggregator::new( - &DdSketchConfig::new(TEST_ALPHA, TEST_MAX_BINS, TEST_KEY_EPSILON), - NumberKind::F64, - )); - ddsketch - .synchronized_move(&moved_ddsketch, &descriptor) - .expect("Fail to sync move"); - let moved_ddsketch = moved_ddsketch - .as_any() - .downcast_ref::() - .expect("Fail to cast dyn Aggregator down to DDSketchAggregator"); - - // assert sum, max, min and count - assert!( - (moved_ddsketch.max().unwrap().to_f64(&NumberKind::F64) - expected_max).abs() - < std::f64::EPSILON - ); - assert!( - (moved_ddsketch.min().unwrap().to_f64(&NumberKind::F64) - expected_min).abs() - < std::f64::EPSILON - ); - assert!( - (moved_ddsketch.sum().unwrap().to_f64(&NumberKind::F64) - expected_sum).abs() - < std::f64::EPSILON - ); - assert_eq!(moved_ddsketch.count().unwrap(), expected_count); - } -} diff --git a/opentelemetry-sdk/src/metrics/aggregators/last_value.rs b/opentelemetry-sdk/src/metrics/aggregators/last_value.rs deleted file mode 100644 index 81de9d086a..0000000000 --- a/opentelemetry-sdk/src/metrics/aggregators/last_value.rs +++ /dev/null @@ -1,118 +0,0 @@ -use crate::export::metrics::aggregation::{Aggregation, AggregationKind, LastValue}; -use crate::metrics::{ - aggregators::Aggregator, - sdk_api::{Descriptor, Number}, -}; -use opentelemetry_api::metrics::{MetricsError, Result}; -use opentelemetry_api::Context; -use std::any::Any; -use std::sync::{Arc, Mutex}; -use std::time::SystemTime; - -/// Create a new `LastValueAggregator` -pub fn last_value() -> LastValueAggregator { - LastValueAggregator { - inner: Mutex::new(Inner::default()), - } -} - -/// Aggregates last value events. -#[derive(Debug)] -pub struct LastValueAggregator { - inner: Mutex, -} - -impl Aggregation for LastValueAggregator { - fn kind(&self) -> &AggregationKind { - &AggregationKind::LAST_VALUE - } -} - -impl Aggregator for LastValueAggregator { - fn aggregation(&self) -> &dyn Aggregation { - self - } - - fn update(&self, _cx: &Context, number: &Number, _descriptor: &Descriptor) -> Result<()> { - self.inner.lock().map_err(Into::into).map(|mut inner| { - inner.state = Some(LastValueData { - value: number.clone(), - timestamp: opentelemetry_api::time::now(), - }); - }) - } - - fn synchronized_move( - &self, - other: &Arc, - _descriptor: &Descriptor, - ) -> Result<()> { - if let Some(other) = other.as_any().downcast_ref::() { - self.inner.lock().map_err(From::from).and_then(|mut inner| { - other.inner.lock().map_err(From::from).map(|mut other| { - other.state = inner.state.take(); - }) - }) - } else { - Err(MetricsError::InconsistentAggregator(format!( - "Expected {:?}, got: {:?}", - self, other - ))) - } - } - fn merge( - &self, - other: &(dyn Aggregator + Send + Sync), - _descriptor: &Descriptor, - ) -> Result<()> { - if let Some(other) = other.as_any().downcast_ref::() { - self.inner.lock().map_err(From::from).and_then(|mut inner| { - other.inner.lock().map_err(From::from).map(|mut other| { - match (&inner.state, &other.state) { - // Take if other timestamp is greater - (Some(checkpoint), Some(other_checkpoint)) - if other_checkpoint.timestamp > checkpoint.timestamp => - { - inner.state = other.state.take() - } - // Take if no value exists currently - (None, Some(_)) => inner.state = other.state.take(), - // Otherwise done - _ => (), - } - }) - }) - } else { - Err(MetricsError::InconsistentAggregator(format!( - "Expected {:?}, got: {:?}", - self, other - ))) - } - } - fn as_any(&self) -> &dyn Any { - self - } -} - -impl LastValue for LastValueAggregator { - fn last_value(&self) -> Result<(Number, SystemTime)> { - self.inner.lock().map_err(Into::into).and_then(|inner| { - if let Some(checkpoint) = &inner.state { - Ok((checkpoint.value.clone(), checkpoint.timestamp)) - } else { - Err(MetricsError::NoDataCollected) - } - }) - } -} - -#[derive(Debug, Default)] -struct Inner { - state: Option, -} - -#[derive(Debug)] -struct LastValueData { - value: Number, - timestamp: SystemTime, -} diff --git a/opentelemetry-sdk/src/metrics/aggregators/mod.rs b/opentelemetry-sdk/src/metrics/aggregators/mod.rs deleted file mode 100644 index 6ddddc5f47..0000000000 --- a/opentelemetry-sdk/src/metrics/aggregators/mod.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! Metric Aggregators -use core::fmt; -use std::{any::Any, sync::Arc}; - -use crate::{ - export::metrics::aggregation::Aggregation, - metrics::sdk_api::{Descriptor, InstrumentKind, Number, NumberKind}, -}; -use opentelemetry_api::{ - metrics::{MetricsError, Result}, - Context, -}; - -mod histogram; -mod last_value; -mod sum; - -pub use histogram::{histogram, HistogramAggregator}; -pub use last_value::{last_value, LastValueAggregator}; -pub use sum::{sum, SumAggregator}; - -/// RangeTest is a common routine for testing for valid input values. This -/// rejects NaN values. This rejects negative values when the metric instrument -/// does not support negative values, including monotonic counter metrics and -/// absolute Histogram metrics. -pub fn range_test(number: &Number, descriptor: &Descriptor) -> Result<()> { - if descriptor.number_kind() == &NumberKind::F64 && number.is_nan() { - return Err(MetricsError::NaNInput); - } - - match descriptor.instrument_kind() { - InstrumentKind::Counter | InstrumentKind::CounterObserver - if descriptor.number_kind() == &NumberKind::F64 => - { - if number.is_negative(descriptor.number_kind()) { - return Err(MetricsError::NegativeInput); - } - } - _ => (), - }; - Ok(()) -} - -/// Aggregator implements a specific aggregation behavior, i.e., a behavior to -/// track a sequence of updates to an instrument. Sum-only instruments commonly -/// use a simple Sum aggregator, but for the distribution instruments -/// (Histogram, ValueObserver) there are a number of possible aggregators -/// with different cost and accuracy tradeoffs. -/// -/// Note that any Aggregator may be attached to any instrument--this is the -/// result of the OpenTelemetry API/SDK separation. It is possible to attach a -/// Sum aggregator to a Histogram instrument or a MinMaxSumCount aggregator -/// to a Counter instrument. -pub trait Aggregator: fmt::Debug { - /// The interface to access the current state of this Aggregator. - fn aggregation(&self) -> &dyn Aggregation; - - /// Update receives a new measured value and incorporates it into the - /// aggregation. Update calls may be called concurrently. - /// - /// `Descriptor::number_kind` should be consulted to determine whether the - /// provided number is an `i64`, `u64` or `f64`. - /// - /// The current Context could be inspected for a `Baggage` or - /// `SpanContext`. - fn update(&self, context: &Context, number: &Number, descriptor: &Descriptor) -> Result<()>; - - /// This method is called during collection to finish one period of aggregation - /// by atomically saving the currently-updating state into the argument - /// Aggregator. - /// - /// `synchronized_move` is called concurrently with `update`. These two methods - /// must be synchronized with respect to each other, for correctness. - /// - /// This method will return an `InconsistentAggregator` error if this - /// `Aggregator` cannot be copied into the destination due to an incompatible - /// type. - /// - /// This call has no `Context` argument because it is expected to perform only - /// computation. - fn synchronized_move( - &self, - destination: &Arc, - descriptor: &Descriptor, - ) -> Result<()>; - - /// This combines the checkpointed state from the argument `Aggregator` into this - /// `Aggregator`. `merge` is not synchronized with respect to `update` or - /// `synchronized_move`. - /// - /// The owner of an `Aggregator` being merged is responsible for synchronization - /// of both `Aggregator` states. - fn merge(&self, other: &(dyn Aggregator + Send + Sync), descriptor: &Descriptor) -> Result<()>; - - /// Returns the implementing aggregator as `Any` for downcasting. - fn as_any(&self) -> &dyn Any; -} diff --git a/opentelemetry-sdk/src/metrics/aggregators/sum.rs b/opentelemetry-sdk/src/metrics/aggregators/sum.rs deleted file mode 100644 index 60c7147a83..0000000000 --- a/opentelemetry-sdk/src/metrics/aggregators/sum.rs +++ /dev/null @@ -1,74 +0,0 @@ -use crate::export::metrics::aggregation::{Aggregation, AggregationKind, Sum}; -use crate::metrics::{ - aggregators::Aggregator, - sdk_api::{AtomicNumber, Descriptor, Number}, -}; -use opentelemetry_api::metrics::{MetricsError, Result}; -use opentelemetry_api::Context; -use std::any::Any; -use std::sync::Arc; - -/// Create a new sum aggregator. -pub fn sum() -> impl Aggregator { - SumAggregator::default() -} - -/// An aggregator for counter events. -#[derive(Debug, Default)] -pub struct SumAggregator { - value: AtomicNumber, -} - -impl Sum for SumAggregator { - fn sum(&self) -> Result { - Ok(self.value.load()) - } -} - -impl Aggregation for SumAggregator { - fn kind(&self) -> &AggregationKind { - &AggregationKind::SUM - } -} - -impl Aggregator for SumAggregator { - fn aggregation(&self) -> &dyn Aggregation { - self - } - - fn update(&self, _cx: &Context, number: &Number, descriptor: &Descriptor) -> Result<()> { - self.value.fetch_add(descriptor.number_kind(), number); - Ok(()) - } - - fn synchronized_move( - &self, - other: &Arc, - descriptor: &Descriptor, - ) -> Result<()> { - if let Some(other) = other.as_any().downcast_ref::() { - let kind = descriptor.number_kind(); - other.value.store(&self.value.load()); - self.value.store(&kind.zero()); - Ok(()) - } else { - Err(MetricsError::InconsistentAggregator(format!( - "Expected {:?}, got: {:?}", - self, other - ))) - } - } - - fn merge(&self, other: &(dyn Aggregator + Send + Sync), descriptor: &Descriptor) -> Result<()> { - if let Some(other_sum) = other.as_any().downcast_ref::() { - self.value - .fetch_add(descriptor.number_kind(), &other_sum.value.load()) - } - - Ok(()) - } - - fn as_any(&self) -> &dyn Any { - self - } -} diff --git a/opentelemetry-sdk/src/metrics/attributes/mod.rs b/opentelemetry-sdk/src/metrics/attributes/mod.rs new file mode 100644 index 0000000000..624f12d33d --- /dev/null +++ b/opentelemetry-sdk/src/metrics/attributes/mod.rs @@ -0,0 +1,3 @@ +mod set; + +pub(crate) use set::AttributeSet; diff --git a/opentelemetry-sdk/src/metrics/attributes/set.rs b/opentelemetry-sdk/src/metrics/attributes/set.rs new file mode 100644 index 0000000000..db50ec603e --- /dev/null +++ b/opentelemetry-sdk/src/metrics/attributes/set.rs @@ -0,0 +1,143 @@ +use std::{ + cmp::Ordering, + collections::{BTreeSet, HashSet}, + hash::{Hash, Hasher}, +}; + +use opentelemetry_api::{Array, Key, KeyValue, Value}; +use ordered_float::OrderedFloat; + +#[derive(Clone, Debug)] +struct HashKeyValue(KeyValue); + +impl Hash for HashKeyValue { + fn hash(&self, state: &mut H) { + self.0.key.hash(state); + match &self.0.value { + Value::F64(f) => OrderedFloat(*f).hash(state), + Value::Array(a) => match a { + Array::Bool(b) => b.hash(state), + Array::I64(i) => i.hash(state), + Array::F64(f) => f.iter().for_each(|f| OrderedFloat(*f).hash(state)), + Array::String(s) => s.hash(state), + }, + Value::Bool(b) => b.hash(state), + Value::I64(i) => i.hash(state), + Value::String(s) => s.hash(state), + }; + } +} + +impl PartialOrd for HashKeyValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HashKeyValue { + fn cmp(&self, other: &Self) -> Ordering { + match self.0.key.cmp(&other.0.key) { + Ordering::Equal => match type_order(&self.0.value).cmp(&type_order(&other.0.value)) { + Ordering::Equal => match (&self.0.value, &other.0.value) { + (Value::F64(f), Value::F64(of)) => OrderedFloat(*f).cmp(&OrderedFloat(*of)), + (Value::Array(Array::Bool(b)), Value::Array(Array::Bool(ob))) => b.cmp(ob), + (Value::Array(Array::I64(i)), Value::Array(Array::I64(oi))) => i.cmp(oi), + (Value::Array(Array::String(s)), Value::Array(Array::String(os))) => s.cmp(os), + (Value::Array(Array::F64(f)), Value::Array(Array::F64(of))) => { + match f.len().cmp(&of.len()) { + Ordering::Equal => f + .iter() + .map(|x| OrderedFloat(*x)) + .collect::>() + .cmp(&of.iter().map(|x| OrderedFloat(*x)).collect()), + other => other, + } + } + (Value::Bool(b), Value::Bool(ob)) => b.cmp(ob), + (Value::I64(i), Value::I64(oi)) => i.cmp(oi), + (Value::String(s), Value::String(os)) => s.cmp(os), + _ => Ordering::Equal, + }, + other => other, // 2nd order by value types + }, + other => other, // 1st order by key + } + } +} + +fn type_order(v: &Value) -> u8 { + match v { + Value::Bool(_) => 1, + Value::I64(_) => 2, + Value::F64(_) => 3, + Value::String(_) => 4, + Value::Array(a) => match a { + Array::Bool(_) => 5, + Array::I64(_) => 6, + Array::F64(_) => 7, + Array::String(_) => 8, + }, + } +} + +impl PartialEq for HashKeyValue { + fn eq(&self, other: &Self) -> bool { + self.0.key == other.0.key + && match (&self.0.value, &other.0.value) { + (Value::F64(f), Value::F64(of)) => OrderedFloat(*f).eq(&OrderedFloat(*of)), + (Value::Array(Array::F64(f)), Value::Array(Array::F64(of))) => { + f.len() == of.len() + && f.iter() + .zip(of.iter()) + .all(|(f, of)| OrderedFloat(*f).eq(&OrderedFloat(*of))) + } + (non_float, other_non_float) => non_float.eq(other_non_float), + } + } +} + +impl Eq for HashKeyValue {} + +/// A unique set of attributes that can be used as instrument identifiers. +/// +/// This must implement [Hash], [PartialEq], and [Eq] so it may be used as +/// HashMap keys and other de-duplication methods. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct AttributeSet(BTreeSet); + +impl From<&[KeyValue]> for AttributeSet { + fn from(values: &[KeyValue]) -> Self { + let mut seen = HashSet::with_capacity(values.len()); + AttributeSet( + values + .iter() + .rev() + .filter_map(|kv| { + if seen.contains(&&kv.key) { + None + } else { + seen.insert(&kv.key); + Some(HashKeyValue(kv.clone())) + } + }) + .collect(), + ) + } +} + +impl AttributeSet { + pub(crate) fn len(&self) -> usize { + self.0.len() + } + + pub(crate) fn retain(&mut self, f: F) + where + F: Fn(&KeyValue) -> bool, + { + self.0.retain(|kv| f(&kv.0)) + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter().map(|kv| (&kv.0.key, &kv.0.value)) + } +} diff --git a/opentelemetry-sdk/src/metrics/controllers/basic.rs b/opentelemetry-sdk/src/metrics/controllers/basic.rs deleted file mode 100644 index 1033e1d29d..0000000000 --- a/opentelemetry-sdk/src/metrics/controllers/basic.rs +++ /dev/null @@ -1,470 +0,0 @@ -use std::{ - collections::HashMap, - fmt, - sync::{Arc, Mutex}, - time::{Duration, SystemTime}, -}; - -use futures_channel::{mpsc, oneshot}; -use futures_util::{stream, StreamExt}; -use opentelemetry_api::{ - global, - metrics::{noop, Meter, MeterProvider, MetricsError, Result}, - Context, InstrumentationLibrary, -}; - -use crate::{ - export::metrics::{ - Checkpointer, CheckpointerFactory, InstrumentationLibraryReader, LockedCheckpointer, - MetricsExporter, Reader, - }, - metrics::{ - accumulator, - registry::{self, UniqueInstrumentMeterCore}, - sdk_api::{ - wrap_meter_core, AsyncInstrumentCore, Descriptor, MeterCore, SyncInstrumentCore, - }, - Accumulator, - }, - runtime::Runtime, - Resource, -}; - -/// DefaultPeriod is used for: -/// -/// - the minimum time between calls to `collect`. -/// - the timeout for `export`. -/// - the timeout for `collect`. -const DEFAULT_PERIOD: Duration = Duration::from_secs(10); - -/// Returns a new builder using the provided checkpointer factory. -/// -/// Use builder options (including optional exporter) to configure a metric -/// export pipeline. -pub fn basic(factory: T) -> BasicControllerBuilder -where - T: CheckpointerFactory + Send + Sync + 'static, -{ - BasicControllerBuilder { - checkpointer_factory: Box::new(factory), - resource: None, - exporter: None, - collect_period: None, - collect_timeout: None, - push_timeout: None, - } -} - -/// Organizes and synchronizes collection of metric data in both "pull" and -/// "push" configurations. -/// -/// This supports two distinct modes: -/// -/// - Push and Pull: `start` must be called to begin calling the exporter; -/// `collect` is called periodically after starting the controller. -/// - Pull-Only: `start` is optional in this case, to call `collect` -/// periodically. If `start` is not called, `collect` can be called manually to -/// initiate collection. -/// -/// The controller supports mixing push and pull access to metric data using the -/// `InstrumentationLibraryReader` interface. -#[derive(Clone)] -pub struct BasicController(Arc); - -struct ControllerInner { - meters: Mutex>>, - checkpointer_factory: Box, - resource: Resource, - exporter: Mutex>>, - worker_channel: Mutex>>, - collect_period: Duration, - collect_timeout: Duration, - push_timeout: Duration, - collected_time: Mutex>, -} - -enum WorkerMessage { - Tick, - Shutdown((Context, oneshot::Sender<()>)), -} - -impl BasicController { - /// This begins a ticker that periodically collects and exports metrics with the - /// configured interval. - /// - /// This is required for calling a configured [`MetricsExporter`] (see - /// [`BasicControllerBuilder::with_exporter`]) and is otherwise optional when - /// only pulling metric data. - /// - /// The passed in context is passed to `collect` and subsequently to - /// asynchronous instrument callbacks. Returns an error when the controller was - /// already started. - /// - /// Note that it is not necessary to start a controller when only pulling data; - /// use the `collect` and `try_for_each` methods directly in this case. - pub fn start(&self, cx: &Context, rt: T) -> Result<()> { - let (message_sender, message_receiver) = mpsc::channel(8); - let ticker = rt - .interval(self.0.collect_period) - .map(|_| WorkerMessage::Tick); - - let exporter = self - .0 - .exporter - .lock() - .map(|mut ex| ex.take()) - .unwrap_or_default(); - let resource = self.resource().clone(); - let reader = self.clone(); - let cx = cx.clone(); - // Spawn worker process via user-defined spawn function. - rt.spawn(Box::pin(async move { - let mut messages = Box::pin(stream::select(message_receiver, ticker)); - while let Some(message) = messages.next().await { - match message { - WorkerMessage::Tick => { - match reader.checkpoint(&cx) { - Ok(_) => { - if let Some(exporter) = &exporter { - // TODO timeout - if let Err(err) = exporter.export(&cx, &resource, &reader) { - global::handle_error(err); - } - } - } - Err(err) => global::handle_error(err), - }; - } - WorkerMessage::Shutdown((cx, channel)) => { - let _ = reader.checkpoint(&cx); - if let Some(exporter) = &exporter { - let _ = exporter.export(&cx, &resource, &reader); - } - let _ = channel.send(()); - break; - } - } - } - })); - - *self.0.worker_channel.lock()? = Some(message_sender); - - Ok(()) - } - - /// This waits for the background worker to return and then collects - /// and exports metrics one last time before returning. - /// - /// The passed context is passed to the final `collect` and subsequently to the - /// final asynchronous instruments. - /// - /// Note that `stop` will not cancel an ongoing collection or export. - pub fn stop(&self, cx: &Context) -> Result<()> { - self.0 - .worker_channel - .lock() - .map_err(Into::into) - .and_then(|mut worker| { - if let Some(mut worker) = worker.take() { - let (res_sender, res_receiver) = oneshot::channel(); - if worker - .try_send(WorkerMessage::Shutdown((cx.clone(), res_sender))) - .is_ok() - { - futures_executor::block_on(res_receiver) - .map_err(|err| MetricsError::Other(err.to_string())) - } else { - Ok(()) - } - } else { - Ok(()) - } - }) - } - - /// true if the controller was started via `start`, indicating that the - /// current `Reader` is being kept up-to-date. - pub fn is_running(&self) -> bool { - self.0 - .worker_channel - .lock() - .map(|wc| wc.is_some()) - .unwrap_or(false) - } - - /// `true` if the collector should collect now, based on the current time, the - /// last collection time, and the configured period. - fn should_collect(&self) -> bool { - self.0 - .collected_time - .lock() - .map(|mut collected_time| { - if self.0.collect_period.as_secs() == 0 && self.0.collect_period.as_nanos() == 0 { - return true; - } - let now = SystemTime::now(); - if let Some(collected_time) = *collected_time { - if now.duration_since(collected_time).unwrap_or_default() - < self.0.collect_period - { - return false; - } - } - - *collected_time = Some(now); - true - }) - .unwrap_or(false) - } - - /// Requests a collection. - /// - /// The collection will be skipped if the last collection is aged less than the - /// configured collection period. - pub fn collect(&self, cx: &Context) -> Result<()> { - if self.is_running() { - // When the ticker is `Some`, there's a component - // computing checkpoints with the collection period. - return Err(MetricsError::Other("controller already started".into())); - } - - if !self.should_collect() { - return Ok(()); - } - - self.checkpoint(cx) - } - - /// Get a reference to the current resource. - pub fn resource(&self) -> &Resource { - &self.0.resource - } - - /// Returns a snapshot of current accumulators registered to this controller. - /// - /// This briefly locks the controller. - fn with_accumulator_list(&self, mut f: F) -> Result - where - F: FnMut(&[&AccumulatorCheckpointer]) -> Result, - { - self.0.meters.lock().map_err(Into::into).and_then(|meters| { - let accs = meters - .values() - .filter_map(|unique| { - unique - .meter_core() - .downcast_ref::() - }) - .collect::>(); - f(&accs) - }) - } - - /// Calls the accumulator and checkpointer interfaces to - /// compute the reader. - fn checkpoint(&self, cx: &Context) -> Result<()> { - self.with_accumulator_list(|accs| { - for acc in accs { - self.checkpoint_single_accumulator(cx, acc)?; - } - - Ok(()) - }) - } - - fn checkpoint_single_accumulator( - &self, - cx: &Context, - ac: &AccumulatorCheckpointer, - ) -> Result<()> { - ac.checkpointer - .checkpoint(&mut |ckpt: &mut dyn LockedCheckpointer| { - ckpt.start_collection(); - - if self.0.collect_timeout.as_secs() != 0 && !self.0.collect_timeout.as_nanos() == 0 - { - // TODO timeouts - } - - ac.accumulator.collect(cx, ckpt.processor()); - - ckpt.finish_collection() - }) - } -} - -impl MeterProvider for BasicController { - fn versioned_meter( - &self, - name: &'static str, - version: Option<&'static str>, - schema_url: Option<&'static str>, - ) -> Meter { - self.0 - .meters - .lock() - .map(|mut meters| { - let library = InstrumentationLibrary::new(name, version, schema_url); - let meter_core = meters.entry(library.clone()).or_insert_with(|| { - let checkpointer = self.0.checkpointer_factory.checkpointer(); - Arc::new(registry::unique_instrument_meter_core( - AccumulatorCheckpointer { - accumulator: accumulator(checkpointer.clone().as_dyn_processor()), - checkpointer, - library: library.clone(), - }, - )) - }); - wrap_meter_core(meter_core.clone(), library) - }) - .unwrap_or_else(|_| { - noop::NoopMeterProvider::new().versioned_meter(name, version, schema_url) - }) - } -} - -struct AccumulatorCheckpointer { - accumulator: Accumulator, - checkpointer: Arc, - library: InstrumentationLibrary, -} - -impl MeterCore for AccumulatorCheckpointer { - fn new_sync_instrument( - &self, - descriptor: Descriptor, - ) -> Result> { - self.accumulator.new_sync_instrument(descriptor) - } - - fn new_async_instrument( - &self, - descriptor: Descriptor, - ) -> Result> { - self.accumulator.new_async_instrument(descriptor) - } - - fn register_callback(&self, f: Box) -> Result<()> { - self.accumulator.register_callback(f) - } -} - -impl InstrumentationLibraryReader for BasicController { - fn try_for_each( - &self, - f: &mut dyn FnMut(&InstrumentationLibrary, &mut dyn Reader) -> Result<()>, - ) -> Result<()> { - let mut res = Ok(()); - self.with_accumulator_list(|acs| { - for ac_pair in acs { - if res.is_err() { - continue; - } - - res = ac_pair - .checkpointer - .checkpoint(&mut |locked| f(&ac_pair.library, locked.reader())) - } - - Ok(()) - })?; - - res - } -} - -impl fmt::Debug for BasicController { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BasicController") - .field("resource", &self.0.resource) - .field("collect_period", &self.0.collect_period) - .field("collect_timeout", &self.0.collect_timeout) - .field("push_timeout", &self.0.push_timeout) - .field("collected_time", &self.0.collect_timeout) - .finish() - } -} - -/// Options for configuring a [`BasicController`] -pub struct BasicControllerBuilder { - checkpointer_factory: Box, - resource: Option, - exporter: Option>, - collect_period: Option, - collect_timeout: Option, - push_timeout: Option, -} - -impl BasicControllerBuilder { - /// Sets the [`Resource`] used for this controller. - pub fn with_resource(mut self, resource: Resource) -> Self { - self.resource = Some(resource); - self - } - - /// Sets the exporter used for exporting metric data. - /// - /// Note: Exporters such as Prometheus that pull data do not implement - /// [`MetricsExporter`]. They will directly call `collect` and `try_for_each`. - pub fn with_exporter(mut self, exporter: impl MetricsExporter + Send + Sync + 'static) -> Self { - self.exporter = Some(Box::new(exporter)); - self - } - - /// Sets the interval between calls to `collect` a checkpoint. - /// - /// When pulling metrics and not exporting, this is the minimum time between - /// calls to a pull-only configuration, collection is performed on - /// demand; set this to `0` to always collect. - /// - /// When exporting metrics, this must be > 0s. - /// - /// Default value is 10s. - pub fn with_collect_period(mut self, collect_period: Duration) -> Self { - self.collect_period = Some(collect_period); - self - } - - /// Sets the timeout of the `collect` and subsequent observer instrument - /// callbacks. - /// - /// Default value is 10s. If zero or none, no collect timeout is applied. - pub fn with_collect_timeout(mut self, collect_timeout: Duration) -> Self { - self.collect_timeout = Some(collect_timeout); - self - } - - /// Sets push controller timeout when a exporter is configured. - /// - /// Default value is 10s. If zero, no export timeout is applied. - pub fn with_push_timeout(mut self, push_timeout: Duration) -> Self { - self.push_timeout = Some(push_timeout); - self - } - - /// Creates a new basic controller. - pub fn build(self) -> BasicController { - BasicController(Arc::new(ControllerInner { - meters: Default::default(), - checkpointer_factory: self.checkpointer_factory, - resource: self.resource.unwrap_or_default(), - exporter: Mutex::new(self.exporter), - worker_channel: Mutex::new(None), - collect_period: self.collect_period.unwrap_or(DEFAULT_PERIOD), - collect_timeout: self.collect_timeout.unwrap_or(DEFAULT_PERIOD), - push_timeout: self.push_timeout.unwrap_or(DEFAULT_PERIOD), - collected_time: Default::default(), - })) - } -} - -impl fmt::Debug for BasicControllerBuilder { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BasicControllerBuilder") - .field("resource", &self.resource) - .field("collect_period", &self.collect_period) - .field("collect_timeout", &self.collect_timeout) - .field("push_timeout", &self.push_timeout) - .finish() - } -} diff --git a/opentelemetry-sdk/src/metrics/controllers/mod.rs b/opentelemetry-sdk/src/metrics/controllers/mod.rs deleted file mode 100644 index c7326193ad..0000000000 --- a/opentelemetry-sdk/src/metrics/controllers/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! SDK Metrics Controllers -mod basic; - -pub use basic::{basic, BasicController, BasicControllerBuilder}; diff --git a/opentelemetry-sdk/src/metrics/controllers/push.rs b/opentelemetry-sdk/src/metrics/controllers/push.rs deleted file mode 100644 index f74c522060..0000000000 --- a/opentelemetry-sdk/src/metrics/controllers/push.rs +++ /dev/null @@ -1,199 +0,0 @@ -use crate::{ - export::metrics::{AggregatorSelector, Checkpointer, ExportKindFor, Exporter}, - metrics::{ - self, - processors::{self, BasicProcessor}, - Accumulator, - }, - Resource, -}; -use futures_channel::mpsc; -use futures_util::{ - future::Future, - stream::{select, Stream, StreamExt as _}, - task, -}; -use opentelemetry_api::global; -use opentelemetry_api::metrics::registry; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::time; - -const DEFAULT_PUSH_PERIOD: time::Duration = time::Duration::from_secs(10); - -/// Create a new `PushControllerBuilder`. -pub fn push( - aggregator_selector: AS, - export_selector: ES, - exporter: E, - spawn: SP, - interval: I, -) -> PushControllerBuilder -where - AS: AggregatorSelector + Send + Sync + 'static, - ES: ExportKindFor + Send + Sync + 'static, - E: Exporter + Send + Sync + 'static, - SP: Fn(PushControllerWorker) -> SO, - I: Fn(time::Duration) -> IO, -{ - PushControllerBuilder { - aggregator_selector: Box::new(aggregator_selector), - export_selector: Box::new(export_selector), - exporter: Box::new(exporter), - spawn, - interval, - resource: None, - period: None, - timeout: None, - } -} - -/// Organizes a periodic push of metric data. -#[derive(Debug)] -pub struct PushController { - message_sender: Mutex>, - provider: registry::RegistryMeterProvider, -} - -#[derive(Debug)] -enum PushMessage { - Tick, - Shutdown, -} - -/// The future which executes push controller work periodically. Can be run on a -/// passed in executor. -#[allow(missing_debug_implementations)] -pub struct PushControllerWorker { - messages: Pin + Send>>, - accumulator: Accumulator, - processor: Arc, - exporter: Box, - _timeout: time::Duration, -} - -impl PushControllerWorker { - fn on_tick(&mut self) { - // TODO handle timeout - if let Err(err) = self.processor.lock().and_then(|mut checkpointer| { - checkpointer.start_collection(); - self.accumulator.0.collect(&mut checkpointer); - checkpointer.finish_collection()?; - self.exporter.export(checkpointer.checkpoint_set()) - }) { - global::handle_error(err) - } - } -} - -impl Future for PushControllerWorker { - type Output = (); - fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> task::Poll { - loop { - match futures_util::ready!(self.messages.poll_next_unpin(cx)) { - // Span batch interval time reached, export current spans. - Some(PushMessage::Tick) => self.on_tick(), - // Stream has terminated or processor is shutdown, return to finish execution. - None | Some(PushMessage::Shutdown) => { - return task::Poll::Ready(()); - } - } - } - } -} - -impl Drop for PushControllerWorker { - fn drop(&mut self) { - // Try to push data one last time - self.on_tick() - } -} - -impl PushController { - /// The controller's meter provider. - pub fn provider(&self) -> registry::RegistryMeterProvider { - self.provider.clone() - } -} - -impl Drop for PushController { - fn drop(&mut self) { - if let Ok(mut sender) = self.message_sender.lock() { - let _ = sender.try_send(PushMessage::Shutdown); - } - } -} - -/// Configuration for building a new `PushController`. -#[derive(Debug)] -pub struct PushControllerBuilder { - aggregator_selector: Box, - export_selector: Box, - exporter: Box, - spawn: S, - interval: I, - resource: Option, - period: Option, - timeout: Option, -} - -impl PushControllerBuilder -where - S: Fn(PushControllerWorker) -> SO, - I: Fn(time::Duration) -> IS, - IS: Stream + Send + 'static, -{ - /// Configure the period of this controller - pub fn with_period(self, period: time::Duration) -> Self { - PushControllerBuilder { - period: Some(period), - ..self - } - } - - /// Configure the resource used by this controller - pub fn with_resource(self, resource: Resource) -> Self { - PushControllerBuilder { - resource: Some(resource), - ..self - } - } - - /// Config the timeout of one request. - pub fn with_timeout(self, duration: time::Duration) -> Self { - PushControllerBuilder { - timeout: Some(duration), - ..self - } - } - - /// Build a new `PushController` with this configuration. - pub fn build(self) -> PushController { - let processor = processors::basic(self.aggregator_selector, self.export_selector, false); - let processor = Arc::new(processor); - let mut accumulator = metrics::accumulator(processor.clone()); - - if let Some(resource) = self.resource { - accumulator = accumulator.with_resource(resource); - } - let accumulator = accumulator.build(); - let provider = registry::meter_provider(Arc::new(accumulator.clone())); - - let (message_sender, message_receiver) = mpsc::channel(256); - let ticker = - (self.interval)(self.period.unwrap_or(DEFAULT_PUSH_PERIOD)).map(|_| PushMessage::Tick); - - (self.spawn)(PushControllerWorker { - messages: Box::pin(select(message_receiver, ticker)), - accumulator, - processor, - exporter: self.exporter, - _timeout: self.timeout.unwrap_or(DEFAULT_PUSH_PERIOD), - }); - - PushController { - message_sender: Mutex::new(message_sender), - provider, - } - } -} diff --git a/opentelemetry-sdk/src/metrics/data/mod.rs b/opentelemetry-sdk/src/metrics/data/mod.rs new file mode 100644 index 0000000000..dc6e69bf5c --- /dev/null +++ b/opentelemetry-sdk/src/metrics/data/mod.rs @@ -0,0 +1,167 @@ +//! Types for delivery of pre-aggregated metric time series data. + +use std::{any, borrow::Cow, fmt, time::SystemTime}; + +use opentelemetry_api::{metrics::Unit, KeyValue}; + +use crate::{instrumentation::Scope, metrics::attributes::AttributeSet, Resource}; + +pub use self::temporality::Temporality; + +mod temporality; + +/// A collection of [ScopeMetrics] and the associated [Resource] that created them. +#[derive(Debug)] +pub struct ResourceMetrics { + /// The entity that collected the metrics. + pub resource: Resource, + /// The collection of metrics with unique [Scope]s. + pub scope_metrics: Vec, +} + +/// A collection of metrics produced by a meter. +#[derive(Default, Debug)] +pub struct ScopeMetrics { + /// The [Scope] that the meter was created with. + pub scope: Scope, + /// The list of aggregations created by the meter. + pub metrics: Vec, +} + +/// A collection of one or more aggregated time series from an [Instrument]. +/// +/// [Instrument]: crate::metrics::Instrument +#[derive(Debug)] +pub struct Metric { + /// The name of the instrument that created this data. + pub name: Cow<'static, str>, + /// The description of the instrument, which can be used in documentation. + pub description: Cow<'static, str>, + /// The unit in which the instrument reports. + pub unit: Unit, + /// The aggregated data from an instrument]. + pub data: Box, +} + +/// The store of data reported by an [Instrument]. +/// +/// It will be one of: [Gauge], [Sum], or [Histogram]. +/// +/// [Instrument]: crate::metrics::Instrument +pub trait Aggregation: fmt::Debug + any::Any + Send + Sync { + /// Support downcasting + fn as_any(&self) -> &dyn any::Any; +} + +/// A measurement of the current value of an instrument. +#[derive(Debug)] +pub struct Gauge { + /// Represents individual aggregated measurements with unique attributes. + pub data_points: Vec>, +} + +impl Aggregation for Gauge { + fn as_any(&self) -> &dyn any::Any { + self + } +} + +/// Represents the sum of all measurements of values from an instrument. +#[derive(Debug)] +pub struct Sum { + /// Represents individual aggregated measurements with unique attributes. + pub data_points: Vec>, + /// Describes if the aggregation is reported as the change from the last report + /// time, or the cumulative changes since a fixed start time. + pub temporality: Temporality, + /// Whether this aggregation only increases or decreases. + pub is_monotonic: bool, +} + +impl Aggregation for Sum { + fn as_any(&self) -> &dyn any::Any { + self + } +} + +/// DataPoint is a single data point in a time series. +#[derive(Debug)] +pub struct DataPoint { + /// Attributes is the set of key value pairs that uniquely identify the + /// time series. + pub attributes: AttributeSet, + /// The time when the time series was started. + pub start_time: Option, + /// The time when the time series was recorded. + pub time: Option, + /// The value of this data point. + pub value: T, + /// The sampled [Exemplar]s collected during the time series. + pub exemplars: Vec>, +} + +/// Represents the histogram of all measurements of values from an instrument. +#[derive(Debug)] +pub struct Histogram { + /// Individual aggregated measurements with unique attributes. + pub data_points: Vec>, + /// Describes if the aggregation is reported as the change from the last report + /// time, or the cumulative changes since a fixed start time. + pub temporality: Temporality, +} + +impl Aggregation for Histogram { + fn as_any(&self) -> &dyn any::Any { + self + } +} + +/// A single histogram data point in a time series. +#[derive(Debug)] +pub struct HistogramDataPoint { + /// The set of key value pairs that uniquely identify the time series. + pub attributes: AttributeSet, + /// The time when the time series was started. + pub start_time: SystemTime, + /// The time when the time series was recorded. + pub time: SystemTime, + + /// The number of updates this histogram has been calculated with. + pub count: u64, + /// The upper bounds of the buckets of the histogram. + /// + /// Because the last boundary is +infinity this one is implied. + pub bounds: Vec, + /// The count of each of the buckets. + pub bucket_counts: Vec, + + /// The minimum value recorded. + pub min: Option, + /// The maximum value recorded. + pub max: Option, + /// The sum of the values recorded. + pub sum: f64, + + /// The sampled [Exemplar]s collected during the time series. + pub exemplars: Vec>, +} + +/// A measurement sampled from a time series providing a typical example. +#[derive(Debug)] +pub struct Exemplar { + /// The attributes recorded with the measurement but filtered out of the + /// time series' aggregated data. + pub filtered_attributes: Vec, + /// The time when the measurement was recorded. + pub time: SystemTime, + /// The measured value. + pub value: T, + /// The ID of the span that was active during the measurement. + /// + /// If no span was active or the span was not sampled this will be empty. + pub span_id: [u8; 8], + /// The ID of the trace the active span belonged to during the measurement. + /// + /// If no span was active or the span was not sampled this will be empty. + pub trace_id: [u8; 16], +} diff --git a/opentelemetry-sdk/src/metrics/data/temporality.rs b/opentelemetry-sdk/src/metrics/data/temporality.rs new file mode 100644 index 0000000000..df64d6d4d7 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/data/temporality.rs @@ -0,0 +1,16 @@ +/// Defines the window that an aggregation was calculated over. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Temporality { + /// A measurement interval that continues to expand forward in time from a + /// starting point. + /// + /// New measurements are added to all previous measurements since a start time. + Cumulative, + + /// A measurement interval that resets each cycle. + /// + /// Measurements from one cycle are recorded independently, measurements from + /// other cycles do not affect them. + Delta, +} diff --git a/opentelemetry-sdk/src/metrics/exporter.rs b/opentelemetry-sdk/src/metrics/exporter.rs new file mode 100644 index 0000000000..8ee8fc2cc5 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/exporter.rs @@ -0,0 +1,34 @@ +//! Interfaces for exporting metrics +use async_trait::async_trait; +use opentelemetry_api::metrics::Result; + +use crate::metrics::{ + data::ResourceMetrics, + reader::{AggregationSelector, TemporalitySelector}, +}; + +/// Exporter handles the delivery of metric data to external receivers. +/// +/// This is the final component in the metric push pipeline. +#[async_trait] +pub trait PushMetricsExporter: + AggregationSelector + TemporalitySelector + Send + Sync + 'static +{ + /// Export serializes and transmits metric data to a receiver. + /// + /// All retry logic must be contained in this function. The SDK does not + /// implement any retry logic. All errors returned by this function are + /// considered unrecoverable and will be reported to a configured error + /// Handler. + async fn export(&self, metrics: &mut ResourceMetrics) -> Result<()>; + + /// Flushes any metric data held by an exporter. + async fn force_flush(&self) -> Result<()>; + + /// Flushes all metric data held by an exporter and releases any held + /// computational resources. + /// + /// After Shutdown is called, calls to Export will perform no operation and + /// instead will return an error indicating the shutdown state. + async fn shutdown(&self) -> Result<()>; +} diff --git a/opentelemetry-sdk/src/metrics/instrument.rs b/opentelemetry-sdk/src/metrics/instrument.rs new file mode 100644 index 0000000000..a234de0180 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/instrument.rs @@ -0,0 +1,375 @@ +use std::{any::Any, borrow::Cow, fmt, hash::Hash, marker, sync::Arc}; + +use opentelemetry_api::{ + metrics::{ + AsyncInstrument, MetricsError, Result, SyncCounter, SyncHistogram, SyncUpDownCounter, Unit, + }, + Context, KeyValue, +}; + +use crate::{ + instrumentation::Scope, + metrics::data::Temporality, + metrics::{aggregation::Aggregation, attributes::AttributeSet, internal::Aggregator}, +}; + +pub(crate) const EMPTY_AGG_MSG: &str = "no aggregators for observable instrument"; + +/// The identifier of a group of instruments that all perform the same function. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum InstrumentKind { + /// Identifies a group of instruments that record increasing values synchronously + /// with the code path they are measuring. + Counter, + /// A group of instruments that record increasing and decreasing values + /// synchronously with the code path they are measuring. + UpDownCounter, + /// A group of instruments that record a distribution of values synchronously with + /// the code path they are measuring. + Histogram, + /// A group of instruments that record increasing values in an asynchronous + /// callback. + ObservableCounter, + /// A group of instruments that record increasing and decreasing values in an + /// asynchronous callback. + ObservableUpDownCounter, + /// a group of instruments that record current values in an asynchronous callback. + ObservableGauge, +} + +/// Describes properties an instrument is created with, also used for filtering +/// in [View](crate::metrics::View)s. +/// +/// # Example +/// +/// Instruments can be used as criteria for views. +/// +/// ``` +/// use opentelemetry_sdk::metrics::{new_view, Aggregation, Instrument, Stream}; +/// +/// let criteria = Instrument::new().name("counter_*"); +/// let mask = Stream::new().aggregation(Aggregation::Sum); +/// +/// let view = new_view(criteria, mask); +/// # drop(view); +/// ``` +#[derive(Clone, Default, Debug, PartialEq)] +#[non_exhaustive] +pub struct Instrument { + /// The human-readable identifier of the instrument. + pub name: Cow<'static, str>, + /// describes the purpose of the instrument. + pub description: Cow<'static, str>, + /// The functional group of the instrument. + pub kind: Option, + /// Unit is the unit of measurement recorded by the instrument. + pub unit: Unit, + /// The instrumentation that created the instrument. + pub scope: Scope, +} + +impl Instrument { + /// Create a new instrument with default values + pub fn new() -> Self { + Instrument::default() + } + + /// Set the instrument name. + pub fn name(mut self, name: impl Into>) -> Self { + self.name = name.into(); + self + } + + /// Set the instrument description. + pub fn description(mut self, description: impl Into>) -> Self { + self.description = description.into(); + self + } + + /// Set the instrument unit. + pub fn unit(mut self, unit: Unit) -> Self { + self.unit = unit; + self + } + + /// Set the instrument scope. + pub fn scope(mut self, scope: Scope) -> Self { + self.scope = scope; + self + } + + /// empty returns if all fields of i are their default-value. + pub(crate) fn is_empty(&self) -> bool { + self.name == "" + && self.description == "" + && self.kind.is_none() + && self.unit.as_str() == "" + && self.scope == Scope::default() + } + + pub(crate) fn matches(&self, other: &Instrument) -> bool { + self.matches_name(other) + && self.matches_description(other) + && self.matches_kind(other) + && self.matches_unit(other) + && self.matches_scope(other) + } + + pub(crate) fn matches_name(&self, other: &Instrument) -> bool { + self.name.is_empty() || self.name.as_ref() == other.name.as_ref() + } + + pub(crate) fn matches_description(&self, other: &Instrument) -> bool { + self.description.is_empty() || self.description.as_ref() == other.description.as_ref() + } + + pub(crate) fn matches_kind(&self, other: &Instrument) -> bool { + self.kind.is_none() || self.kind == other.kind + } + + pub(crate) fn matches_unit(&self, other: &Instrument) -> bool { + self.unit.as_str() == "" || self.unit == other.unit + } + + pub(crate) fn matches_scope(&self, other: &Instrument) -> bool { + (self.scope.name.is_empty() || self.scope.name.as_ref() == other.scope.name.as_ref()) + && (self.scope.version.is_none() + || self.scope.version.as_ref().map(AsRef::as_ref) + == other.scope.version.as_ref().map(AsRef::as_ref)) + && (self.scope.schema_url.is_none() + || self.scope.schema_url.as_ref().map(AsRef::as_ref) + == other.scope.schema_url.as_ref().map(AsRef::as_ref)) + } +} + +/// Describes the stream of data an instrument produces. +/// +/// # Example +/// +/// Streams can be used as masks in views. +/// +/// ``` +/// use opentelemetry_sdk::metrics::{new_view, Aggregation, Instrument, Stream}; +/// +/// let criteria = Instrument::new().name("counter_*"); +/// let mask = Stream::new().aggregation(Aggregation::Sum); +/// +/// let view = new_view(criteria, mask); +/// # drop(view); +/// ``` +#[derive(Default)] +#[non_exhaustive] +pub struct Stream { + /// The human-readable identifier of the stream. + pub name: Cow<'static, str>, + /// Describes the purpose of the data. + pub description: Cow<'static, str>, + /// the unit of measurement recorded. + pub unit: Unit, + /// Aggregation the stream uses for an instrument. + pub aggregation: Option, + /// applied to all attributes recorded for an instrument. + pub attribute_filter: Option, +} + +type Filter = Arc bool + Send + Sync>; + +impl Stream { + /// Create a new stream with empty values. + pub fn new() -> Self { + Stream::default() + } + + /// Set the stream name. + pub fn name(mut self, name: impl Into>) -> Self { + self.name = name.into(); + self + } + + /// Set the stream description. + pub fn description(mut self, description: impl Into>) -> Self { + self.description = description.into(); + self + } + + /// Set the stream unit. + pub fn unit(mut self, unit: Unit) -> Self { + self.unit = unit; + self + } + + /// Set the stream aggregation. + pub fn aggregation(mut self, aggregation: Aggregation) -> Self { + self.aggregation = Some(aggregation); + self + } + + /// Set the stream attribute filter. + pub fn attribute_filter( + mut self, + filter: impl Fn(&KeyValue) -> bool + Send + Sync + 'static, + ) -> Self { + self.attribute_filter = Some(Arc::new(filter)); + self + } +} + +impl fmt::Debug for Stream { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Stream") + .field("name", &self.name) + .field("description", &self.description) + .field("unit", &self.unit) + .field("aggregation", &self.aggregation) + .field("attribute_filter", &self.attribute_filter.is_some()) + .finish() + } +} + +/// the identifying properties of a stream. +#[derive(Debug, PartialEq, Eq, Hash)] +pub(crate) struct StreamId { + /// The human-readable identifier of the stream. + pub(crate) name: Cow<'static, str>, + /// Describes the purpose of the data. + pub(crate) description: Cow<'static, str>, + /// the unit of measurement recorded. + pub(crate) unit: Unit, + /// The stream uses for an instrument. + pub(crate) aggregation: String, + /// Monotonic is the monotonicity of an instruments data type. This field is + /// not used for all data types, so a zero value needs to be understood in the + /// context of Aggregation. + pub(crate) monotonic: bool, + /// Temporality is the temporality of a stream's data type. This field is + /// not used by some data types. + pub(crate) temporality: Option, + /// Number is the number type of the stream. + pub(crate) number: Cow<'static, str>, +} + +pub(crate) struct InstrumentImpl { + pub(crate) aggregators: Vec>>, +} + +impl SyncCounter for InstrumentImpl { + fn add(&self, _cx: &Context, val: T, attrs: &[KeyValue]) { + for agg in &self.aggregators { + agg.aggregate(val, AttributeSet::from(attrs)) + } + } +} + +impl SyncUpDownCounter for InstrumentImpl { + fn add(&self, _cx: &Context, val: T, attrs: &[KeyValue]) { + for agg in &self.aggregators { + agg.aggregate(val, AttributeSet::from(attrs)) + } + } +} + +impl SyncHistogram for InstrumentImpl { + fn record(&self, _cx: &Context, val: T, attrs: &[KeyValue]) { + for agg in &self.aggregators { + agg.aggregate(val, AttributeSet::from(attrs)) + } + } +} + +/// A comparable unique identifier of an observable. +#[derive(Clone, Debug)] +pub(crate) struct ObservableId { + pub(crate) inner: IdInner, + _marker: marker::PhantomData, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub(crate) struct IdInner { + /// The human-readable identifier of the instrument. + pub(crate) name: Cow<'static, str>, + /// describes the purpose of the instrument. + pub(crate) description: Cow<'static, str>, + /// The functional group of the instrument. + kind: InstrumentKind, + /// Unit is the unit of measurement recorded by the instrument. + pub(crate) unit: Unit, + /// The instrumentation that created the instrument. + scope: Scope, +} + +impl Hash for ObservableId { + fn hash(&self, state: &mut H) { + self.inner.hash(state) + } +} + +impl PartialEq for ObservableId { + fn eq(&self, other: &Self) -> bool { + self.inner == other.inner + } +} + +impl Eq for ObservableId {} + +#[derive(Clone)] +pub(crate) struct Observable { + pub(crate) id: ObservableId, + aggregators: Vec>>, +} + +impl Observable { + pub(crate) fn new( + scope: Scope, + kind: InstrumentKind, + name: Cow<'static, str>, + description: Cow<'static, str>, + unit: Unit, + aggregators: Vec>>, + ) -> Self { + Self { + id: ObservableId { + inner: IdInner { + name, + description, + kind, + unit, + scope, + }, + _marker: marker::PhantomData, + }, + aggregators, + } + } + + /// Returns `Err` if the observable should not be registered, and `Ok` if it + /// should. + /// + /// An error is returned if this observable is effectively a no-op because it does not have + /// any aggregators. Also, an error is returned if scope defines a Meter other + /// than the observable it was created by. + pub(crate) fn registerable(&self, scope: &Scope) -> Result<()> { + if self.aggregators.is_empty() { + return Err(MetricsError::Other(EMPTY_AGG_MSG.into())); + } + if &self.id.inner.scope != scope { + return Err(MetricsError::Other(format!( + "invalid registration: observable {} from Meter {:?}, registered with Meter {}", + self.id.inner.name, self.id.inner.scope, scope.name, + ))); + } + + Ok(()) + } +} + +impl AsyncInstrument for Observable { + fn observe(&self, _cx: &Context, measurement: T, attrs: &[KeyValue]) { + for agg in &self.aggregators { + agg.aggregate(measurement, AttributeSet::from(attrs)) + } + } + + fn as_any(&self) -> Arc { + Arc::new(self.clone()) + } +} diff --git a/opentelemetry-sdk/src/metrics/internal/aggregator.rs b/opentelemetry-sdk/src/metrics/internal/aggregator.rs new file mode 100644 index 0000000000..4f7e1f1f3b --- /dev/null +++ b/opentelemetry-sdk/src/metrics/internal/aggregator.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; + +use crate::metrics::{attributes::AttributeSet, data::Aggregation}; + +/// Forms an aggregation from a collection of recorded measurements. +pub(crate) trait Aggregator: Send + Sync { + /// Records the measurement, scoped by attr, and aggregates it into an aggregation. + fn aggregate(&self, measurement: T, attrs: AttributeSet); + + /// Returns an Aggregation, for all the aggregated measurements made and ends an aggregation + /// cycle. + fn aggregation(&self) -> Option>; + + /// Used when filtering aggregators + fn as_precompute_aggregator(&self) -> Option>> { + None + } +} + +/// An `Aggregator` that receives values to aggregate that have been pre-computed by the caller. +pub(crate) trait PrecomputeAggregator: Aggregator { + /// Records measurements scoped by attributes that have been filtered by an + /// attribute filter. + /// + /// Pre-computed measurements of filtered attributes need to be recorded separate + /// from those that haven't been filtered so they can be added to the non-filtered + /// pre-computed measurements in a collection cycle and then resets after the + /// cycle (the non-filtered pre-computed measurements are not reset). + fn aggregate_filtered(&self, measurement: T, attrs: AttributeSet); +} diff --git a/opentelemetry-sdk/src/metrics/internal/filter.rs b/opentelemetry-sdk/src/metrics/internal/filter.rs new file mode 100644 index 0000000000..f901951586 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/internal/filter.rs @@ -0,0 +1,74 @@ +use std::sync::Arc; + +use opentelemetry_api::KeyValue; + +use crate::metrics::{attributes::AttributeSet, data::Aggregation}; + +use super::{aggregator::PrecomputeAggregator, Aggregator, Number}; + +/// Returns an [Aggregator] that wraps the passed in aggregator with an +/// attribute filtering function. +/// +/// Both pre-computed non-pre-computed [Aggregator]s can be passed in. An +/// appropriate [Aggregator] will be returned for the detected type. +pub(crate) fn new_filter>( + agg: Arc>, + filter: Arc bool + Send + Sync>, +) -> Arc> { + if let Some(agg) = agg.as_precompute_aggregator() { + Arc::new(PrecomputeFilter { agg, filter }) + } else { + Arc::new(Filter { agg, filter }) + } +} + +/// Wraps an aggregator with an attribute filter. +/// +/// All recorded measurements will have their attributes filtered before they +/// are passed to the underlying aggregator's aggregate method. +/// +/// This should not be used to wrap a pre-computed aggregator. Use a +/// [PrecomputedFilter] instead. +struct Filter { + filter: Arc bool + Send + Sync>, + agg: Arc>, +} + +impl> Aggregator for Filter { + fn aggregate(&self, measurement: T, mut attrs: AttributeSet) { + attrs.retain(self.filter.as_ref()); + self.agg.aggregate(measurement, attrs) + } + + fn aggregation(&self) -> Option> { + self.agg.aggregation() + } +} + +/// An aggregator that applies attribute filter when aggregating for +/// pre-computed aggregations. +/// +/// The pre-computed aggregations need to operate normally when no attribute +/// filtering is done (for sums this means setting the value), but when +/// attribute filtering is done it needs to be added to any set value. +struct PrecomputeFilter> { + filter: Arc bool + Send + Sync>, + agg: Arc>, +} + +impl> Aggregator for PrecomputeFilter { + fn aggregate(&self, measurement: T, mut attrs: AttributeSet) { + let pre_len = attrs.len(); + attrs.retain(self.filter.as_ref()); + if pre_len == attrs.len() { + // No filtering done. + self.agg.aggregate(measurement, attrs) + } else { + self.agg.aggregate_filtered(measurement, attrs) + } + } + + fn aggregation(&self) -> Option> { + self.agg.aggregation() + } +} diff --git a/opentelemetry-sdk/src/metrics/internal/histogram.rs b/opentelemetry-sdk/src/metrics/internal/histogram.rs new file mode 100644 index 0000000000..f4bb057fbb --- /dev/null +++ b/opentelemetry-sdk/src/metrics/internal/histogram.rs @@ -0,0 +1,288 @@ +use std::{ + collections::HashMap, + marker, + sync::{Arc, Mutex}, + time::SystemTime, +}; + +use crate::metrics::{ + aggregation, + attributes::AttributeSet, + data::{self, Aggregation}, +}; + +use super::{Aggregator, Number}; + +#[derive(Default)] +struct Buckets { + counts: Vec, + count: u64, + sum: f64, + min: f64, + max: f64, +} + +impl Buckets { + /// returns buckets with `n` bins. + fn new(n: usize) -> Buckets { + Buckets { + counts: vec![0; n], + ..Default::default() + } + } + + fn bin(&mut self, idx: usize, value: f64) { + self.counts[idx] += 1; + self.count += 1; + self.sum += value; + if value < self.min { + self.min = value; + } else if value > self.max { + self.max = value + } + } +} + +/// Summarizes a set of measurements as an histValues with explicitly defined buckets. +struct HistValues { + bounds: Vec, + values: Mutex>, + _marker: marker::PhantomData, +} + +impl> HistValues { + fn new(mut bounds: Vec) -> Self { + bounds.retain(|v| !v.is_nan()); + bounds.sort_by(|a, b| a.partial_cmp(b).expect("NaNs filtered out")); + + HistValues { + bounds, + values: Mutex::new(Default::default()), + _marker: marker::PhantomData, + } + } +} + +impl Aggregator for HistValues +where + T: Number, +{ + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + // Accept all types to satisfy the Aggregator interface. However, since + // the Aggregation produced by this Aggregator is only float64, convert + // here to only use this type. + let v = match measurement.try_into_float() { + Ok(v) => v, + Err(_) => return, + }; + + // This search will return an index in the range `[0, bounds.len()]`, where + // it will return `bounds.len()` if value is greater than the last element + // of `bounds`. This aligns with the buckets in that the length of buckets + // is `bounds.len()+1`, with the last bucket representing: + // `(bounds[bounds.len()-1], +∞)`. + let idx = self.bounds.partition_point(|x| x < &v); + + let mut values = match self.values.lock() { + Ok(guard) => guard, + Err(_) => return, + }; + + let b = values.entry(attrs).or_insert_with(|| { + // N+1 buckets. For example: + // + // bounds = [0, 5, 10] + // + // Then, + // + // buckets = (-∞, 0], (0, 5.0], (5.0, 10.0], (10.0, +∞) + let mut b = Buckets::new(self.bounds.len() + 1); + // Ensure min and max are recorded values (not zero), for new buckets. + (b.min, b.max) = (v, v); + + b + }); + b.bin(idx, v) + } + + fn aggregation(&self) -> Option> { + None // Never used + } +} + +/// Returns an Aggregator that summarizes a set of +/// measurements as an histogram. Each histogram is scoped by attributes and +/// the aggregation cycle the measurements were made in. +/// +/// Each aggregation cycle is treated independently. When the returned +/// Aggregator's Aggregations method is called it will reset all histogram +/// counts to zero. +pub(crate) fn new_delta_histogram(cfg: &aggregation::Aggregation) -> Arc> +where + T: Number, +{ + let (boundaries, no_min_max) = match cfg { + aggregation::Aggregation::ExplicitBucketHistogram { + boundaries, + no_min_max, + } => (boundaries.clone(), *no_min_max), + _ => (Vec::new(), false), + }; + + Arc::new(DeltaHistogram { + hist_values: HistValues::new(boundaries), + no_min_max, + start: Mutex::new(SystemTime::now()), + }) +} + +/// Summarizes a set of measurements made in a single aggregation cycle as an +/// histogram with explicitly defined buckets. +struct DeltaHistogram { + hist_values: HistValues, + no_min_max: bool, + start: Mutex, +} + +impl> Aggregator for DeltaHistogram { + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + self.hist_values.aggregate(measurement, attrs) + } + + fn aggregation(&self) -> Option> { + let mut values = match self.hist_values.values.lock() { + Ok(guard) if !guard.is_empty() => guard, + _ => return None, + }; + let mut start = match self.start.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; + + let t = SystemTime::now(); + + let data_points = values + .drain() + .map(|(a, b)| { + let mut hdp = data::HistogramDataPoint { + attributes: a, + start_time: *start, + time: t, + count: b.count, + bounds: self.hist_values.bounds.clone(), + bucket_counts: b.counts, + sum: b.sum, + min: None, + max: None, + exemplars: vec![], + }; + + if !self.no_min_max { + hdp.min = Some(b.min); + hdp.max = Some(b.max); + } + + hdp + }) + .collect::>>(); + + // The delta collection cycle resets. + *start = t; + drop(start); + + Some(Box::new(data::Histogram { + temporality: data::Temporality::Delta, + data_points, + })) + } +} + +/// An [Aggregator] that summarizes a set of measurements as an histogram. +/// +/// Each histogram is scoped by attributes. +/// +/// Each aggregation cycle builds from the previous, the histogram counts are +/// the bucketed counts of all values aggregated since the returned Aggregator +/// was created. +pub(crate) fn new_cumulative_histogram(cfg: &aggregation::Aggregation) -> Arc> +where + T: Number, +{ + let (boundaries, no_min_max) = match cfg { + aggregation::Aggregation::ExplicitBucketHistogram { + boundaries, + no_min_max, + } => (boundaries.clone(), *no_min_max), + _ => (Vec::new(), false), + }; + + Arc::new(CumulativeHistogram { + hist_values: HistValues::new(boundaries), + no_min_max, + start: Mutex::new(SystemTime::now()), + }) +} + +/// Summarizes a set of measurements made over all aggregation cycles as an +/// histogram with explicitly defined buckets. +struct CumulativeHistogram { + hist_values: HistValues, + + no_min_max: bool, + start: Mutex, +} + +impl Aggregator for CumulativeHistogram +where + T: Number, +{ + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + self.hist_values.aggregate(measurement, attrs) + } + + fn aggregation(&self) -> Option> { + let mut values = match self.hist_values.values.lock() { + Ok(guard) if !guard.is_empty() => guard, + _ => return None, + }; + let t = SystemTime::now(); + let start = self + .start + .lock() + .map(|s| *s) + .unwrap_or_else(|_| SystemTime::now()); + + // TODO: This will use an unbounded amount of memory if there are unbounded + // number of attribute sets being aggregated. Attribute sets that become + // "stale" need to be forgotten so this will not overload the system. + let data_points = values + .iter_mut() + .map(|(a, b)| { + let mut hdp = data::HistogramDataPoint { + attributes: a.clone(), + start_time: start, + time: t, + count: b.count, + bounds: self.hist_values.bounds.clone(), + bucket_counts: b.counts.clone(), + sum: b.sum, + min: None, + max: None, + exemplars: vec![], + }; + + if !self.no_min_max { + hdp.min = Some(b.min); + hdp.max = Some(b.max); + } + + hdp + }) + .collect::>>(); + + Some(Box::new(data::Histogram { + temporality: data::Temporality::Cumulative, + data_points, + })) + } +} diff --git a/opentelemetry-sdk/src/metrics/internal/last_value.rs b/opentelemetry-sdk/src/metrics/internal/last_value.rs new file mode 100644 index 0000000000..ace190c160 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/internal/last_value.rs @@ -0,0 +1,59 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::SystemTime, +}; + +use crate::metrics::{ + attributes::AttributeSet, + data::{self, Gauge}, +}; + +use super::{Aggregator, Number}; + +/// Timestamped measurement data. +struct DataPointValue { + timestamp: SystemTime, + value: T, +} + +/// An Aggregator that summarizes a set of measurements as the last one made. +pub(crate) fn new_last_value>() -> Arc> { + Arc::new(LastValue::default()) +} + +/// Summarizes a set of measurements as the last one made. +#[derive(Default)] +struct LastValue { + values: Mutex>>, +} + +impl> Aggregator for LastValue { + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + let d = DataPointValue { + timestamp: SystemTime::now(), + value: measurement, + }; + let _ = self.values.lock().map(|mut values| values.insert(attrs, d)); + } + + fn aggregation(&self) -> Option> { + let mut values = match self.values.lock() { + Ok(guard) if !guard.is_empty() => guard, + _ => return None, + }; + + let data_points = values + .drain() + .map(|(attrs, value)| data::DataPoint { + attributes: attrs, + time: Some(value.timestamp), + value: value.value, + start_time: None, + exemplars: vec![], + }) + .collect(); + + Some(Box::new(Gauge { data_points })) + } +} diff --git a/opentelemetry-sdk/src/metrics/internal/mod.rs b/opentelemetry-sdk/src/metrics/internal/mod.rs new file mode 100644 index 0000000000..cba47180d6 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/internal/mod.rs @@ -0,0 +1,51 @@ +mod aggregator; +mod filter; +mod histogram; +mod last_value; +mod sum; + +use core::fmt; +use opentelemetry_api::metrics::Result; +use std::ops::{Add, AddAssign, Sub}; + +pub(crate) use aggregator::Aggregator; +pub(crate) use filter::new_filter; +pub(crate) use histogram::{new_cumulative_histogram, new_delta_histogram}; +pub(crate) use last_value::new_last_value; +pub(crate) use sum::{ + new_cumulative_sum, new_delta_sum, new_precomputed_cumulative_sum, new_precomputed_delta_sum, +}; + +pub(crate) trait Number: + Add + + AddAssign + + Sub + + fmt::Debug + + Clone + + Copy + + PartialEq + + Default + + Send + + Sync + + 'static +{ + fn try_into_float(&self) -> Result; +} + +impl Number for i64 { + fn try_into_float(&self) -> Result { + // May have precision loss at high values + Ok(*self as f64) + } +} +impl Number for u64 { + fn try_into_float(&self) -> Result { + // May have precision loss at high values + Ok(*self as f64) + } +} +impl Number for f64 { + fn try_into_float(&self) -> Result { + Ok(*self) + } +} diff --git a/opentelemetry-sdk/src/metrics/internal/sum.rs b/opentelemetry-sdk/src/metrics/internal/sum.rs new file mode 100644 index 0000000000..af8f5f62db --- /dev/null +++ b/opentelemetry-sdk/src/metrics/internal/sum.rs @@ -0,0 +1,441 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::SystemTime, +}; + +use crate::metrics::{ + attributes::AttributeSet, + data::{self, Aggregation, DataPoint, Temporality}, +}; + +use super::{aggregator::PrecomputeAggregator, Aggregator, Number}; + +/// The storage for sums. +#[derive(Default)] +struct ValueMap> { + values: Mutex>, +} + +impl> ValueMap { + pub(crate) fn new() -> Self { + ValueMap { + values: Mutex::new(HashMap::new()), + } + } +} + +impl> Aggregator for ValueMap { + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + if let Ok(mut values) = self.values.lock() { + values + .entry(attrs) + .and_modify(|val| *val += measurement) + .or_insert(measurement); + } + } + + fn aggregation(&self) -> Option> { + None // Never called directly + } +} + +/// Returns an [Aggregator] that summarizes a set of measurements as their +/// arithmetic sum. +/// +/// Each sum is scoped by attributes and the aggregation cycle the measurements +/// were made in. +/// +/// The `monotonic` value is used to specify if the produced [Aggregation] is +/// monotonic or not. The returned [Aggregator] does not make any guarantees this +/// value is accurate. It is up to the caller to ensure it. +/// +/// Each aggregation cycle is treated independently. When the returned +/// [Aggregator::aggregation] method is called it will reset all sums to zero. +pub(crate) fn new_delta_sum(monotonic: bool) -> Arc> +where + T: Number, +{ + Arc::new(DeltaSum { + value_map: ValueMap::new(), + monotonic, + start: Mutex::new(SystemTime::now()), + }) +} + +/// Summarizes a set of measurements made in a single aggregation cycle as their +/// arithmetic sum. +struct DeltaSum> { + value_map: ValueMap, + monotonic: bool, + start: Mutex, +} + +impl Aggregator for DeltaSum +where + T: Number, +{ + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + self.value_map.aggregate(measurement, attrs) + } + + fn aggregation(&self) -> Option> { + let mut values = match self.value_map.values.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; + + if values.is_empty() { + return None; + } + + let t = SystemTime::now(); + let prev_start = self.start.lock().map(|start| *start).unwrap_or(t); + let data_points = values + .drain() + .map(|(attrs, value)| DataPoint { + attributes: attrs, + start_time: Some(prev_start), + time: Some(t), + value, + exemplars: vec![], + }) + .collect(); + let out = data::Sum { + temporality: Temporality::Delta, + is_monotonic: self.monotonic, + data_points, + }; + + // The delta collection cycle resets. + if let Ok(mut start) = self.start.lock() { + *start = t; + } + + Some(Box::new(out)) + } +} + +/// Returns an [Aggregator] that summarizes a set of measurements as their +/// arithmetic sum. +/// +/// Each sum is scoped by attributes and the aggregation cycle the measurements +/// were made in. +/// +/// The monotonic value is used to communicate the produced [Aggregation] is +/// monotonic or not. The returned [Aggregator] does not make any guarantees this +/// value is accurate. It is up to the caller to ensure it. +/// +/// Each aggregation cycle is treated independently. When the returned +/// Aggregator's Aggregation method is called it will reset all sums to zero. +pub(crate) fn new_cumulative_sum>(monotonic: bool) -> Arc> { + Arc::new(CumulativeSum { + value_map: ValueMap::new(), + monotonic, + start: Mutex::new(SystemTime::now()), + }) +} + +/// Summarizes a set of measurements made over all aggregation cycles as their +/// arithmetic sum. +struct CumulativeSum> { + value_map: ValueMap, + monotonic: bool, + start: Mutex, +} + +impl> Aggregator for CumulativeSum { + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + self.value_map.aggregate(measurement, attrs) + } + + fn aggregation(&self) -> Option> { + let values = match self.value_map.values.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; + + if values.is_empty() { + return None; + } + + let t = SystemTime::now(); + let prev_start = self.start.lock().map(|start| *start).unwrap_or(t); + // TODO: This will use an unbounded amount of memory if there + // are unbounded number of attribute sets being aggregated. Attribute + // sets that become "stale" need to be forgotten so this will not + // overload the system. + let data_points = values + .iter() + .map(|(attrs, value)| DataPoint { + attributes: attrs.clone(), + start_time: Some(prev_start), + time: Some(t), + value: *value, + exemplars: vec![], + }) + .collect(); + + let out: data::Sum = data::Sum { + temporality: Temporality::Cumulative, + is_monotonic: self.monotonic, + data_points, + }; + + Some(Box::new(out)) + } +} + +/// The recorded measurement value for a set of attributes. +#[derive(Default)] +struct PrecomputedValue> { + /// The last value measured for a set of attributes that were not filtered. + measured: T, + /// The sum of values from measurements that had their attributes filtered. + filtered: T, +} + +/// The storage for precomputed sums. +#[derive(Default)] +struct PrecomputedMap> { + values: Mutex>>, +} + +impl> PrecomputedMap { + pub(crate) fn new() -> Self { + Default::default() + } +} + +impl> Aggregator for PrecomputedMap { + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + let mut values = match self.values.lock() { + Ok(guard) => guard, + Err(_) => return, + }; + + values + .entry(attrs) + .and_modify(|v| v.measured = measurement) + .or_insert(PrecomputedValue { + measured: measurement, + ..Default::default() + }); + } + + fn aggregation(&self) -> Option> { + None // Never called + } +} + +impl> PrecomputeAggregator for PrecomputedMap { + fn aggregate_filtered(&self, measurement: T, attrs: AttributeSet) { + let mut values = match self.values.lock() { + Ok(guard) => guard, + Err(_) => return, + }; + + values + .entry(attrs) + .and_modify(|v| v.filtered = measurement) + .or_insert(PrecomputedValue { + filtered: measurement, + ..Default::default() + }); + } +} + +/// An [Aggregator] that summarizes a set of pre-computed sums. +/// +/// Each sum is scoped by attributes and the aggregation cycle the measurements +/// were made in. +/// +/// The `monotonic` value is used to specify if the produced [Aggregation] is +/// monotonic or not. The returned [Aggregator] does not make any guarantees this +/// value is accurate. It is up to the caller to ensure it. +/// +/// The output [Aggregation] will report recorded values as delta temporality. +pub(crate) fn new_precomputed_delta_sum(monotonic: bool) -> Arc> +where + T: Number, +{ + Arc::new(PrecomputedDeltaSum { + precomputed_map: PrecomputedMap::new(), + reported: Default::default(), + monotonic, + start: Mutex::new(SystemTime::now()), + }) +} + +/// Summarizes a set of pre-computed sums recorded over all aggregation cycles +/// as the delta of these sums. +pub(crate) struct PrecomputedDeltaSum> { + precomputed_map: PrecomputedMap, + reported: Mutex>, + monotonic: bool, + start: Mutex, +} + +impl Aggregator for PrecomputedDeltaSum +where + T: Number, +{ + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + self.precomputed_map.aggregate(measurement, attrs) + } + + /// Returns the recorded pre-computed sums as an [Aggregation]. + /// + /// The sum values are expressed as the delta between what was measured this + /// collection cycle and the previous. + /// + /// All pre-computed sums that were recorded for attributes sets reduced by an + /// attribute filter (filtered-sums) are summed together and added to any + /// pre-computed sum value recorded directly for the resulting attribute set + /// (unfiltered-sum). The filtered-sums are reset to zero for the next + /// collection cycle, and the unfiltered-sum is kept for the next collection + /// cycle. + fn aggregation(&self) -> Option> { + let mut values = match self.precomputed_map.values.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; + let mut reported = match self.reported.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; + + if values.is_empty() { + return None; + } + + let t = SystemTime::now(); + let prev_start = self.start.lock().map(|start| *start).unwrap_or(t); + // TODO: This will use an unbounded amount of memory if there + // are unbounded number of attribute sets being aggregated. Attribute + // sets that become "stale" need to be forgotten so this will not + // overload the system. + let data_points = values + .iter_mut() + .map(|(attrs, value)| { + let v: T = value.measured.sub(value.filtered); + let default = T::default(); + let delta = v - *reported.get(attrs).unwrap_or(&default); + if delta != default { + reported.insert(attrs.clone(), v); + } + value.filtered = T::default(); + DataPoint { + attributes: attrs.clone(), + start_time: Some(prev_start), + time: Some(t), + value: delta, + exemplars: vec![], + } + }) + .collect(); + let out = data::Sum { + temporality: Temporality::Delta, + is_monotonic: self.monotonic, + data_points, + }; + + // The delta collection cycle resets. + let _ = self.start.lock().map(|mut start| *start = t); + + drop(reported); // drop before values guard is dropped + + Some(Box::new(out)) + } +} + +/// An [Aggregator] that summarizes a set of pre-computed sums. +/// +/// Each sum is scoped by attributes and the aggregation cycle the measurements +/// were made in. +/// +/// The `monotonic` value is used to specify if the produced [Aggregation] is +/// monotonic or not. The returned [Aggregator] does not make any guarantees this +/// value is accurate. It is up to the caller to ensure it. +/// +/// The output [Aggregation] will report recorded values as cumulative +/// temporality. +pub(crate) fn new_precomputed_cumulative_sum(monotonic: bool) -> Arc> +where + T: Number, +{ + Arc::new(PrecomputedCumulativeSum { + precomputed_map: PrecomputedMap::default(), + monotonic, + start: Mutex::new(SystemTime::now()), + }) +} + +/// Directly records and reports a set of pre-computed sums. +pub(crate) struct PrecomputedCumulativeSum> { + precomputed_map: PrecomputedMap, + monotonic: bool, + start: Mutex, +} + +impl Aggregator for PrecomputedCumulativeSum +where + T: Number, +{ + fn aggregate(&self, measurement: T, attrs: AttributeSet) { + self.precomputed_map.aggregate(measurement, attrs) + } + + /// Returns the recorded pre-computed sums as an [Aggregation]. + /// + /// The sum values are expressed directly as they are assumed to be recorded as + /// the cumulative sum of a some measured phenomena. + /// + /// All pre-computed sums that were recorded for attributes sets reduced by an + /// attribute filter (filtered-sums) are summed together and added to any + /// pre-computed sum value recorded directly for the resulting attribute set + /// (unfiltered-sum). The filtered-sums are reset to zero for the next + /// collection cycle, and the unfiltered-sum is kept for the next collection + /// cycle. + fn aggregation(&self) -> Option> { + let mut values = match self.precomputed_map.values.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; + + if values.is_empty() { + return None; + } + + let t = SystemTime::now(); + let prev_start = self.start.lock().map(|start| *start).unwrap_or(t); + // TODO: This will use an unbounded amount of memory if there + // are unbounded number of attribute sets being aggregated. Attribute + // sets that become "stale" need to be forgotten so this will not + // overload the system. + let data_points = values + .iter_mut() + .map(|(attrs, value)| { + let v = value.measured + value.filtered; + value.filtered = T::default(); + DataPoint { + attributes: attrs.clone(), + start_time: Some(prev_start), + time: Some(t), + value: v, + exemplars: vec![], + } + }) + .collect(); + let out = data::Sum { + temporality: Temporality::Cumulative, + is_monotonic: self.monotonic, + data_points, + }; + + // The delta collection cycle resets. + let _ = self.start.lock().map(|mut start| *start = t); + + Some(Box::new(out)) + } +} diff --git a/opentelemetry-sdk/src/metrics/manual_reader.rs b/opentelemetry-sdk/src/metrics/manual_reader.rs new file mode 100644 index 0000000000..96576b8180 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/manual_reader.rs @@ -0,0 +1,215 @@ +use std::{ + fmt, + sync::{Mutex, Weak}, +}; + +use opentelemetry_api::{ + global, + metrics::{MetricsError, Result}, + Context, +}; + +use super::{ + data::{ResourceMetrics, Temporality}, + instrument::InstrumentKind, + pipeline::Pipeline, + reader::{ + AggregationSelector, DefaultAggregationSelector, DefaultTemporalitySelector, + MetricProducer, MetricReader, SdkProducer, TemporalitySelector, + }, +}; + +/// A simple [MetricReader] that allows an application to read metrics on demand. +/// +/// See [ManualReaderBuilder] for configuration options. +/// +/// # Example +/// +/// ``` +/// use opentelemetry_sdk::metrics::ManualReader; +/// +/// // can specify additional reader configuration +/// let reader = ManualReader::builder().build(); +/// # drop(reader) +/// ``` +pub struct ManualReader { + inner: Box>, + temporality_selector: Box, + aggregation_selector: Box, +} + +impl fmt::Debug for ManualReader { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ManualReader") + } +} + +#[derive(Debug, Default)] +struct ManualReaderInner { + sdk_producer: Option>, + is_shutdown: bool, + external_producers: Vec>, +} + +impl ManualReader { + /// Configuration for this reader + pub fn builder() -> ManualReaderBuilder { + ManualReaderBuilder::default() + } + + /// A [MetricReader] which is directly called to collect metrics. + pub(crate) fn new( + temporality_selector: Box, + aggregation_selector: Box, + ) -> Self { + ManualReader { + inner: Box::new(Mutex::new(ManualReaderInner::default())), + temporality_selector, + aggregation_selector, + } + } +} + +impl TemporalitySelector for ManualReader { + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.temporality_selector.temporality(kind) + } +} + +impl AggregationSelector for ManualReader { + fn aggregation(&self, kind: InstrumentKind) -> super::aggregation::Aggregation { + self.aggregation_selector.aggregation(kind) + } +} + +impl MetricReader for ManualReader { + /// Register a pipeline which enables the caller to read metrics from the SDK + /// on demand. + fn register_pipeline(&self, pipeline: Weak) { + let _ = self.inner.lock().map(|mut inner| { + // Only register once. If producer is already set, do nothing. + if inner.sdk_producer.is_none() { + inner.sdk_producer = Some(pipeline); + } else { + global::handle_error(MetricsError::Config( + "duplicate reader registration, did not register manual reader".into(), + )) + } + }); + } + + /// Stores the external [MetricProducer] which enables the caller to read + /// metrics on demand. + fn register_producer(&self, producer: Box) { + let _ = self.inner.lock().map(|mut inner| { + if !inner.is_shutdown { + inner.external_producers.push(producer); + } + }); + } + + /// Gathers all metrics from the SDK and other [MetricProducer]s, calling any + /// callbacks necessary and returning the results. + /// + /// Returns an error if called after shutdown. + fn collect(&self, cx: &Context, rm: &mut ResourceMetrics) -> Result<()> { + let inner = self.inner.lock()?; + match &inner.sdk_producer.as_ref().and_then(|w| w.upgrade()) { + Some(producer) => producer.produce(cx, rm)?, + None => { + return Err(MetricsError::Other( + "reader is shut down or not registered".into(), + )) + } + }; + + let mut errs = vec![]; + for producer in &inner.external_producers { + match producer.produce(cx) { + Ok(metrics) => rm.scope_metrics.push(metrics), + Err(err) => errs.push(err), + } + } + + if errs.is_empty() { + Ok(()) + } else { + Err(MetricsError::Other(format!("{:?}", errs))) + } + } + + /// ForceFlush is a no-op, it always returns nil. + fn force_flush(&self, _cx: &Context) -> Result<()> { + Ok(()) + } + + /// Closes any connections and frees any resources used by the reader. + fn shutdown(&self) -> Result<()> { + let mut inner = self.inner.lock()?; + + // Any future call to collect will now return an error. + inner.sdk_producer = None; + inner.is_shutdown = true; + inner.external_producers = Vec::new(); + + Ok(()) + } +} + +/// Configuration for a [ManualReader] +pub struct ManualReaderBuilder { + temporality_selector: Box, + aggregation_selector: Box, +} + +impl fmt::Debug for ManualReaderBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ManualReaderBuilder") + } +} + +impl Default for ManualReaderBuilder { + fn default() -> Self { + ManualReaderBuilder { + temporality_selector: Box::new(DefaultTemporalitySelector { _private: () }), + aggregation_selector: Box::new(DefaultAggregationSelector { _private: () }), + } + } +} + +impl ManualReaderBuilder { + /// New manual builder configuration + pub fn new() -> Self { + Default::default() + } + + /// Sets the [TemporalitySelector] a reader will use to determine the [Temporality] of + /// an instrument based on its kind. If this option is not used, the reader will use + /// the default temporality selector. + pub fn with_temporality_selector( + mut self, + temporality_selector: impl TemporalitySelector + 'static, + ) -> Self { + self.temporality_selector = Box::new(temporality_selector); + self + } + + /// Sets the [AggregationSelector] a reader will use to determine the + /// aggregation to use for an instrument based on its kind. + /// + /// If this option is not used, the reader will use the default aggregation + /// selector or the aggregation explicitly passed for a view matching an + /// instrument. + pub fn with_aggregation_selector( + mut self, + aggregation_selector: Box, + ) -> Self { + self.aggregation_selector = aggregation_selector; + self + } + + /// Create a new [ManualReader] from this configuration. + pub fn build(self) -> ManualReader { + ManualReader::new(self.temporality_selector, self.aggregation_selector) + } +} diff --git a/opentelemetry-sdk/src/metrics/meter.rs b/opentelemetry-sdk/src/metrics/meter.rs new file mode 100644 index 0000000000..5fee675c52 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/meter.rs @@ -0,0 +1,628 @@ +use core::fmt; +use std::{ + any::Any, + borrow::Cow, + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, +}; + +use opentelemetry_api::{ + global, + metrics::{ + noop::NoopRegistration, AsyncInstrument, Callback, Counter, Histogram, InstrumentProvider, + MetricsError, ObservableCounter, ObservableGauge, ObservableUpDownCounter, + Observer as ApiObserver, Registration, Result, Unit, UpDownCounter, + }, + Context, KeyValue, +}; + +use crate::instrumentation::Scope; +use crate::metrics::{ + instrument::{ + Instrument, InstrumentImpl, InstrumentKind, Observable, ObservableId, StreamId, + EMPTY_AGG_MSG, + }, + internal::{self, Number}, + pipeline::{Pipelines, Resolver}, +}; + +/// Handles the creation and coordination of all metric instruments. +/// +/// A meter represents a single instrumentation scope; all metric telemetry +/// produced by an instrumentation scope will use metric instruments from a +/// single meter. +/// +/// See the [Meter API] docs for usage. +/// +/// [Meter API]: opentelemetry_api::metrics::Meter +pub struct Meter { + scope: Scope, + pipes: Arc, + u64_inst_provider: InstProvider, + i64_inst_provider: InstProvider, + f64_inst_provider: InstProvider, +} + +impl Meter { + pub(crate) fn new(scope: Scope, pipes: Arc) -> Self { + let view_cache = Default::default(); + + Meter { + scope: scope.clone(), + pipes: Arc::clone(&pipes), + u64_inst_provider: InstProvider::new( + scope.clone(), + Arc::clone(&pipes), + Arc::clone(&view_cache), + ), + i64_inst_provider: InstProvider::new( + scope.clone(), + Arc::clone(&pipes), + Arc::clone(&view_cache), + ), + f64_inst_provider: InstProvider::new(scope, pipes, view_cache), + } + } +} + +#[doc(hidden)] +impl InstrumentProvider for Meter { + fn u64_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.u64_inst_provider + .lookup( + InstrumentKind::Counter, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| Counter::new(Arc::new(i))) + } + + fn f64_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.f64_inst_provider + .lookup( + InstrumentKind::Counter, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| Counter::new(Arc::new(i))) + } + + fn u64_observable_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.u64_inst_provider.aggregators( + InstrumentKind::ObservableCounter, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableCounter, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableCounter::new(observable)) + } + + fn f64_observable_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.f64_inst_provider.aggregators( + InstrumentKind::ObservableCounter, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableCounter, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableCounter::new(observable)) + } + + fn i64_up_down_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.i64_inst_provider + .lookup( + InstrumentKind::UpDownCounter, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| UpDownCounter::new(Arc::new(i))) + } + + fn f64_up_down_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.f64_inst_provider + .lookup( + InstrumentKind::UpDownCounter, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| UpDownCounter::new(Arc::new(i))) + } + + fn i64_observable_up_down_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.i64_inst_provider.aggregators( + InstrumentKind::ObservableUpDownCounter, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableUpDownCounter, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableUpDownCounter::new(observable)) + } + + fn f64_observable_up_down_counter( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.f64_inst_provider.aggregators( + InstrumentKind::ObservableUpDownCounter, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableUpDownCounter, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableUpDownCounter::new(observable)) + } + + fn u64_observable_gauge( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.u64_inst_provider.aggregators( + InstrumentKind::ObservableGauge, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableGauge, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableGauge::new(observable)) + } + + fn i64_observable_gauge( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.i64_inst_provider.aggregators( + InstrumentKind::ObservableGauge, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableGauge, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableGauge::new(observable)) + } + + fn f64_observable_gauge( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + callback: Option>, + ) -> Result> { + let aggs = self.f64_inst_provider.aggregators( + InstrumentKind::ObservableGauge, + name.clone(), + description.clone(), + unit.clone().unwrap_or_default(), + )?; + let is_drop = aggs.is_empty(); + + let observable = Arc::new(Observable::new( + self.scope.clone(), + InstrumentKind::ObservableGauge, + name, + description.unwrap_or_default(), + unit.unwrap_or_default(), + aggs, + )); + let cb_inst = Arc::clone(&observable); + + if let Some(callback) = callback.filter(|_| !is_drop) { + self.pipes + .register_callback(move |cx: &Context| callback(cx, cb_inst.as_ref())); + } + + Ok(ObservableGauge::new(observable)) + } + + fn f64_histogram( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.f64_inst_provider + .lookup( + InstrumentKind::Histogram, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| Histogram::new(Arc::new(i))) + } + + fn u64_histogram( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.u64_inst_provider + .lookup( + InstrumentKind::Histogram, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| Histogram::new(Arc::new(i))) + } + + fn i64_histogram( + &self, + name: Cow<'static, str>, + description: Option>, + unit: Option, + ) -> Result> { + self.i64_inst_provider + .lookup( + InstrumentKind::Histogram, + name, + description, + unit.unwrap_or_default(), + ) + .map(|i| Histogram::new(Arc::new(i))) + } + + fn register_callback( + &self, + insts: &[Arc], + callback: Box, + ) -> Result> { + if insts.is_empty() { + return Ok(Box::new(NoopRegistration::new())); + } + + let mut reg = Observer::default(); + let mut errs = vec![]; + for inst in insts { + if let Some(i64_obs) = inst.downcast_ref::>() { + if let Err(err) = i64_obs.registerable(&self.scope) { + if !err.to_string().contains(EMPTY_AGG_MSG) { + errs.push(err); + } + continue; + } + reg.register_i64(i64_obs.id.clone()); + } else if let Some(u64_obs) = inst.downcast_ref::>() { + if let Err(err) = u64_obs.registerable(&self.scope) { + if !err.to_string().contains(EMPTY_AGG_MSG) { + errs.push(err); + } + continue; + } + reg.register_u64(u64_obs.id.clone()); + } else if let Some(f64_obs) = inst.downcast_ref::>() { + if let Err(err) = f64_obs.registerable(&self.scope) { + if !err.to_string().contains(EMPTY_AGG_MSG) { + errs.push(err); + } + continue; + } + reg.register_f64(f64_obs.id.clone()); + } else { + // Instrument external to the SDK. + return Err(MetricsError::Other( + "invalid observable: from different implementation".into(), + )); + } + } + + if !errs.is_empty() { + return Err(MetricsError::Other(format!("{errs:?}"))); + } + + if reg.is_empty() { + // All instruments use drop aggregation. + return Ok(Box::new(NoopRegistration::new())); + } + + self.pipes + .register_multi_callback(move |cx: &Context| callback(cx, ®)) + } +} + +#[derive(Default)] +struct Observer { + f64s: HashSet>, + i64s: HashSet>, + u64s: HashSet>, +} + +impl Observer { + fn is_empty(&self) -> bool { + self.f64s.is_empty() && self.i64s.is_empty() && self.u64s.is_empty() + } + + pub(crate) fn register_i64(&mut self, id: ObservableId) { + self.i64s.insert(id); + } + + pub(crate) fn register_f64(&mut self, id: ObservableId) { + self.f64s.insert(id); + } + + pub(crate) fn register_u64(&mut self, id: ObservableId) { + self.u64s.insert(id); + } +} + +impl ApiObserver for Observer { + fn observe_f64( + &self, + cx: &Context, + inst: &dyn AsyncInstrument, + measurement: f64, + attrs: &[KeyValue], + ) { + if let Some(f64_obs) = inst.as_any().downcast_ref::>() { + if self.f64s.contains(&f64_obs.id) { + f64_obs.observe(cx, measurement, attrs) + } else { + global::handle_error( + MetricsError::Other(format!("observable instrument not registered for callback, failed to record. name: {}, description: {}, unit: {:?}, number: f64", + f64_obs.id.inner.name, + f64_obs.id.inner.description, + f64_obs.id.inner.unit, + ))) + } + } else { + global::handle_error(MetricsError::Other( + "unknown observable instrument, failed to record.".into(), + )) + } + } + + fn observe_u64( + &self, + cx: &Context, + inst: &dyn AsyncInstrument, + measurement: u64, + attrs: &[KeyValue], + ) { + if let Some(u64_obs) = inst.as_any().downcast_ref::>() { + if self.u64s.contains(&u64_obs.id) { + u64_obs.observe(cx, measurement, attrs) + } else { + global::handle_error( + MetricsError::Other(format!("observable instrument not registered for callback, failed to record. name: {}, description: {}, unit: {:?}, number: f64", + u64_obs.id.inner.name, + u64_obs.id.inner.description, + u64_obs.id.inner.unit, + ))) + } + } else { + global::handle_error(MetricsError::Other( + "unknown observable instrument, failed to record.".into(), + )) + } + } + + fn observe_i64( + &self, + cx: &Context, + inst: &dyn AsyncInstrument, + measurement: i64, + attrs: &[KeyValue], + ) { + if let Some(i64_obs) = inst.as_any().downcast_ref::>() { + if self.i64s.contains(&i64_obs.id) { + i64_obs.observe(cx, measurement, attrs) + } else { + global::handle_error( + MetricsError::Other(format!("observable instrument not registered for callback, failed to record. name: {}, description: {}, unit: {:?}, number: f64", + i64_obs.id.inner.name, + i64_obs.id.inner.description, + i64_obs.id.inner.unit, + ))) + } + } else { + global::handle_error(MetricsError::Other( + "unknown observable instrument, failed to record.".into(), + )) + } + } +} + +impl fmt::Debug for Meter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Meter").field("scope", &self.scope).finish() + } +} + +/// Provides all OpenTelemetry instruments. +struct InstProvider { + scope: Scope, + resolve: Resolver, +} + +impl InstProvider +where + T: Number, +{ + fn new( + scope: Scope, + pipes: Arc, + cache: Arc, StreamId>>>, + ) -> Self { + InstProvider { + scope, + resolve: Resolver::new(pipes, cache), + } + } + + /// lookup returns the resolved InstrumentImpl. + fn lookup( + &self, + kind: InstrumentKind, + name: Cow<'static, str>, + description: Option>, + unit: Unit, + ) -> Result> { + let aggregators = self.aggregators(kind, name, description, unit)?; + Ok(InstrumentImpl { aggregators }) + } + + fn aggregators( + &self, + kind: InstrumentKind, + name: Cow<'static, str>, + description: Option>, + unit: Unit, + ) -> Result>>> { + let inst = Instrument { + name, + description: description.unwrap_or_default(), + unit, + kind: Some(kind), + scope: self.scope.clone(), + }; + + self.resolve.aggregators(inst) + } +} diff --git a/opentelemetry-sdk/src/metrics/meter_provider.rs b/opentelemetry-sdk/src/metrics/meter_provider.rs new file mode 100644 index 0000000000..59374d4601 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/meter_provider.rs @@ -0,0 +1,137 @@ +use core::fmt; +use std::sync::Arc; + +use opentelemetry_api::{ + metrics::{Meter as ApiMeter, Result}, + Context, +}; + +use crate::{instrumentation::Scope, Resource}; + +use super::{meter::Meter as SdkMeter, pipeline::Pipelines, reader::MetricReader, view::View}; + +/// Handles the creation and coordination of [Meter]s. +/// +/// All `Meter`s created by a `MeterProvider` will be associated with the same +/// [Resource], have the same [View]s applied to them, and have their produced +/// metric telemetry passed to the configured [MetricReader]s. +/// +/// [Meter]: crate::metrics::Meter +#[derive(Clone, Debug)] +pub struct MeterProvider { + pipes: Arc, +} + +impl MeterProvider { + /// Flushes all pending telemetry. + /// + /// There is no guaranteed that all telemetry be flushed or all resources have + /// been released on error. + pub fn builder() -> MeterProviderBuilder { + MeterProviderBuilder::default() + } + + /// Flushes all pending telemetry. + /// + /// There is no guaranteed that all telemetry be flushed or all resources have + /// been released on error. + pub fn force_flush(&self, cx: &Context) -> Result<()> { + self.pipes.force_flush(cx) + } + + /// Shuts down the meter provider flushing all pending telemetry and releasing + /// any held computational resources. + /// + /// This call is idempotent. The first call will perform all flush and releasing + /// operations. Subsequent calls will perform no action and will return an error + /// stating this. + /// + /// Measurements made by instruments from meters this MeterProvider created will + /// not be exported after Shutdown is called. + /// + /// There is no guaranteed that all telemetry be flushed or all resources have + /// been released on error. + pub fn shutdown(&self) -> Result<()> { + self.pipes.shutdown() + } +} + +impl opentelemetry_api::metrics::MeterProvider for MeterProvider { + fn versioned_meter( + &self, + name: &'static str, + version: Option<&'static str>, + schema_url: Option<&'static str>, + ) -> ApiMeter { + let scope = Scope::new(name, version, schema_url); + ApiMeter::new( + scope.clone(), + Arc::new(SdkMeter::new(scope, self.pipes.clone())), + ) + } +} + +/// Configuration options for a [MeterProvider]. +#[derive(Default)] +pub struct MeterProviderBuilder { + resource: Option, + readers: Vec>, + views: Vec>, +} + +impl MeterProviderBuilder { + /// Associates a [Resource] with a [MeterProvider]. + /// + /// This [Resource] represents the entity producing telemetry and is associated + /// with all [Meter]s the [MeterProvider] will create. + /// + /// By default, if this option is not used, the default [Resource] will be used. + /// + /// [Meter]: crate::metrics::Meter + pub fn with_resource(mut self, resource: Resource) -> Self { + self.resource = Some(resource); + self + } + + /// Associates a [MetricReader] with a [MeterProvider]. + /// + /// By default, if this option is not used, the [MeterProvider] will perform no + /// operations; no data will be exported without a [MetricReader]. + pub fn with_reader(mut self, reader: T) -> Self { + self.readers.push(Box::new(reader)); + self + } + + /// Associates a [View] with a [MeterProvider]. + /// + /// [View]s are appended to existing ones in a [MeterProvider] if this option is + /// used multiple times. + /// + /// By default, if this option is not used, the [MeterProvider] will use the + /// default view. + pub fn with_view(mut self, view: T) -> Self { + self.views.push(Arc::new(view)); + self + } + + /// Construct a new [MeterProvider] with this configuration. + pub fn build(self) -> MeterProvider { + MeterProvider { + pipes: Arc::new(Pipelines::new( + self.resource.unwrap_or_default(), + self.readers, + self.views, + )), + } + } +} + +impl fmt::Debug for MeterProviderBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MeterProviderBuilder") + .field("resource", &self.resource) + .field("readers", &self.readers) + .field("views", &self.views.len()) + .finish() + } +} diff --git a/opentelemetry-sdk/src/metrics/mod.rs b/opentelemetry-sdk/src/metrics/mod.rs index 2c242631e9..d765c834d3 100644 --- a/opentelemetry-sdk/src/metrics/mod.rs +++ b/opentelemetry-sdk/src/metrics/mod.rs @@ -1,403 +1,62 @@ -//! # OpenTelemetry Metrics SDK -use crate::export; -use crate::export::metrics::{LockedProcessor, Processor}; -use crate::metrics::{ - aggregators::Aggregator, - sdk_api::{ - AsyncInstrumentCore, AtomicNumber, Descriptor, InstrumentCore, MeterCore, Number, - NumberKind, SyncInstrumentCore, - }, -}; -use fnv::FnvHasher; -use opentelemetry_api::{ - attributes::{hash_attributes, AttributeSet}, - global, - metrics::Result, - Context, KeyValue, -}; -use std::{ - any::Any, - cmp::Ordering, - fmt, - hash::{Hash, Hasher}, - sync::{Arc, Mutex}, -}; -pub mod aggregators; -pub mod controllers; -pub mod processors; -pub mod registry; -pub mod sdk_api; -pub mod selectors; - -/// Creates a new accumulator builder -pub fn accumulator(processor: Arc) -> Accumulator { - Accumulator(Arc::new(AccumulatorCore::new(processor))) -} - -/// Accumulator implements the OpenTelemetry Meter API. The Accumulator is bound -/// to a single `Processor`. -/// -/// The Accumulator supports a collect API to gather and export current data. -/// `Collect` should be arranged according to the processor model. Push-based -/// processors will setup a timer to call `collect` periodically. Pull-based -/// processors will call `collect` when a pull request arrives. -#[derive(Debug, Clone)] -pub struct Accumulator(Arc); - -impl Accumulator { - /// Traverses the list of active records and observers and - /// exports data for each active instrument. - /// - /// During the collection pass, the [`LockedProcessor`] will receive - /// one `export` call per current aggregation. - /// - /// Returns the number of records that were checkpointed. - pub fn collect(&self, cx: &Context, locked_processor: &mut dyn LockedProcessor) -> usize { - self.0.collect(cx, locked_processor) - } -} - -impl MeterCore for Accumulator { - fn new_sync_instrument( - &self, - descriptor: Descriptor, - ) -> Result> { - Ok(Arc::new(SyncInstrument { - instrument: Arc::new(BaseInstrument { - meter: self.clone(), - descriptor, - }), - })) - } - - fn new_async_instrument( - &self, - descriptor: Descriptor, - ) -> Result> { - Ok(Arc::new(AsyncInstrument { - instrument: Arc::new(BaseInstrument { - meter: self.clone(), - descriptor, - }), - })) - } - - fn register_callback(&self, f: Box) -> Result<()> { - self.0 - .callbacks - .lock() - .map_err(Into::into) - .map(|mut callbacks| callbacks.push(f)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] -struct MapKey { - instrument_hash: u64, -} - -#[derive(Debug)] -struct AsyncContextKey; - -type Callback = Box; - -struct AccumulatorCore { - /// A concurrent map of current sync instrument state. - current: dashmap::DashMap>, - - /// Async instrument callbacks - callbacks: Mutex>, - - /// The current epoch number. It is incremented in `collect`. - current_epoch: AtomicNumber, - - /// The configured processor. - processor: Arc, -} - -impl AccumulatorCore { - fn new(processor: Arc) -> Self { - AccumulatorCore { - current: dashmap::DashMap::new(), - current_epoch: NumberKind::U64.zero().to_atomic(), - processor, - callbacks: Default::default(), - } - } - - fn collect(&self, cx: &Context, locked_processor: &mut dyn LockedProcessor) -> usize { - self.run_async_callbacks(cx); - let checkpointed = self.collect_instruments(locked_processor); - self.current_epoch.fetch_add(&NumberKind::U64, &1u64.into()); - - checkpointed - } - - fn run_async_callbacks(&self, cx: &Context) { - match self.callbacks.lock() { - Ok(callbacks) => { - let cx = cx.with_value(AsyncContextKey); - for f in callbacks.iter() { - f(&cx) - } - } - Err(err) => global::handle_error(err), - } - } - - fn collect_instruments(&self, locked_processor: &mut dyn LockedProcessor) -> usize { - let mut checkpointed = 0; - - self.current.retain(|_key, value| { - let mods = &value.update_count.load(); - let coll = &value.collected_count.load(); - - if mods.partial_cmp(&NumberKind::U64, coll) != Some(Ordering::Equal) { - // Updates happened in this interval, - // checkpoint and continue. - checkpointed += self.checkpoint_record(value, locked_processor); - value.collected_count.store(mods); - } else { - // Having no updates since last collection, try to remove if - // there are no bound handles - if Arc::strong_count(value) == 1 { - // There's a potential race between loading collected count and - // loading the strong count in this function. Since this is the - // last we'll see of this record, checkpoint. - if mods.partial_cmp(&NumberKind::U64, coll) != Some(Ordering::Equal) { - checkpointed += self.checkpoint_record(value, locked_processor); - } - return false; - } - }; - true - }); - - checkpointed - } - - fn checkpoint_record( - &self, - record: &Record, - locked_processor: &mut dyn LockedProcessor, - ) -> usize { - if let (Some(current), Some(checkpoint)) = (&record.current, &record.checkpoint) { - if let Err(err) = current.synchronized_move(checkpoint, record.instrument.descriptor()) - { - global::handle_error(err); - - return 0; - } - - let accumulation = export::metrics::accumulation( - record.instrument.descriptor(), - &record.attributes, - checkpoint, - ); - if let Err(err) = locked_processor.process(accumulation) { - global::handle_error(err); - } - - 1 - } else { - 0 - } - } - - // fn checkpoint_async( - // &self, - // instrument: &AsyncInstrument, - // locked_processor: &mut dyn LockedProcessor, - // ) -> usize { - // instrument.recorders.lock().map_or(0, |mut recorders| { - // let mut checkpointed = 0; - // match recorders.as_mut() { - // None => return checkpointed, - // Some(recorders) => { - // recorders.retain(|_key, attribute_recorder| { - // let epoch_diff = self.current_epoch.load().partial_cmp( - // &NumberKind::U64, - // &attribute_recorder.observed_epoch.into(), - // ); - // if epoch_diff == Some(Ordering::Equal) { - // if let Some(observed) = &attribute_recorder.observed { - // let accumulation = export::metrics::accumulation( - // instrument.descriptor(), - // &attribute_recorder.attributes, - // &self.resource, - // observed, - // ); - // - // if let Err(err) = locked_processor.process(accumulation) { - // global::handle_error(err); - // } - // checkpointed += 1; - // } - // } - // - // // Retain if this is not second collection cycle with no - // // observations for this AttributeSet. - // epoch_diff == Some(Ordering::Greater) - // }); - // } - // } - // if recorders.as_ref().map_or(false, |map| map.is_empty()) { - // *recorders = None; - // } - // - // checkpointed - // }) - // } -} - -impl fmt::Debug for AccumulatorCore { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("AccumulatorCore").finish() - } -} - -#[derive(Debug, Clone)] -struct SyncInstrument { - instrument: Arc, -} - -impl SyncInstrumentCore for SyncInstrument { - fn record_one(&self, cx: &Context, number: sdk_api::Number, kvs: &'_ [KeyValue]) { - self.instrument.acquire_handle(kvs).capture_one(cx, number) - } -} - -impl sdk_api::InstrumentCore for SyncInstrument { - fn descriptor(&self) -> &Descriptor { - self.instrument.descriptor() - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -#[derive(Debug, Clone)] -struct AsyncInstrument { - instrument: Arc, -} - -impl AsyncInstrumentCore for AsyncInstrument { - fn observe_one(&self, cx: &Context, number: Number, kvs: &'_ [KeyValue]) { - self.instrument.acquire_handle(kvs).capture_one(cx, number) - } -} - -impl sdk_api::InstrumentCore for AsyncInstrument { - fn descriptor(&self) -> &Descriptor { - self.instrument.descriptor() - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -#[derive(Debug, Clone)] -struct BaseInstrument { - meter: Accumulator, - descriptor: Descriptor, -} - -impl BaseInstrument { - // acquireHandle gets or creates a `*record` corresponding to `kvs`, - // the input attributes. - fn acquire_handle(&self, kvs: &[KeyValue]) -> Arc { - let mut hasher = FnvHasher::default(); - self.descriptor.attribute_hash().hash(&mut hasher); - - hash_attributes(&mut hasher, kvs.iter().map(|kv| (&kv.key, &kv.value))); - - let map_key = MapKey { - instrument_hash: hasher.finish(), - }; - let current = &self.meter.0.current; - if let Some(existing_record) = current.get(&map_key) { - return existing_record.value().clone(); - } - - let record = Arc::new(Record { - update_count: NumberKind::U64.zero().to_atomic(), - collected_count: NumberKind::U64.zero().to_atomic(), - attributes: AttributeSet::from_attributes(kvs.iter().cloned()), - instrument: self.clone(), - current: self - .meter - .0 - .processor - .aggregator_selector() - .aggregator_for(&self.descriptor), - checkpoint: self - .meter - .0 - .processor - .aggregator_selector() - .aggregator_for(&self.descriptor), - }); - current.insert(map_key, record.clone()); - - record - } -} - -impl InstrumentCore for BaseInstrument { - fn descriptor(&self) -> &Descriptor { - &self.descriptor - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -/// record maintains the state of one metric instrument. Due -/// the use of lock-free algorithms, there may be more than one -/// `record` in existence at a time, although at most one can -/// be referenced from the `Accumulator.current` map. -#[derive(Debug)] -struct Record { - /// Incremented on every call to `update`. - update_count: AtomicNumber, - - /// Set to `update_count` on collection, supports checking for no updates during - /// a round. - collected_count: AtomicNumber, - - /// The processed attribute set for this record. - /// - /// TODO: look at perf here. - attributes: AttributeSet, - - /// The corresponding instrument. - instrument: BaseInstrument, - - /// current implements the actual `record_one` API, depending on the type of - /// aggregation. If `None`, the metric was disabled by the exporter. - current: Option>, - checkpoint: Option>, -} - -impl Record { - fn capture_one(&self, cx: &Context, number: Number) { - let current = match &self.current { - Some(current) => current, - // The instrument is disabled according to the AggregatorSelector. - None => return, - }; - if let Err(err) = aggregators::range_test(&number, &self.instrument.descriptor) - .and_then(|_| current.update(cx, &number, &self.instrument.descriptor)) - { - global::handle_error(err); - return; - } - - // Record was modified, inform the collect() that things need - // to be collected while the record is still mapped. - self.update_count.fetch_add(&NumberKind::U64, &1u64.into()); - } -} +//! The rust of the OpenTelemetry metrics SDK. +//! +//! ## Configuration +//! +//! The metrics SDK configuration is stored with each [MeterProvider]. +//! Configuration for [Resource]s, [View]s, and [ManualReader] or +//! [PeriodicReader] instances can be specified. +//! +//! ### Example +//! +//! ``` +//! use opentelemetry_api::{ +//! metrics::{MeterProvider as _, Unit}, +//! Context, KeyValue, +//! }; +//! use opentelemetry_sdk::{metrics::MeterProvider, Resource}; +//! +//! let cx = Context::current(); +//! +//! // Generate SDK configuration, resource, views, etc +//! let resource = Resource::default(); // default attributes about the current process +//! +//! // Create a meter provider with the desired config +//! let provider = MeterProvider::builder().with_resource(resource).build(); +//! +//! // Use the meter provider to create meter instances +//! let meter = provider.meter("my_app"); +//! +//! // Create instruments scoped to the meter +//! let counter = meter +//! .u64_counter("power_consumption") +//! .with_unit(Unit::new("kWh")) +//! .init(); +//! +//! // use instruments to record measurements +//! counter.add(&cx, 10, &[KeyValue::new("rate", "standard")]); +//! ``` +//! +//! [Resource]: crate::Resource + +pub(crate) mod aggregation; +pub(crate) mod attributes; +pub mod data; +pub mod exporter; +pub(crate) mod instrument; +pub(crate) mod internal; +pub(crate) mod manual_reader; +pub(crate) mod meter; +mod meter_provider; +pub(crate) mod periodic_reader; +pub(crate) mod pipeline; +pub mod reader; +pub(crate) mod view; + +pub use aggregation::*; +pub use instrument::*; +pub use manual_reader::*; +pub use meter::*; +pub use meter_provider::*; +pub use periodic_reader::*; +pub use pipeline::Pipeline; +pub use view::*; diff --git a/opentelemetry-sdk/src/metrics/periodic_reader.rs b/opentelemetry-sdk/src/metrics/periodic_reader.rs new file mode 100644 index 0000000000..ef2713d48e --- /dev/null +++ b/opentelemetry-sdk/src/metrics/periodic_reader.rs @@ -0,0 +1,372 @@ +use std::{ + env, fmt, + sync::{Arc, Mutex, Weak}, + time::Duration, +}; + +use futures_channel::{mpsc, oneshot}; +use futures_util::{ + future::{self, Either}, + pin_mut, + stream::{self, FusedStream}, + Stream, StreamExt, +}; +use opentelemetry_api::{ + global, + metrics::{MetricsError, Result}, + Context, +}; + +use crate::runtime::Runtime; +use crate::{ + metrics::{ + exporter::PushMetricsExporter, + reader::{MetricProducer, SdkProducer}, + }, + Resource, +}; + +use super::{ + aggregation::Aggregation, + data::{ResourceMetrics, Temporality}, + instrument::InstrumentKind, + reader::{AggregationSelector, MetricReader, TemporalitySelector}, + Pipeline, +}; + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_INTERVAL: Duration = Duration::from_secs(60); + +const METRIC_EXPORT_INTERVAL_NAME: &str = "OTEL_METRIC_EXPORT_INTERVAL"; +const METRIC_EXPORT_TIMEOUT_NAME: &str = "OTEL_METRIC_EXPORT_TIMEOUT"; + +/// Configuration options for [PeriodicReader]. +/// +/// A periodic reader is a [MetricReader] that collects and exports metric data +/// to the exporter at a defined interval. +/// +/// By default, the returned [MetricReader] will collect and export data every +/// 60 seconds, and will cancel export attempts that exceed 30 seconds. The +/// export time is not counted towards the interval between attempts. +/// +/// The [collect] method of the returned [MetricReader] continues to gather and +/// return metric data to the user. It will not automatically send that data to +/// the exporter outside of the predefined interval. +/// +/// [collect]: MetricReader::collect +#[derive(Debug)] +pub struct PeriodicReaderBuilder { + interval: Duration, + timeout: Duration, + exporter: E, + runtime: RT, +} + +impl PeriodicReaderBuilder +where + E: PushMetricsExporter, + RT: Runtime, +{ + fn new(exporter: E, runtime: RT) -> Self { + let interval = env::var(METRIC_EXPORT_INTERVAL_NAME) + .ok() + .and_then(|v| v.parse().map(Duration::from_millis).ok()) + .unwrap_or(DEFAULT_INTERVAL); + let timeout = env::var(METRIC_EXPORT_TIMEOUT_NAME) + .ok() + .and_then(|v| v.parse().map(Duration::from_millis).ok()) + .unwrap_or(DEFAULT_TIMEOUT); + + PeriodicReaderBuilder { + interval, + timeout, + exporter, + runtime, + } + } + + /// Configures the intervening time between exports for a [PeriodicReader]. + /// + /// This option overrides any value set for the `OTEL_METRIC_EXPORT_INTERVAL` + /// environment variable. + /// + /// If this option is not used or `interval` is equal to zero, 60 seconds is + /// used as the default. + pub fn with_interval(mut self, interval: Duration) -> Self { + if !interval.is_zero() { + self.interval = interval; + } + self + } + + /// Configures the time a [PeriodicReader] waits for an export to complete + /// before canceling it. + /// + /// This option overrides any value set for the `OTEL_METRIC_EXPORT_TIMEOUT` + /// environment variable. + /// + /// If this option is not used or `timeout` is equal to zero, 30 seconds is used + /// as the default. + pub fn with_timeout(mut self, timeout: Duration) -> Self { + if !timeout.is_zero() { + self.timeout = timeout; + } + self + } + + /// Create a [PeriodicReader] with the given config. + pub fn build(self) -> PeriodicReader { + let (message_sender, message_receiver) = mpsc::channel(256); + let ticker = self + .runtime + .interval(self.interval) + .map(|_| Message::Export); + + let messages = Box::pin(stream::select(message_receiver, ticker)); + let reader = PeriodicReader { + exporter: Arc::new(self.exporter), + inner: Arc::new(Mutex::new(PeriodicReaderInner { + message_sender, + sdk_producer: None, + is_shutdown: false, + external_producers: vec![], + })), + }; + + let runtime = self.runtime.clone(); + self.runtime.spawn(Box::pin( + PeriodicReaderWorker { + reader: reader.clone(), + timeout: self.timeout, + runtime, + rm: ResourceMetrics { + resource: Resource::empty(), + scope_metrics: Vec::new(), + }, + } + .run(messages), + )); + + reader + } +} + +/// A [MetricReader] that continuously collects and exports metric data at a set +/// interval. +/// +/// By default it will collect and export data every 60 seconds, and will cancel +/// export attempts that exceed 30 seconds. The export time is not counted +/// towards the interval between attempts. +/// +/// The [collect] method of the returned continues to gather and +/// return metric data to the user. It will not automatically send that data to +/// the exporter outside of the predefined interval. +/// +/// The [runtime] can be selected based on feature flags set for this crate. +/// +/// The exporter can be any exporter that implements [PushMetricsExporter] such +/// as [opentelemetry-otlp]. +/// +/// [collect]: MetricReader::collect +/// [runtime]: crate::runtime +/// [opentelemetry-otlp]: https://docs.rs/opentelemetry-otlp/latest/opentelemetry_otlp/ +/// +/// # Example +/// +/// ```no_run +/// use opentelemetry_sdk::metrics::PeriodicReader; +/// # fn example(get_exporter: impl Fn() -> E, get_runtime: impl Fn() -> R) +/// # where +/// # E: opentelemetry_sdk::metrics::exporter::PushMetricsExporter, +/// # R: opentelemetry_sdk::runtime::Runtime, +/// # { +/// +/// let exporter = get_exporter(); // set up a push exporter like OTLP +/// let runtime = get_runtime(); // select runtime: e.g. opentelemetry_sdk:runtime::Tokio +/// +/// let reader = PeriodicReader::builder(exporter, runtime).build(); +/// # drop(reader); +/// # } +/// ``` +#[derive(Clone)] +pub struct PeriodicReader { + exporter: Arc, + inner: Arc>, +} + +impl PeriodicReader { + /// Configuration options for a periodic reader + pub fn builder(exporter: E, runtime: RT) -> PeriodicReaderBuilder + where + E: PushMetricsExporter, + RT: Runtime, + { + PeriodicReaderBuilder::new(exporter, runtime) + } +} + +impl fmt::Debug for PeriodicReader { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PeriodicReader").finish() + } +} + +struct PeriodicReaderInner { + message_sender: mpsc::Sender, + sdk_producer: Option>, + is_shutdown: bool, + external_producers: Vec>, +} + +#[derive(Debug)] +enum Message { + Export, + Flush(oneshot::Sender>), + Shutdown(oneshot::Sender>), +} + +struct PeriodicReaderWorker { + reader: PeriodicReader, + timeout: Duration, + runtime: RT, + rm: ResourceMetrics, +} + +impl PeriodicReaderWorker { + async fn collect_and_export(&mut self) -> Result<()> { + self.reader.collect(&Context::current(), &mut self.rm)?; + + let export = self.reader.exporter.export(&mut self.rm); + let timeout = self.runtime.delay(self.timeout); + pin_mut!(export); + pin_mut!(timeout); + + match future::select(export, timeout).await { + Either::Left(_) => Ok(()), + Either::Right(_) => Err(MetricsError::Other("export timed out".into())), + } + } + + async fn process_message(&mut self, message: Message) -> bool { + match message { + Message::Export => { + if let Err(err) = self.collect_and_export().await { + global::handle_error(err) + } + } + Message::Flush(ch) => { + let res = self.collect_and_export().await; + if ch.send(res).is_err() { + global::handle_error(MetricsError::Other("flush channel closed".into())) + } + } + Message::Shutdown(ch) => { + let res = self.collect_and_export().await; + if ch.send(res).is_err() { + global::handle_error(MetricsError::Other("shutdown channel closed".into())) + } + return false; + } + } + + true + } + + async fn run(mut self, mut messages: impl Stream + Unpin + FusedStream) { + while let Some(message) = messages.next().await { + if !self.process_message(message).await { + break; + } + } + } +} + +impl AggregationSelector for PeriodicReader { + fn aggregation(&self, kind: InstrumentKind) -> Aggregation { + self.exporter.aggregation(kind) + } +} + +impl TemporalitySelector for PeriodicReader { + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.exporter.temporality(kind) + } +} + +impl MetricReader for PeriodicReader { + fn register_pipeline(&self, pipeline: Weak) { + let mut inner = match self.inner.lock() { + Ok(guard) => guard, + Err(_) => return, + }; + + // Only register once. If producer is already set, do nothing. + if inner.sdk_producer.is_none() { + inner.sdk_producer = Some(pipeline); + } else { + global::handle_error(MetricsError::Other( + "duplicate meter registration, did not register manual reader".into(), + )) + } + } + + fn register_producer(&self, producer: Box) { + let _ = self.inner.lock().map(|mut inner| { + if !inner.is_shutdown { + inner.external_producers.push(producer); + } + }); + } + + fn collect(&self, cx: &Context, rm: &mut ResourceMetrics) -> Result<()> { + let inner = self.inner.lock()?; + match &inner.sdk_producer.as_ref().and_then(|w| w.upgrade()) { + Some(producer) => producer.produce(cx, rm)?, + None => { + return Err(MetricsError::Other( + "reader is shut down or not registered".into(), + )) + } + }; + + let mut errs = vec![]; + for producer in &inner.external_producers { + match producer.produce(cx) { + Ok(metrics) => rm.scope_metrics.push(metrics), + Err(err) => errs.push(err), + } + } + + if errs.is_empty() { + Ok(()) + } else { + Err(MetricsError::Other(format!("{:?}", errs))) + } + } + + fn force_flush(&self, _cx: &Context) -> Result<()> { + let mut inner = self.inner.lock()?; + let (sender, receiver) = oneshot::channel(); + inner + .message_sender + .try_send(Message::Flush(sender)) + .map_err(|e| MetricsError::Other(e.to_string()))?; + + futures_executor::block_on(receiver) + .map_err(|err| MetricsError::Other(err.to_string())) + .and_then(|res| res) + } + + fn shutdown(&self) -> Result<()> { + let mut inner = self.inner.lock()?; + let (sender, receiver) = oneshot::channel(); + inner + .message_sender + .try_send(Message::Shutdown(sender)) + .map_err(|e| MetricsError::Other(e.to_string()))?; + + futures_executor::block_on(receiver) + .map_err(|err| MetricsError::Other(err.to_string())) + .and_then(|res| res) + } +} diff --git a/opentelemetry-sdk/src/metrics/pipeline.rs b/opentelemetry-sdk/src/metrics/pipeline.rs new file mode 100644 index 0000000000..1261e62d63 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/pipeline.rs @@ -0,0 +1,728 @@ +use core::fmt; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, +}; + +use opentelemetry_api::{ + global, + metrics::{MetricsError, Registration, Result, Unit}, + Context, +}; + +use crate::{ + instrumentation::Scope, + metrics::{aggregation, data, internal, view::View}, + Resource, +}; + +use super::{ + data::{Metric, ResourceMetrics, ScopeMetrics, Temporality}, + instrument::{Instrument, InstrumentKind, Stream, StreamId}, + internal::Number, + reader::{MetricReader, SdkProducer}, +}; + +/// Connects all of the instruments created by a meter provider to a [MetricReader]. +/// +/// This is the object that will be registered when a meter provider is +/// created. +/// +/// As instruments are created the instrument should be checked if it exists in +/// the views of a the Reader, and if so each aggregator should be added to the +/// pipeline. +#[doc(hidden)] +pub struct Pipeline { + resource: Resource, + reader: Box, + views: Vec>, + inner: Box>, +} + +impl fmt::Debug for Pipeline { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Pipeline") + } +} + +/// Single or multi-instrument callbacks +type GenericCallback = Arc; + +#[derive(Default)] +struct PipelineInner { + aggregations: HashMap>, + callbacks: Vec, + multi_callbacks: Vec>, +} + +impl fmt::Debug for PipelineInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PipelineInner") + .field("aggregations", &self.aggregations) + .field("callbacks", &self.callbacks.len()) + .finish() + } +} + +impl Pipeline { + /// Adds the [InstrumentSync] to pipeline with scope. + /// + /// This method is not idempotent. Duplicate calls will result in duplicate + /// additions, it is the callers responsibility to ensure this is called with + /// unique values. + fn add_sync(&self, scope: Scope, i_sync: InstrumentSync) { + let _ = self.inner.lock().map(|mut inner| { + inner.aggregations.entry(scope).or_default().push(i_sync); + }); + } + + /// Registers a single instrument callback to be run when `produce` is called. + fn add_callback(&self, callback: Arc) { + let _ = self + .inner + .lock() + .map(|mut inner| inner.callbacks.push(callback)); + } + + /// Registers a multi-instrument callback to be run when `produce` is called. + fn add_multi_callback( + &self, + callback: Arc, + ) -> Result Result<()>> { + let mut inner = self.inner.lock()?; + inner.multi_callbacks.push(Some(callback)); + let idx = inner.multi_callbacks.len() - 1; + + Ok(move |this: &Pipeline| { + let mut inner = this.inner.lock()?; + // can't compare trait objects so use index + toumbstones to drop + inner.multi_callbacks[idx] = None; + Ok(()) + }) + } + + /// Send accumulated telemetry + fn force_flush(&self, cx: &Context) -> Result<()> { + self.reader.force_flush(cx) + } + + /// Shut down pipeline + fn shutdown(&self) -> Result<()> { + self.reader.shutdown() + } +} + +impl SdkProducer for Pipeline { + /// Returns aggregated metrics from a single collection. + fn produce(&self, cx: &Context, rm: &mut ResourceMetrics) -> Result<()> { + let inner = self.inner.lock()?; + for cb in &inner.callbacks { + // TODO consider parallel callbacks. + cb(cx); + } + + for mcb in inner.multi_callbacks.iter().flatten() { + // TODO consider parallel multi callbacks. + mcb(cx); + } + + rm.resource = self.resource.clone(); + rm.scope_metrics.reserve(inner.aggregations.len()); + + let mut i = 0; + for (scope, instruments) in inner.aggregations.iter() { + let sm = match rm.scope_metrics.get_mut(i) { + Some(sm) => sm, + None => { + rm.scope_metrics.push(ScopeMetrics::default()); + rm.scope_metrics.last_mut().unwrap() + } + }; + sm.metrics.reserve(instruments.len()); + + let mut j = 0; + for inst in instruments { + if let Some(data) = inst.aggregator.aggregation() { + let m = Metric { + name: inst.name.clone(), + description: inst.description.clone(), + unit: inst.unit.clone(), + data, + }; + match sm.metrics.get_mut(j) { + Some(old) => *old = m, + None => sm.metrics.push(m), + }; + j += 1; + } + } + + sm.metrics.truncate(j); + if !sm.metrics.is_empty() { + sm.scope = scope.clone(); + i += 1; + } + } + + rm.scope_metrics.truncate(i); + + Ok(()) + } +} + +trait Aggregator: Send + Sync { + fn aggregation(&self) -> Option>; +} + +impl Aggregator for Arc> { + fn aggregation(&self) -> Option> { + self.as_ref().aggregation() + } +} + +/// A synchronization point between a [Pipeline] and an instrument's aggregators. +struct InstrumentSync { + name: Cow<'static, str>, + description: Cow<'static, str>, + unit: Unit, + aggregator: Box, +} + +impl fmt::Debug for InstrumentSync { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InstrumentSync") + .field("name", &self.name) + .field("description", &self.description) + .field("unit", &self.unit) + .finish() + } +} + +type Cache = Mutex>>>>>; + +/// Facilitates inserting of new instruments from a single scope into a pipeline. +struct Inserter { + /// A cache that holds [Aggregator]s inserted into the underlying reader pipeline. + /// + /// This cache ensures no duplicate `Aggregator`s are inserted into the reader + /// pipeline and if a new request during an instrument creation asks for the same + /// `Aggregator` the same instance is returned. + aggregators: Cache, + + /// A cache that holds instrument identifiers for all the instruments a [Meter] has + /// created. + /// + /// It is provided from the `Meter` that owns this inserter. This cache ensures + /// that during the creation of instruments with the same name but different + /// options (e.g. description, unit) a warning message is logged. + views: Arc, StreamId>>>, + + pipeline: Arc, +} + +impl Inserter +where + T: Number, +{ + fn new(p: Arc, vc: Arc, StreamId>>>) -> Self { + Inserter { + aggregators: Default::default(), + views: vc, + pipeline: Arc::clone(&p), + } + } + + /// Inserts the provided instrument into a pipeline. + /// + /// All views the pipeline contains are matched against, and any matching view + /// that creates a unique [Aggregator] will be inserted into the pipeline and + /// included in the returned list. + /// + /// The returned `Aggregator`s are ensured to be deduplicated and unique. If + /// another view in another pipeline that is cached by this inserter's cache has + /// already inserted the same `Aggregator` for the same instrument, that + /// `Aggregator` instance is returned. + /// + /// If another instrument has already been inserted by this inserter, or any + /// other using the same cache, and it conflicts with the instrument being + /// inserted in this call, an `Aggregator` matching the arguments will still be + /// returned but a log message will also be logged to the OTel global logger. + /// + /// If the passed instrument would result in an incompatible `Aggregator`, an + /// error is returned and that `Aggregator` is not inserted or returned. + /// + /// If an instrument is determined to use a [aggregation::Aggregation::Drop], that instrument is + /// not inserted nor returned. + fn instrument(&self, inst: Instrument) -> Result>>> { + let mut matched = false; + let mut aggs = vec![]; + let mut errs = vec![]; + let kind = match inst.kind { + Some(kind) => kind, + None => return Err(MetricsError::Other("instrument must have a kind".into())), + }; + + // The cache will return the same Aggregator instance. Use stream ids to de duplicate. + let mut seen = HashSet::new(); + for v in &self.pipeline.views { + let stream = match v.match_inst(&inst) { + Some(stream) => stream, + None => continue, + }; + matched = true; + + let id = self.stream_id(kind, &stream); + if seen.contains(&id) { + continue; // This aggregator has already been added + } + + let agg = match self.cached_aggregator(&inst.scope, kind, stream) { + Ok(Some(agg)) => agg, + Ok(None) => continue, // Drop aggregator. + Err(err) => { + errs.push(err); + continue; + } + }; + seen.insert(id); + aggs.push(agg); + } + + if matched { + if errs.is_empty() { + return Ok(aggs); + } else { + return Err(MetricsError::Other(format!("{errs:?}"))); + } + } + + // Apply implicit default view if no explicit matched. + let stream = Stream { + name: inst.name, + description: inst.description, + unit: inst.unit, + aggregation: None, + attribute_filter: None, + }; + + match self.cached_aggregator(&inst.scope, kind, stream) { + Ok(agg) => { + if errs.is_empty() { + if let Some(agg) = agg { + aggs.push(agg); + } + Ok(aggs) + } else { + Err(MetricsError::Other(format!("{errs:?}"))) + } + } + Err(err) => { + errs.push(err); + Err(MetricsError::Other(format!("{errs:?}"))) + } + } + } + + /// Returns the appropriate Aggregator for an instrument + /// configuration. If the exact instrument has been created within the + /// inst.Scope, that Aggregator instance will be returned. Otherwise, a new + /// computed Aggregator will be cached and returned. + /// + /// If the instrument configuration conflicts with an instrument that has + /// already been created (e.g. description, unit, data type) a warning will be + /// logged at the "Info" level with the global OTel logger. A valid new + /// Aggregator for the instrument configuration will still be returned without + /// an error. + /// + /// If the instrument defines an unknown or incompatible aggregation, an error + /// is returned. + fn cached_aggregator( + &self, + scope: &Scope, + kind: InstrumentKind, + mut stream: Stream, + ) -> Result>>> { + let agg = if let Some(agg) = stream.aggregation.as_ref() { + agg + } else { + stream.aggregation = Some(self.pipeline.reader.aggregation(kind)); + stream.aggregation.as_ref().unwrap() + }; + + if let Err(err) = is_aggregator_compatible(&kind, agg) { + return Err(MetricsError::Other(format!( + "creating aggregator with instrumentKind: {:?}, aggregation {:?}: {:?}", + kind, stream.aggregation, err, + ))); + } + + let id = self.stream_id(kind, &stream); + // If there is a conflict, the specification says the view should + // still be applied and a warning should be logged. + self.log_conflict(&id); + let (id_temporality, id_monotonic) = (id.temporality, id.monotonic); + let mut cache = self.aggregators.lock()?; + let cached = cache.entry(id).or_insert_with(|| { + let mut agg = match self.aggregator(agg, kind, id_temporality, id_monotonic) { + Ok(Some(agg)) => agg, + other => return other, // Drop aggregator or error + }; + + if let Some(filter) = &stream.attribute_filter { + agg = internal::new_filter(agg, Arc::clone(filter)); + } + + self.pipeline.add_sync( + scope.clone(), + InstrumentSync { + name: stream.name, + description: stream.description, + unit: stream.unit, + aggregator: Box::new(Arc::clone(&agg)), + }, + ); + + Ok(Some(agg)) + }); + + cached + .as_ref() + .map(|o| o.as_ref().map(Arc::clone)) + .map_err(|e| MetricsError::Other(e.to_string())) + } + + /// Validates if an instrument with the same name as id has already been created. + /// + /// If that instrument conflicts with id, a warning is logged. + fn log_conflict(&self, id: &StreamId) { + let _ = self.views.lock().map(|views| { + if let Some(existing) = views.get(&id.name) { + if existing == id { return; } + global::handle_error(MetricsError::Other(format!( + "duplicate metric stream definitions, names: ({} and {}), descriptions: ({} and {}), units: ({:?} and {:?}), numbers: ({} and {}), aggregations: ({:?} and {:?}), monotonics: ({} and {}), temporalities: ({:?} and {:?})", + existing.name, id.name, + existing.description, id.description, + existing.unit, id.unit, + existing.number, id.number, + existing.aggregation, id.aggregation, + existing.monotonic, id.monotonic, + existing.temporality, id.temporality))) + } + }); + } + + fn stream_id(&self, kind: InstrumentKind, stream: &Stream) -> StreamId { + let aggregation = stream + .aggregation + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(); + + StreamId { + name: stream.name.clone(), + description: stream.description.clone(), + unit: stream.unit.clone(), + aggregation, + temporality: Some(self.pipeline.reader.temporality(kind)), + number: Cow::Borrowed(std::any::type_name::()), + monotonic: matches!( + kind, + InstrumentKind::ObservableCounter + | InstrumentKind::Counter + | InstrumentKind::Histogram + ), + } + } + + /// Returns a new [Aggregator] for the given params. + /// + /// If the aggregation is unknown or temporality is invalid, an error is returned. + fn aggregator( + &self, + agg: &aggregation::Aggregation, + kind: InstrumentKind, + temporality: Option, + monotonic: bool, + ) -> Result>>> { + use aggregation::Aggregation; + match agg { + Aggregation::Drop => Ok(None), + Aggregation::LastValue => Ok(Some(internal::new_last_value())), + Aggregation::Sum => { + match kind { + InstrumentKind::ObservableCounter | InstrumentKind::ObservableUpDownCounter => { + // Asynchronous counters and up-down-counters are defined to record + // the absolute value of the count: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#asynchronous-counter-creation + match temporality { + Some(Temporality::Cumulative) => { + return Ok(Some(internal::new_precomputed_cumulative_sum( + monotonic, + ))) + } + Some(Temporality::Delta) => { + return Ok(Some(internal::new_precomputed_delta_sum(monotonic))) + } + _ => { + return Err(MetricsError::Other(format!( + "unrecognized temporality: {:?}", + temporality + ))) + } + } + } + _ => {} + }; + + match temporality { + Some(Temporality::Cumulative) => { + Ok(Some(internal::new_cumulative_sum(monotonic))) + } + Some(Temporality::Delta) => Ok(Some(internal::new_delta_sum(monotonic))), + _ => Err(MetricsError::Other(format!( + "unrecognized temporality: {:?}", + temporality + ))), + } + } + a @ Aggregation::ExplicitBucketHistogram { .. } => match temporality { + Some(Temporality::Cumulative) => Ok(Some(internal::new_cumulative_histogram(a))), + Some(Temporality::Delta) => Ok(Some(internal::new_delta_histogram(a))), + _ => Err(MetricsError::Other(format!( + "unrecognized temporality: {:?}", + temporality + ))), + }, + _ => Err(MetricsError::Other("unknown aggregation".into())), + } + } +} + +/// Checks if the aggregation can be used by the instrument. +/// +/// Current compatibility: +/// +/// | Instrument Kind | Drop | LastValue | Sum | Histogram | Exponential Histogram | +/// |--------------------------|------|-----------|-----|-----------|-----------------------| +/// | Counter | X | | X | X | X | +/// | UpDownCounter | X | | X | | | +/// | Histogram | X | | X | X | X | +/// | Observable Counter | X | | X | | | +/// | Observable UpDownCounter | X | | X | | | +/// | Observable Gauge | X | X | | | |. +fn is_aggregator_compatible(kind: &InstrumentKind, agg: &aggregation::Aggregation) -> Result<()> { + use aggregation::Aggregation; + match agg { + Aggregation::ExplicitBucketHistogram { .. } => { + if kind == &InstrumentKind::Counter || kind == &InstrumentKind::Histogram { + return Ok(()); + } + // TODO: review need for aggregation check after + // https://github.com/open-telemetry/opentelemetry-specification/issues/2710 + Err(MetricsError::Other("incompatible aggregation".into())) + } + Aggregation::Sum => { + match kind { + InstrumentKind::ObservableCounter + | InstrumentKind::ObservableUpDownCounter + | InstrumentKind::Counter + | InstrumentKind::Histogram + | InstrumentKind::UpDownCounter => Ok(()), + _ => { + // TODO: review need for aggregation check after + // https://github.com/open-telemetry/opentelemetry-specification/issues/2710 + Err(MetricsError::Other("incompatible aggregation".into())) + } + } + } + Aggregation::LastValue => { + if kind == &InstrumentKind::ObservableGauge { + return Ok(()); + } + // TODO: review need for aggregation check after + // https://github.com/open-telemetry/opentelemetry-specification/issues/2710 + Err(MetricsError::Other("incompatible aggregation".into())) + } + Aggregation::Drop => Ok(()), + _ => { + // This is used passed checking for default, it should be an error at this point. + Err(MetricsError::Other(format!( + "unknown aggregation {:?}", + agg + ))) + } + } +} + +/// The group of pipelines connecting Readers with instrument measurement. +#[derive(Clone, Debug)] +pub(crate) struct Pipelines(Vec>); + +impl Pipelines { + pub(crate) fn new( + res: Resource, + readers: Vec>, + views: Vec>, + ) -> Self { + let mut pipes = Vec::with_capacity(readers.len()); + for r in readers { + let p = Arc::new(Pipeline { + resource: res.clone(), + reader: r, + views: views.clone(), + inner: Default::default(), + }); + p.reader.register_pipeline(Arc::downgrade(&p)); + pipes.push(p); + } + + Pipelines(pipes) + } + + pub(crate) fn register_callback(&self, callback: F) + where + F: Fn(&Context) + Send + Sync + 'static, + { + let cb = Arc::new(callback); + for pipe in &self.0 { + pipe.add_callback(cb.clone()) + } + } + + /// Registers a multi-instrument callback to be run when `produce` is called. + pub(crate) fn register_multi_callback(&self, f: F) -> Result> + where + F: Fn(&Context) + Send + Sync + 'static, + { + let cb = Arc::new(f); + + let fns = self + .0 + .iter() + .map(|pipe| { + let pipe = Arc::clone(pipe); + let unreg = pipe.add_multi_callback(cb.clone())?; + Ok(Box::new(move || unreg(pipe.as_ref())) as _) + }) + .collect::>()?; + + Ok(Box::new(Unregister(fns))) + } + + /// Force flush all pipelines + pub(crate) fn force_flush(&self, cx: &Context) -> Result<()> { + let mut errs = vec![]; + for pipeline in &self.0 { + if let Err(err) = pipeline.force_flush(cx) { + errs.push(err); + } + } + + if errs.is_empty() { + Ok(()) + } else { + Err(MetricsError::Other(format!("{errs:?}"))) + } + } + + /// Shut down all pipelines + pub(crate) fn shutdown(&self) -> Result<()> { + let mut errs = vec![]; + for pipeline in &self.0 { + if let Err(err) = pipeline.shutdown() { + errs.push(err); + } + } + + if errs.is_empty() { + Ok(()) + } else { + Err(MetricsError::Other(format!("{errs:?}"))) + } + } +} + +struct Unregister(Vec Result<()>>>); + +impl Registration for Unregister { + fn unregister(&mut self) -> Result<()> { + let mut errs = vec![]; + while let Some(unreg) = self.0.pop() { + if let Err(err) = unreg() { + errs.push(err); + } + } + + if errs.is_empty() { + Ok(()) + } else { + Err(MetricsError::Other(format!("{errs:?}"))) + } + } +} + +// func (p pipelines) registerMultiCallback(c multiCallback) metric.Registration { +// unregs := make([]func(), len(p)) +// for i, pipe := range p { +// unregs[i] = pipe.addMultiCallback(c) +// } +// return unregisterFuncs(unregs) +// } +// +// type unregisterFuncs []func() +// +// func (u unregisterFuncs) Unregister() error { +// for _, f := range u { +// f() +// } +// return nil +// } + +/// resolver facilitates resolving Aggregators an instrument needs to aggregate +/// measurements with while updating all pipelines that need to pull from those +/// aggregations. +pub(crate) struct Resolver { + inserters: Vec>, +} + +impl Resolver +where + T: Number, +{ + pub(crate) fn new( + pipelines: Arc, + view_cache: Arc, StreamId>>>, + ) -> Self { + let inserters = pipelines.0.iter().fold(Vec::new(), |mut acc, pipe| { + acc.push(Inserter::new(Arc::clone(pipe), Arc::clone(&view_cache))); + acc + }); + + Resolver { inserters } + } + + /// Aggregators returns the Aggregators that must be updated by the instrument + /// defined by key. + pub(crate) fn aggregators( + &self, + id: Instrument, + ) -> Result>>> { + let (aggs, errs) = + self.inserters + .iter() + .fold((vec![], vec![]), |(mut aggs, mut errs), inserter| { + match inserter.instrument(id.clone()) { + Ok(agg) => aggs.extend(agg), + Err(err) => errs.push(err), + }; + (aggs, errs) + }); + + if errs.is_empty() { + Ok(aggs) + } else { + Err(MetricsError::Other(format!("{errs:?}"))) + } + } +} diff --git a/opentelemetry-sdk/src/metrics/processors/basic.rs b/opentelemetry-sdk/src/metrics/processors/basic.rs deleted file mode 100644 index 378a030777..0000000000 --- a/opentelemetry-sdk/src/metrics/processors/basic.rs +++ /dev/null @@ -1,396 +0,0 @@ -use crate::{ - export::metrics::{ - self, - aggregation::{Temporality, TemporalitySelector}, - Accumulation, AggregatorSelector, Checkpointer, CheckpointerFactory, LockedCheckpointer, - LockedProcessor, Processor, Reader, Record, - }, - metrics::{aggregators::Aggregator, sdk_api::Descriptor}, -}; -use core::fmt; -use fnv::FnvHasher; -use opentelemetry_api::{ - attributes::{hash_attributes, AttributeSet}, - metrics::{MetricsError, Result}, -}; -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::SystemTime; - -/// Create a new basic processor -pub fn factory(aggregator_selector: A, temporality_selector: T) -> BasicProcessorBuilder -where - A: AggregatorSelector + Send + Sync + 'static, - T: TemporalitySelector + Send + Sync + 'static, -{ - BasicProcessorBuilder { - aggregator_selector: Arc::new(aggregator_selector), - temporality_selector: Arc::new(temporality_selector), - } -} - -pub struct BasicProcessorBuilder { - aggregator_selector: Arc, - temporality_selector: Arc, -} - -impl fmt::Debug for BasicProcessorBuilder { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BasicProcessorBuilder").finish() - } -} - -impl CheckpointerFactory for BasicProcessorBuilder { - fn checkpointer(&self) -> Arc { - Arc::new(BasicProcessor { - aggregator_selector: Arc::clone(&self.aggregator_selector), - temporality_selector: Arc::clone(&self.temporality_selector), - state: Mutex::new(BasicProcessorState::default()), - }) - } -} - -/// Basic metric integration strategy -pub struct BasicProcessor { - aggregator_selector: Arc, - temporality_selector: Arc, - state: Mutex, -} - -impl Processor for BasicProcessor { - fn aggregator_selector(&self) -> &dyn AggregatorSelector { - self.aggregator_selector.as_ref() - } -} - -impl Checkpointer for BasicProcessor { - fn checkpoint( - &self, - f: &mut dyn FnMut(&mut dyn LockedCheckpointer) -> Result<()>, - ) -> Result<()> { - self.state.lock().map_err(From::from).and_then(|locked| { - f(&mut BasicLockedProcessor { - parent: self, - state: locked, - }) - }) - } -} - -impl fmt::Debug for BasicProcessor { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BasicProcessor") - .field("state", &self.state) - .finish() - } -} - -/// A locked representation of the processor used where mutable references are necessary. -#[derive(Debug)] -struct BasicLockedProcessor<'a> { - parent: &'a BasicProcessor, - state: MutexGuard<'a, BasicProcessorState>, -} - -impl<'a> LockedProcessor for BasicLockedProcessor<'a> { - fn process(&mut self, accumulation: Accumulation<'_>) -> Result<()> { - if self.state.started_collection != self.state.finished_collection.wrapping_add(1) { - return Err(MetricsError::InconsistentState); - } - - let desc = accumulation.descriptor(); - let mut hasher = FnvHasher::default(); - desc.attribute_hash().hash(&mut hasher); - hash_attributes(&mut hasher, accumulation.attributes().into_iter()); - let key = StateKey(hasher.finish()); - let agg = accumulation.aggregator(); - let finished_collection = self.state.finished_collection; - if let Some(value) = self.state.values.get_mut(&key) { - // Advance the update sequence number. - let same_collection = finished_collection == value.updated; - value.updated = finished_collection; - - // At this point in the code, we have located an existing - // value for some stateKey. This can be because: - // - // (a) stateful aggregation is being used, the entry was - // entered during a prior collection, and this is the first - // time processing an accumulation for this stateKey in the - // current collection. Since this is the first time - // processing an accumulation for this stateKey during this - // collection, we don't know yet whether there are multiple - // accumulators at work. If there are multiple accumulators, - // they'll hit case (b) the second time through. - // - // (b) multiple accumulators are being used, whether stateful - // or not. - // - // Case (a) occurs when the instrument and the exporter - // require memory to work correctly, either because the - // instrument reports a PrecomputedSum to a DeltaExporter or - // the reverse, a non-PrecomputedSum instrument with a - // CumulativeExporter. This logic is encapsulated in - // ExportKind.MemoryRequired(MetricKind). - // - // Case (b) occurs when the variable `sameCollection` is true, - // indicating that the stateKey for Accumulation has already - // been seen in the same collection. When this happens, it - // implies that multiple Accumulators are being used, or that - // a single Accumulator has been configured with an attribute key - // filter. - - if !same_collection { - if !value.current_owned { - // This is the first Accumulation we've seen for this - // stateKey during this collection. Just keep a - // reference to the Accumulator's Aggregator. All the other cases - // copy Aggregator state. - value.current = agg.clone(); - return Ok(()); - } - return agg.synchronized_move(&value.current, desc); - } - - // If the current is not owned, take ownership of a copy - // before merging below. - if !value.current_owned { - let tmp = value.current.clone(); - if let Some(current) = self.parent.aggregator_selector.aggregator_for(desc) { - value.current = current; - value.current_owned = true; - tmp.synchronized_move(&value.current, desc)?; - } - } - - // Combine this `Accumulation` with the prior `Accumulation`. - return value.current.merge(agg.as_ref(), desc); - } - - let stateful = self - .parent - .temporality_selector - .temporality_for(desc, agg.aggregation().kind()) - .memory_required(desc.instrument_kind()); - - let cumulative = if stateful { - if desc.instrument_kind().precomputed_sum() { - // If we know we need to compute deltas, allocate one. - return Err(MetricsError::Other("No cumulative to sum support".into())); - } - // Always allocate a cumulative aggregator if stateful - self.parent.aggregator_selector.aggregator_for(desc) - } else { - None - }; - - self.state.values.insert( - key, - StateValue { - descriptor: desc.clone(), - attributes: accumulation.attributes().clone(), - current_owned: false, - current: agg.clone(), - cumulative, - stateful, - updated: finished_collection, - }, - ); - - Ok(()) - } -} - -impl LockedCheckpointer for BasicLockedProcessor<'_> { - fn processor(&mut self) -> &mut dyn LockedProcessor { - self - } - - fn reader(&mut self) -> &mut dyn Reader { - &mut *self.state - } - - fn start_collection(&mut self) { - if self.state.started_collection != 0 { - self.state.interval_start = self.state.interval_end; - } - self.state.started_collection = self.state.started_collection.wrapping_add(1); - } - - fn finish_collection(&mut self) -> Result<()> { - self.state.interval_end = opentelemetry_api::time::now(); - if self.state.started_collection != self.state.finished_collection.wrapping_add(1) { - return Err(MetricsError::InconsistentState); - } - let finished_collection = self.state.finished_collection; - self.state.finished_collection = self.state.finished_collection.wrapping_add(1); - - let mut result = Ok(()); - - self.state.values.retain(|_key, value| { - // Return early if previous error - if result.is_err() { - return true; - } - - let mkind = value.descriptor.instrument_kind(); - - let stale = value.updated != finished_collection; - let stateless = !value.stateful; - - // The following branch updates stateful aggregators. Skip these updates - // if the aggregator is not stateful or if the aggregator is stale. - if stale || stateless { - // If this processor does not require memory, stale, stateless - // entries can be removed. This implies that they were not updated - // over the previous full collection interval. - if stale && stateless { - return false; - } - return true; - } - - // The only kind of aggregators that are not stateless - // are the ones needing delta to cumulative - // conversion. Merge aggregator state in this case. - if !mkind.precomputed_sum() { - // This line is equivalent to: - // value.cumulative = value.cumulative + value.delta - if let Some(cumulative) = value.cumulative.as_ref() { - result = cumulative.merge(value.current.as_ref(), &value.descriptor) - } - } - - true - }); - - result - } -} - -#[derive(Debug)] -struct BasicProcessorState { - values: HashMap, - // Note: the timestamp logic currently assumes all exports are deltas. - process_start: SystemTime, - interval_start: SystemTime, - interval_end: SystemTime, - started_collection: u64, - finished_collection: u64, -} - -impl Default for BasicProcessorState { - fn default() -> Self { - BasicProcessorState { - values: HashMap::default(), - process_start: opentelemetry_api::time::now(), - interval_start: opentelemetry_api::time::now(), - interval_end: opentelemetry_api::time::now(), - started_collection: 0, - finished_collection: 0, - } - } -} - -impl Reader for BasicProcessorState { - fn try_for_each( - &mut self, - temporality_selector: &dyn TemporalitySelector, - f: &mut dyn FnMut(&Record<'_>) -> Result<()>, - ) -> Result<()> { - if self.started_collection != self.finished_collection { - return Err(MetricsError::InconsistentState); - } - - self.values.iter().try_for_each(|(_key, value)| { - let instrument_kind = value.descriptor.instrument_kind(); - - let agg; - let start; - - match temporality_selector - .temporality_for(&value.descriptor, value.current.aggregation().kind()) - { - Temporality::Cumulative => { - // If stateful, the sum has been computed. If stateless, the - // input was already cumulative. Either way, use the - // checkpointed value: - if value.stateful { - agg = value.cumulative.as_ref(); - } else { - agg = Some(&value.current); - } - - start = self.process_start; - } - Temporality::Delta => { - // Precomputed sums are a special case. - if instrument_kind.precomputed_sum() { - return Err(MetricsError::Other("No cumulative to delta".into())); - } - - if value.updated != self.finished_collection.wrapping_sub(1) { - // skip processing if there is no update in last collection internal and - // temporality is Delta - return Ok(()); - } - - agg = Some(&value.current); - start = self.interval_start; - } - } - - let res = f(&metrics::record( - &value.descriptor, - &value.attributes, - agg, - start, - self.interval_end, - )); - - if let Err(MetricsError::NoDataCollected) = res { - Ok(()) - } else { - res - } - }) - } -} - -#[derive(Debug, PartialEq, Eq, Hash)] -struct StateKey(u64); - -#[derive(Debug)] -struct StateValue { - /// Instrument descriptor - descriptor: Descriptor, - - /// Instrument attributes - attributes: AttributeSet, - - /// Indicates the last sequence number when this value had process called by an - /// accumulator. - updated: u64, - - /// Indicates that a cumulative aggregation is being maintained, taken from the - /// process start time. - stateful: bool, - - /// Indicates that "current" was allocated - /// by the processor in order to merge results from - /// multiple `Accumulator`s during a single collection - /// round, which may happen either because: - /// - /// (1) multiple `Accumulator`s output the same `Accumulation`. - /// (2) one `Accumulator` is configured with dimensionality reduction. - current_owned: bool, - - /// The output from a single `Accumulator` (if !current_owned) or an - /// `Aggregator` owned by the processor used to accumulate multiple values in a - /// single collection round. - current: Arc, - - /// If `Some`, refers to an `Aggregator` owned by the processor used to store - /// the last cumulative value. - cumulative: Option>, -} diff --git a/opentelemetry-sdk/src/metrics/processors/mod.rs b/opentelemetry-sdk/src/metrics/processors/mod.rs deleted file mode 100644 index 5267c631de..0000000000 --- a/opentelemetry-sdk/src/metrics/processors/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Metric Processors -mod basic; - -pub use basic::{factory, BasicProcessor}; diff --git a/opentelemetry-sdk/src/metrics/reader.rs b/opentelemetry-sdk/src/metrics/reader.rs new file mode 100644 index 0000000000..989bad77c0 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/reader.rs @@ -0,0 +1,160 @@ +//! Interfaces for reading and producing metrics +use std::{fmt, sync::Weak}; + +use opentelemetry_api::{metrics::Result, Context}; + +use super::{ + aggregation::Aggregation, + data::{ResourceMetrics, ScopeMetrics, Temporality}, + instrument::InstrumentKind, + pipeline::Pipeline, +}; + +/// The interface used between the SDK and an exporter. +/// +/// Control flow is bi-directional through the `MetricReader`, since the SDK +/// initiates `force_flush` and `shutdown` while the reader initiates +/// collection. The `register_pipeline` method here informs the metric reader +/// that it can begin reading, signaling the start of bi-directional control +/// flow. +/// +/// Typically, push-based exporters that are periodic will implement +/// `MetricExporter` themselves and construct a `PeriodicReader` to satisfy this +/// interface. +/// +/// Pull-based exporters will typically implement `MetricReader` themselves, +/// since they read on demand. +pub trait MetricReader: + AggregationSelector + TemporalitySelector + fmt::Debug + Send + Sync + 'static +{ + /// Registers a [MetricReader] with a [Pipeline]. + /// + /// The pipeline argument allows the `MetricReader` to signal the sdk to collect + /// and send aggregated metric measurements. + fn register_pipeline(&self, pipeline: Weak); + + /// Registers a an external Producer with this [MetricReader]. The Producer is + /// used as a source of aggregated metric data which is incorporated into + /// metrics collected from the SDK. + fn register_producer(&self, producer: Box); + + /// Gathers and returns all metric data related to the [MetricReader] from the SDK and + /// stores it in out. An error is returned if this is called after shutdown + fn collect(&self, cx: &Context, rm: &mut ResourceMetrics) -> Result<()>; + + /// Flushes all metric measurements held in an export pipeline. + /// + /// There is no guaranteed that all telemetry be flushed or all resources have + /// been released on error. + fn force_flush(&self, cx: &Context) -> Result<()>; + + /// Flushes all metric measurements held in an export pipeline and releases any + /// held computational resources. + /// + /// There is no guaranteed that all telemetry be flushed or all resources have + /// been released on error. + /// + /// After `shutdown` is called, calls to `collect` will perform no operation and + /// instead will return an error indicating the shutdown state. + fn shutdown(&self) -> Result<()>; +} + +/// Produces metrics for a [MetricReader]. +pub(crate) trait SdkProducer: fmt::Debug + Send + Sync { + /// Returns aggregated metrics from a single collection. + fn produce(&self, cx: &Context, rm: &mut ResourceMetrics) -> Result<()>; +} + +/// Produces metrics for a [MetricReader] from an external source. +pub trait MetricProducer: fmt::Debug + Send + Sync { + /// Returns aggregated metrics from an external source. + fn produce(&self, cx: &Context) -> Result; +} + +/// An interface for selecting the temporality for an [InstrumentKind]. +pub trait TemporalitySelector: Send + Sync { + /// Selects the temporality to use based on the [InstrumentKind]. + fn temporality(&self, kind: InstrumentKind) -> Temporality; +} + +/// The default temporality used if not specified for a given [InstrumentKind]. +/// +/// [Temporality::Cumulative] will be used for all instrument kinds if this +/// [TemporalitySelector] is used. +#[derive(Clone, Default, Debug)] +pub struct DefaultTemporalitySelector { + pub(crate) _private: (), +} + +impl DefaultTemporalitySelector { + /// Create a new default temporality selector. + pub fn new() -> Self { + Self::default() + } +} + +impl TemporalitySelector for DefaultTemporalitySelector { + fn temporality(&self, _kind: InstrumentKind) -> Temporality { + Temporality::Cumulative + } +} + +/// An interface for selecting the aggregation and the parameters for an +/// [InstrumentKind]. +pub trait AggregationSelector: Send + Sync { + /// Selects the aggregation and the parameters to use for that aggregation based on + /// the [InstrumentKind]. + fn aggregation(&self, kind: InstrumentKind) -> Aggregation; +} + +impl AggregationSelector for T +where + T: Fn(InstrumentKind) -> Aggregation + Send + Sync, +{ + fn aggregation(&self, kind: InstrumentKind) -> Aggregation { + self(kind) + } +} + +/// The default aggregation and parameters for an instrument of [InstrumentKind]. +/// +/// This [AggregationSelector] uses the following selection mapping per [the spec]: +/// +/// * Counter ⇨ Sum +/// * Observable Counter ⇨ Sum +/// * UpDownCounter ⇨ Sum +/// * Observable UpDownCounter ⇨ Sum +/// * Observable Gauge ⇨ LastValue +/// * Histogram ⇨ ExplicitBucketHistogram +/// +/// [the spec]: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.19.0/specification/metrics/sdk.md#default-aggregation +#[derive(Clone, Default, Debug)] +pub struct DefaultAggregationSelector { + pub(crate) _private: (), +} + +impl DefaultAggregationSelector { + /// Create a new default aggregation selector. + pub fn new() -> Self { + Self::default() + } +} + +impl AggregationSelector for DefaultAggregationSelector { + fn aggregation(&self, kind: InstrumentKind) -> Aggregation { + match kind { + InstrumentKind::Counter + | InstrumentKind::UpDownCounter + | InstrumentKind::ObservableCounter + | InstrumentKind::ObservableUpDownCounter => Aggregation::Sum, + InstrumentKind::ObservableGauge => Aggregation::LastValue, + InstrumentKind::Histogram => Aggregation::ExplicitBucketHistogram { + boundaries: vec![ + 0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 750.0, 1000.0, 2500.0, + 5000.0, 7500.0, 10000.0, + ], + no_min_max: false, + }, + } + } +} diff --git a/opentelemetry-sdk/src/metrics/registry.rs b/opentelemetry-sdk/src/metrics/registry.rs deleted file mode 100644 index 6c48849f6a..0000000000 --- a/opentelemetry-sdk/src/metrics/registry.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Metrics Registry API -use crate::metrics::sdk_api::{Descriptor, SyncInstrumentCore}; -use core::fmt; -use opentelemetry_api::{ - metrics::{MetricsError, Result}, - Context, -}; -use std::sync::{Arc, Mutex}; -use std::{any::Any, collections::HashMap}; - -use super::sdk_api::{AsyncInstrumentCore, InstrumentCore, MeterCore}; - -/// Create a new `UniqueInstrumentMeterCore` from a `InstrumentProvider`. -pub fn unique_instrument_meter_core(core: T) -> UniqueInstrumentMeterCore -where - T: AnyMeterCore + Send + Sync + 'static, -{ - UniqueInstrumentMeterCore::wrap(core) -} - -/// An extension trait that allows meters to be downcast -pub trait AnyMeterCore: MeterCore { - /// Returns the current type as [`Any`] - fn as_any(&self) -> &dyn Any; -} - -impl AnyMeterCore for T { - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Implements the [`MeterCore`] interface, adding uniqueness checking for -/// instrument descriptors. -pub struct UniqueInstrumentMeterCore { - inner: Box, - state: Mutex>>, -} - -impl fmt::Debug for UniqueInstrumentMeterCore { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("UniqueInstrumentMeterCore") - } -} - -impl UniqueInstrumentMeterCore { - fn wrap(inner: T) -> Self - where - T: AnyMeterCore + Send + Sync + 'static, - { - UniqueInstrumentMeterCore { - inner: Box::new(inner), - state: Mutex::new(HashMap::default()), - } - } - - pub(crate) fn meter_core(&self) -> &dyn Any { - self.inner.as_any() - } -} - -impl MeterCore for UniqueInstrumentMeterCore { - fn new_sync_instrument( - &self, - descriptor: Descriptor, - ) -> Result> { - self.state.lock().map_err(Into::into).and_then(|mut state| { - let instrument = check_uniqueness(&state, &descriptor)?; - match instrument { - Some(instrument) => Ok(instrument), - None => { - let instrument = self.inner.new_sync_instrument(descriptor.clone())?; - state.insert(descriptor.name().into(), instrument.clone().as_dyn_core()); - - Ok(instrument) - } - } - }) - } - - fn new_async_instrument( - &self, - descriptor: Descriptor, - ) -> Result> { - self.state.lock().map_err(Into::into).and_then(|mut state| { - let instrument = check_uniqueness(&state, &descriptor)?; - match instrument { - Some(instrument) => Ok(instrument), - None => { - let instrument = self.inner.new_async_instrument(descriptor)?; - state.insert( - instrument.descriptor().name().into(), - instrument.clone().as_dyn_core(), - ); - - Ok(instrument) - } - } - }) - } - - fn register_callback(&self, f: Box) -> Result<()> { - self.inner.register_callback(f) - } -} - -fn check_uniqueness( - instruments: &HashMap>, - descriptor: &Descriptor, -) -> Result> { - if let Some(instrument) = instruments.get(descriptor.name()) { - if is_equal(instrument.descriptor(), descriptor) { - Ok(instrument.as_any().downcast_ref::().cloned()) - } else { - Err(MetricsError::MetricKindMismatch(format!( - "metric {} registered as a {:?} {:?}", - descriptor.name(), - descriptor.number_kind(), - descriptor.instrument_kind() - ))) - } - } else { - Ok(None) - } -} - -fn is_equal(a: &Descriptor, b: &Descriptor) -> bool { - a.instrument_kind() == b.instrument_kind() && a.number_kind() == b.number_kind() -} diff --git a/opentelemetry-sdk/src/metrics/sdk_api/async_instrument.rs b/opentelemetry-sdk/src/metrics/sdk_api/async_instrument.rs deleted file mode 100644 index cf80e4933b..0000000000 --- a/opentelemetry-sdk/src/metrics/sdk_api/async_instrument.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Async metrics -use crate::{ - global, - metrics::{sdk_api, MetricsError, Number}, - KeyValue, -}; -use std::fmt; -use std::marker; -use std::sync::Arc; - -/// Observation is used for reporting an asynchronous batch of metric values. -/// Instances of this type should be created by asynchronous instruments (e.g., -/// [ValueObserver::observation]). -/// -/// [ValueObserver::observation]: crate::metrics::ValueObserver::observation() -#[derive(Debug)] -pub struct Observation { - number: Number, - instrument: Arc, -} - -impl Observation { - /// Create a new observation for an instrument - pub(crate) fn new(number: Number, instrument: Arc) -> Self { - Observation { number, instrument } - } - - /// The value of this observation - pub fn number(&self) -> &Number { - &self.number - } - /// The instrument used to record this observation - pub fn instrument(&self) -> &Arc { - &self.instrument - } -} - -/// A type of callback that `f64` observers run. -type F64ObserverCallback = Box) + Send + Sync>; - -/// A type of callback that `u64` observers run. -type U64ObserverCallback = Box) + Send + Sync>; - -/// A type of callback that `u64` observers run. -type I64ObserverCallback = Box) + Send + Sync>; - -/// A callback argument for use with any Observer instrument that will be -/// reported as a batch of observations. -type BatchObserverCallback = Box; - -/// Data passed to an observer callback to capture observations for one -/// asynchronous metric instrument. -pub struct ObserverResult { - instrument: Arc, - f: fn(&[KeyValue], &[Observation]), - _marker: marker::PhantomData, -} - -impl ObserverResult -where - T: Into, -{ - /// New observer result for a given metric instrument - fn new( - instrument: Arc, - f: fn(&[KeyValue], &[Observation]), - ) -> Self { - ObserverResult { - instrument, - f, - _marker: marker::PhantomData, - } - } - - /// Observe captures a single value from the associated instrument callback, - /// with the given attributes. - pub fn observe(&self, value: T, attributes: &[KeyValue]) { - (self.f)( - attributes, - &[Observation { - number: value.into(), - instrument: self.instrument.clone(), - }], - ) - } -} - -impl fmt::Debug for ObserverResult { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ObserverResult") - .field("instrument", &self.instrument) - .field("f", &"fn(&[KeyValue], &[Observation])") - .finish() - } -} - -/// Passed to a batch observer callback to capture observations for multiple -/// asynchronous instruments. -pub struct BatchObserverResult { - f: fn(&[KeyValue], &[Observation]), -} - -impl fmt::Debug for BatchObserverResult { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BatchObserverResult") - .field("f", &"fn(&[KeyValue], &[Observation])") - .finish() - } -} - -impl BatchObserverResult { - /// New observer result for a given metric instrument - fn new(f: fn(&[KeyValue], &[Observation])) -> Self { - BatchObserverResult { f } - } - - /// Captures multiple observations from the associated batch instrument - /// callback, with the given attributes. - pub fn observe(&self, attributes: &[KeyValue], observations: &[Observation]) { - (self.f)(attributes, observations) - } -} diff --git a/opentelemetry-sdk/src/metrics/sdk_api/descriptor.rs b/opentelemetry-sdk/src/metrics/sdk_api/descriptor.rs deleted file mode 100644 index 59f83401b3..0000000000 --- a/opentelemetry-sdk/src/metrics/sdk_api/descriptor.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::metrics::sdk_api::{InstrumentKind, NumberKind}; -use fnv::FnvHasher; -use opentelemetry_api::metrics::Unit; -use std::hash::{Hash, Hasher}; - -/// Descriptor contains all the settings that describe an instrument, including -/// its name, metric kind, number kind, and the configurable options. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Descriptor { - name: String, - instrument_kind: InstrumentKind, - number_kind: NumberKind, - description: Option, - unit: Option, - attribute_hash: u64, -} - -impl Descriptor { - /// Create a new descriptor - pub fn new( - name: String, - instrument_kind: InstrumentKind, - number_kind: NumberKind, - description: Option, - unit: Option, - ) -> Self { - let mut hasher = FnvHasher::default(); - name.hash(&mut hasher); - instrument_kind.hash(&mut hasher); - number_kind.hash(&mut hasher); - if let Some(description) = &description { - description.hash(&mut hasher); - } - if let Some(unit) = &unit { - unit.hash(&mut hasher); - } - - Descriptor { - name, - instrument_kind, - number_kind, - description, - unit, - attribute_hash: hasher.finish(), - } - } - - /// The metric instrument's name. - pub fn name(&self) -> &str { - self.name.as_str() - } - - /// The specific kind of instrument. - pub fn instrument_kind(&self) -> &InstrumentKind { - &self.instrument_kind - } - - /// NumberKind returns whether this instrument is declared over int64, float64, or uint64 - /// values. - pub fn number_kind(&self) -> &NumberKind { - &self.number_kind - } - - /// A human-readable description of the metric instrument. - pub fn description(&self) -> Option<&String> { - self.description.as_ref() - } - - /// Assign a new description - pub fn set_description(&mut self, description: String) { - self.description = Some(description); - } - - /// Unit describes the units of the metric instrument. - pub fn unit(&self) -> Option<&str> { - self.unit.as_ref().map(|unit| unit.as_ref()) - } - - /// The pre-computed hash of the descriptor data - pub fn attribute_hash(&self) -> u64 { - self.attribute_hash - } -} diff --git a/opentelemetry-sdk/src/metrics/sdk_api/instrument_kind.rs b/opentelemetry-sdk/src/metrics/sdk_api/instrument_kind.rs deleted file mode 100644 index 3163dd1e74..0000000000 --- a/opentelemetry-sdk/src/metrics/sdk_api/instrument_kind.rs +++ /dev/null @@ -1,60 +0,0 @@ -/// Kinds of OpenTelemetry metric instruments -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum InstrumentKind { - /// A histogram instrument - Histogram, - /// A gauge observer instrument - GaugeObserver, - /// A synchronous per-request part of a monotonic sum. - Counter, - /// A synchronous per-request part of a non-monotonic sum. - UpDownCounter, - /// An asynchronous per-interval recorder of a monotonic sum. - CounterObserver, - /// An asynchronous per-interval recorder of a non-monotonic sum. - UpDownCounterObserver, -} - -impl InstrumentKind { - /// Whether this is a synchronous kind of instrument. - pub fn synchronous(&self) -> bool { - matches!( - self, - InstrumentKind::Counter | InstrumentKind::UpDownCounter | InstrumentKind::Histogram - ) - } - - /// Whether this is a synchronous kind of instrument. - pub fn asynchronous(&self) -> bool { - !self.synchronous() - } - - /// Whether this kind of instrument adds its inputs (as opposed to grouping). - pub fn adding(&self) -> bool { - matches!( - self, - InstrumentKind::Counter - | InstrumentKind::UpDownCounter - | InstrumentKind::CounterObserver - | InstrumentKind::UpDownCounterObserver - ) - } - - /// Whether this kind of instrument groups its inputs (as opposed to adding). - pub fn grouping(&self) -> bool { - !self.adding() - } - - /// Whether this kind of instrument exposes a non-decreasing sum. - pub fn monotonic(&self) -> bool { - matches!( - self, - InstrumentKind::Counter | InstrumentKind::CounterObserver - ) - } - - /// Whether this kind of instrument receives precomputed sums. - pub fn precomputed_sum(&self) -> bool { - self.adding() && self.asynchronous() - } -} diff --git a/opentelemetry-sdk/src/metrics/sdk_api/mod.rs b/opentelemetry-sdk/src/metrics/sdk_api/mod.rs deleted file mode 100644 index 388670eca7..0000000000 --- a/opentelemetry-sdk/src/metrics/sdk_api/mod.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! SDK API - -// mod async_instrument; -mod descriptor; -mod instrument_kind; -mod number; -mod wrap; -// mod sync_instrument; - -use std::any::Any; -use std::sync::Arc; - -pub use descriptor::*; -pub use instrument_kind::*; -pub use number::*; -use opentelemetry_api::{metrics::Result, Context, KeyValue}; -pub use wrap::wrap_meter_core; - -/// The interface an SDK must implement to supply a Meter implementation. -pub trait MeterCore { - /// Create a new synchronous instrument implementation. - fn new_sync_instrument( - &self, - descriptor: Descriptor, - ) -> Result>; - - /// Create a new asynchronous instrument implementation. - /// - /// Runner is `None` if used in batch as the batch runner is registered separately. - fn new_async_instrument( - &self, - descriptor: Descriptor, - ) -> Result>; - - /// Register a batch observer - fn register_callback(&self, f: Box) -> Result<()>; -} - -/// A utility extension to allow upcasting. -/// -/// Can be removed once [trait_upcasting] is stablized. -/// -/// [trait_upcasting]: https://doc.rust-lang.org/unstable-book/language-features/trait-upcasting.html -pub trait AsDynInstrumentCore { - /// Create an `Arc` from an impl of `InstrumentCore`. - fn as_dyn_core<'a>(self: Arc) -> Arc - where - Self: 'a; -} - -impl AsDynInstrumentCore for T { - fn as_dyn_core<'a>(self: Arc) -> Arc - where - Self: 'a, - { - self - } -} - -/// A common interface for synchronous and asynchronous instruments. -pub trait InstrumentCore: AsDynInstrumentCore { - /// Description of the instrument's descriptor - fn descriptor(&self) -> &Descriptor; - - /// Returns self as any - fn as_any(&self) -> &dyn Any; -} - -/// The implementation-level interface to a generic synchronous instrument -/// (e.g., Histogram and Counter instruments). -pub trait SyncInstrumentCore: InstrumentCore { - /// Capture a single synchronous metric event. - fn record_one(&self, cx: &Context, number: Number, attributes: &'_ [KeyValue]); -} - -/// An implementation-level interface to an asynchronous instrument (e.g., -/// Observable instruments). -pub trait AsyncInstrumentCore: InstrumentCore { - /// Captures a single asynchronous metric event. - fn observe_one(&self, cx: &Context, number: Number, attributes: &'_ [KeyValue]); -} diff --git a/opentelemetry-sdk/src/metrics/sdk_api/sync_instrument.rs b/opentelemetry-sdk/src/metrics/sdk_api/sync_instrument.rs deleted file mode 100644 index c084885a78..0000000000 --- a/opentelemetry-sdk/src/metrics/sdk_api/sync_instrument.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::{ - metrics::{sdk_api, Number}, - KeyValue, -}; -use std::marker; -use std::sync::Arc; - -/// Measurement is used for reporting a synchronous batch of metric values. -/// Instances of this type should be created by synchronous instruments (e.g., -/// `Counter::measurement`). -#[derive(Debug)] -pub struct Measurement { - number: Number, - instrument: Arc, -} - -impl Measurement { - /// Create a new measurement for an instrument - pub(crate) fn new(number: Number, instrument: Arc) -> Self { - Measurement { number, instrument } - } - - /// The number recorded by this measurement - pub fn number(&self) -> &Number { - &self.number - } - - /// Convert this measurement into the underlying number - pub fn into_number(self) -> Number { - self.number - } - - /// The instrument that recorded this measurement - pub fn instrument(&self) -> &Arc { - &self.instrument - } -} - -/// Wrapper around a sdk-implemented sync instrument for a given type -#[derive(Clone, Debug)] -pub(crate) struct SyncInstrument { - instrument: Arc, - _marker: marker::PhantomData, -} - -impl SyncInstrument { - /// Create a new sync instrument from an sdk-implemented sync instrument - pub(crate) fn new(instrument: Arc) -> Self { - SyncInstrument { - instrument, - _marker: marker::PhantomData, - } - } - - /// Create a new bound sync instrument - pub(crate) fn bind(&self, attributes: &[KeyValue]) -> SyncBoundInstrument { - let bound_instrument = self.instrument.bind(attributes); - SyncBoundInstrument { - bound_instrument, - _marker: marker::PhantomData, - } - } - - /// Record a value directly to the underlying instrument - pub(crate) fn direct_record(&self, number: Number, attributes: &[KeyValue]) { - self.instrument.record_one(number, attributes) - } - - /// Reference to the underlying sdk-implemented instrument - pub(crate) fn instrument(&self) -> &Arc { - &self.instrument - } -} - -/// Wrapper around a sdk-implemented sync bound instrument -#[derive(Clone, Debug)] -pub(crate) struct SyncBoundInstrument { - bound_instrument: Arc, - _marker: marker::PhantomData, -} - -impl SyncBoundInstrument { - /// Record a value directly to the underlying instrument - pub(crate) fn direct_record(&self, number: Number) { - self.bound_instrument.record_one(number) - } -} diff --git a/opentelemetry-sdk/src/metrics/sdk_api/wrap.rs b/opentelemetry-sdk/src/metrics/sdk_api/wrap.rs deleted file mode 100644 index 636e1579b5..0000000000 --- a/opentelemetry-sdk/src/metrics/sdk_api/wrap.rs +++ /dev/null @@ -1,319 +0,0 @@ -use crate::metrics::sdk_api::MeterCore; -use crate::metrics::sdk_api::{ - AsyncInstrumentCore, Descriptor, InstrumentKind, Number, NumberKind, SyncInstrumentCore, -}; -use opentelemetry_api::metrics::{ - AsyncCounter, AsyncUpDownCounter, ObservableUpDownCounter, SyncCounter, SyncHistogram, - SyncUpDownCounter, UpDownCounter, -}; -use opentelemetry_api::KeyValue; -use opentelemetry_api::{ - metrics::{ - AsyncGauge, Counter, Histogram, InstrumentProvider, Meter, ObservableCounter, - ObservableGauge, Result, Unit, - }, - Context, InstrumentationLibrary, -}; -use std::sync::Arc; - -/// wraps impl to be a full implementation of a Meter. -pub fn wrap_meter_core( - core: Arc, - library: InstrumentationLibrary, -) -> Meter { - Meter::new(library, Arc::new(MeterImpl(core))) -} - -struct MeterImpl(Arc); - -struct SyncInstrument(Arc); - -impl> SyncCounter for SyncInstrument { - fn add(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.record_one(cx, value.into(), attributes) - } -} - -impl> SyncUpDownCounter for SyncInstrument { - fn add(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.record_one(cx, value.into(), attributes) - } -} - -impl> SyncHistogram for SyncInstrument { - fn record(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.record_one(cx, value.into(), attributes) - } -} - -struct AsyncInstrument(Arc); - -impl> AsyncCounter for AsyncInstrument { - fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.observe_one(cx, value.into(), attributes) - } -} - -impl> AsyncUpDownCounter for AsyncInstrument { - fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.observe_one(cx, value.into(), attributes) - } -} - -impl> AsyncGauge for AsyncInstrument { - fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { - self.0.observe_one(cx, value.into(), attributes) - } -} - -impl InstrumentProvider for MeterImpl { - fn u64_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::Counter, - NumberKind::U64, - description, - unit, - ))?; - - Ok(Counter::new(Arc::new(SyncInstrument(instrument)))) - } - - fn f64_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::Counter, - NumberKind::F64, - description, - unit, - ))?; - - Ok(Counter::new(Arc::new(SyncInstrument(instrument)))) - } - - fn u64_observable_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::Counter, - NumberKind::U64, - description, - unit, - ))?; - - Ok(ObservableCounter::new(Arc::new(AsyncInstrument( - instrument, - )))) - } - - fn f64_observable_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::Counter, - NumberKind::F64, - description, - unit, - ))?; - - Ok(ObservableCounter::new(Arc::new(AsyncInstrument( - instrument, - )))) - } - - fn i64_up_down_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::UpDownCounter, - NumberKind::I64, - description, - unit, - ))?; - - Ok(UpDownCounter::new(Arc::new(SyncInstrument(instrument)))) - } - - fn f64_up_down_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::UpDownCounter, - NumberKind::F64, - description, - unit, - ))?; - - Ok(UpDownCounter::new(Arc::new(SyncInstrument(instrument)))) - } - - fn i64_observable_up_down_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::UpDownCounterObserver, - NumberKind::I64, - description, - unit, - ))?; - - Ok(ObservableUpDownCounter::new(Arc::new(AsyncInstrument( - instrument, - )))) - } - - fn f64_observable_up_down_counter( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::UpDownCounterObserver, - NumberKind::F64, - description, - unit, - ))?; - - Ok(ObservableUpDownCounter::new(Arc::new(AsyncInstrument( - instrument, - )))) - } - - fn u64_observable_gauge( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::GaugeObserver, - NumberKind::U64, - description, - unit, - ))?; - - Ok(ObservableGauge::new(Arc::new(AsyncInstrument(instrument)))) - } - - fn i64_observable_gauge( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::GaugeObserver, - NumberKind::I64, - description, - unit, - ))?; - - Ok(ObservableGauge::new(Arc::new(AsyncInstrument(instrument)))) - } - - fn f64_observable_gauge( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_async_instrument(Descriptor::new( - name, - InstrumentKind::GaugeObserver, - NumberKind::F64, - description, - unit, - ))?; - - Ok(ObservableGauge::new(Arc::new(AsyncInstrument(instrument)))) - } - - fn f64_histogram( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::Histogram, - NumberKind::F64, - description, - unit, - ))?; - - Ok(Histogram::new(Arc::new(SyncInstrument(instrument)))) - } - - fn u64_histogram( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::Histogram, - NumberKind::U64, - description, - unit, - ))?; - - Ok(Histogram::new(Arc::new(SyncInstrument(instrument)))) - } - - fn i64_histogram( - &self, - name: String, - description: Option, - unit: Option, - ) -> Result> { - let instrument = self.0.new_sync_instrument(Descriptor::new( - name, - InstrumentKind::Histogram, - NumberKind::I64, - description, - unit, - ))?; - - Ok(Histogram::new(Arc::new(SyncInstrument(instrument)))) - } - - fn register_callback(&self, callback: Box) -> Result<()> { - self.0.register_callback(callback) - } -} diff --git a/opentelemetry-sdk/src/metrics/selectors/mod.rs b/opentelemetry-sdk/src/metrics/selectors/mod.rs deleted file mode 100644 index 5aab06f72e..0000000000 --- a/opentelemetry-sdk/src/metrics/selectors/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -//! Aggregator Selectors -pub mod simple; diff --git a/opentelemetry-sdk/src/metrics/selectors/simple.rs b/opentelemetry-sdk/src/metrics/selectors/simple.rs deleted file mode 100644 index 550ce34643..0000000000 --- a/opentelemetry-sdk/src/metrics/selectors/simple.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Simple Metric Selectors -use crate::export::metrics::AggregatorSelector; -use crate::metrics::aggregators::{self, Aggregator}; -use crate::metrics::sdk_api::{Descriptor, InstrumentKind}; -use std::sync::Arc; - -/// This selector is faster and uses less memory than the others in this package. -pub fn inexpensive() -> impl AggregatorSelector { - InexpensiveSelector -} - -#[derive(Debug, Clone)] -struct InexpensiveSelector; - -impl AggregatorSelector for InexpensiveSelector { - fn aggregator_for(&self, descriptor: &Descriptor) -> Option> { - match descriptor.instrument_kind() { - InstrumentKind::GaugeObserver => Some(Arc::new(aggregators::last_value())), - _ => Some(Arc::new(aggregators::sum())), - } - } -} - -/// A simple aggregator selector that uses histogram aggregators for `Histogram` -/// instruments. -/// -/// This selector is a good default choice for most metric exporters. -pub fn histogram(boundaries: impl Into>) -> impl AggregatorSelector { - HistogramSelector(boundaries.into()) -} - -#[derive(Debug, Clone)] -struct HistogramSelector(Vec); - -impl AggregatorSelector for HistogramSelector { - fn aggregator_for(&self, descriptor: &Descriptor) -> Option> { - match descriptor.instrument_kind() { - InstrumentKind::GaugeObserver => Some(Arc::new(aggregators::last_value())), - InstrumentKind::Histogram => Some(Arc::new(aggregators::histogram(&self.0))), - _ => Some(Arc::new(aggregators::sum())), - } - } -} diff --git a/opentelemetry-sdk/src/metrics/view.rs b/opentelemetry-sdk/src/metrics/view.rs new file mode 100644 index 0000000000..156989eab7 --- /dev/null +++ b/opentelemetry-sdk/src/metrics/view.rs @@ -0,0 +1,176 @@ +use opentelemetry_api::{ + global, + metrics::{MetricsError, Result}, +}; +use regex::Regex; + +use super::instrument::{Instrument, Stream}; + +fn empty_view(_inst: &Instrument) -> Option { + None +} + +/// Used to customize the metrics that are output by the SDK. +/// +/// Here are some examples when a [View] might be needed: +/// +/// * Customize which Instruments are to be processed/ignored. For example, an +/// instrumented library can provide both temperature and humidity, but the +/// application developer might only want temperature. +/// * Customize the aggregation - if the default aggregation associated with the +/// [Instrument] does not meet the needs of the user. For example, an HTTP client +/// library might expose HTTP client request duration as Histogram by default, +/// but the application developer might only want the total count of outgoing +/// requests. +/// * Customize which attribute(s) are to be reported on metrics. For example, +/// an HTTP server library might expose HTTP verb (e.g. GET, POST) and HTTP +/// status code (e.g. 200, 301, 404). The application developer might only care +/// about HTTP status code (e.g. reporting the total count of HTTP requests for +/// each HTTP status code). There could also be extreme scenarios in which the +/// application developer does not need any attributes (e.g. just get the total +/// count of all incoming requests). +/// +/// # Example Custom View +/// +/// View is implemented for all `Fn(&Instrument) -> Option`. +/// +/// ``` +/// use opentelemetry_sdk::metrics::{Instrument, MeterProvider, Stream}; +/// +/// // return streams for the given instrument +/// let my_view = |i: &Instrument| { +/// // return Some(Stream) or +/// None +/// }; +/// +/// let provider = MeterProvider::builder().with_view(my_view).build(); +/// # drop(provider) +/// ``` +pub trait View: Send + Sync + 'static { + /// Defines how data should be collected for certain instruments. + /// + /// Return `true` and the exact [Stream] to use for matching [Instrument]s, + /// otherwise if there is no match, return `false`. + fn match_inst(&self, inst: &Instrument) -> Option; +} + +impl View for T +where + T: Fn(&Instrument) -> Option + Send + Sync + 'static, +{ + fn match_inst(&self, inst: &Instrument) -> Option { + self(inst) + } +} + +impl View for Box { + fn match_inst(&self, inst: &Instrument) -> Option { + (**self).match_inst(inst) + } +} + +/// Creates a [View] that applies the [Stream] mask for all instruments that +/// match criteria. +/// +/// The returned [View] will only apply the mask if all non-empty fields of +/// criteria match the corresponding [Instrument] passed to the view. If all +/// fields of the criteria are their default values, a view that matches no +/// instruments is returned. If you need to match an empty-value field, create a +/// [View] directly. +/// +/// The [Instrument::name] field of criteria supports wildcard pattern matching. +/// The wildcard `*` is recognized as matching zero or more characters, and `?` +/// is recognized as matching exactly one character. For example, a pattern of +/// `*` will match all instrument names. +/// +/// The [Stream] mask only applies updates for non-empty fields. By default, the +/// [Instrument] the [View] matches against will be use for the name, +/// description, and unit of the returned [Stream] and no `aggregation` or +/// `attribute_filter` are set. All non-empty fields of mask are used instead of +/// the default. If you need to set a an empty value in the returned stream, +/// create a custom [View] directly. +/// +/// # Example +/// +/// ``` +/// use opentelemetry_sdk::metrics::{new_view, Aggregation, Instrument, Stream}; +/// +/// let criteria = Instrument::new().name("counter_*"); +/// let mask = Stream::new().aggregation(Aggregation::Sum); +/// +/// let view = new_view(criteria, mask); +/// # drop(view); +/// ``` +pub fn new_view(criteria: Instrument, mask: Stream) -> Result> { + if criteria.is_empty() { + return Ok(Box::new(empty_view)); + } + let contains_wildcard = criteria.name.contains(|c| c == '*' || c == '?'); + let err_msg_criteria = criteria.clone(); + + let match_fn: Box bool + Send + Sync> = if contains_wildcard { + if mask.name != "" { + global::handle_error(MetricsError::Config(format!( + "name replacement for multiple instruments, dropping view, criteria: {criteria:?}, mask: {mask:?}" + ))); + return Ok(Box::new(empty_view)); + } + + let pattern = criteria + .name + .trim_start_matches('^') + .trim_end_matches('$') + .replace('?', ".") + .replace('*', ".*"); + let re = + Regex::new(&format!("^{pattern}$")).map_err(|e| MetricsError::Config(e.to_string()))?; + Box::new(move |i| { + re.is_match(&i.name) + && criteria.matches_description(i) + && criteria.matches_kind(i) + && criteria.matches_unit(i) + && criteria.matches_scope(i) + }) + } else { + Box::new(move |i| criteria.matches(i)) + }; + + let mut agg = None; + if let Some(ma) = &mask.aggregation { + match ma.validate() { + Ok(_) => agg = Some(ma.clone()), + Err(err) => { + global::handle_error(MetricsError::Other(format!( + "{}, not using aggregation with view. criteria: {:?}, mask: {:?}", + err, err_msg_criteria, mask + ))); + } + } + } + + Ok(Box::new(move |i: &Instrument| -> Option { + if match_fn(i) { + Some(Stream { + name: if !mask.name.is_empty() { + mask.name.clone() + } else { + i.name.clone() + }, + description: if !mask.description.is_empty() { + mask.description.clone() + } else { + i.description.clone() + }, + unit: if !mask.unit.as_str().is_empty() { + mask.unit.clone() + } else { + i.unit.clone() + }, + aggregation: agg.clone(), + attribute_filter: mask.attribute_filter.clone(), + }) + } else { + None + } + })) +} diff --git a/opentelemetry-sdk/src/resource/env.rs b/opentelemetry-sdk/src/resource/env.rs index ab1a958477..0efee4c7e0 100644 --- a/opentelemetry-sdk/src/resource/env.rs +++ b/opentelemetry-sdk/src/resource/env.rs @@ -88,9 +88,10 @@ impl ResourceDetector for SdkProvidedResourceDetector { .get(Key::new("service.name")) .map(|v| v.to_string()) .filter(|s| !s.is_empty()) - .unwrap_or_else(|| match option_env!("CARGO_BIN_NAME") { - Some(s) => s.to_string(), - None => "unknown_service".to_string(), + .unwrap_or_else(|| { + option_env!("CARGO_BIN_NAME") + .unwrap_or("unknown_service") + .into() }) }), )]) diff --git a/opentelemetry-sdk/src/resource/mod.rs b/opentelemetry-sdk/src/resource/mod.rs index b83450a559..4faaf8f731 100644 --- a/opentelemetry-sdk/src/resource/mod.rs +++ b/opentelemetry-sdk/src/resource/mod.rs @@ -30,8 +30,6 @@ pub use os::OsResourceDetector; pub use process::ProcessResourceDetector; pub use telemetry::TelemetryResourceDetector; -#[cfg(feature = "metrics")] -use opentelemetry_api::attributes; use opentelemetry_api::{Key, KeyValue, Value}; use std::borrow::Cow; use std::collections::{hash_map, HashMap}; @@ -191,13 +189,6 @@ impl Resource { pub fn get(&self, key: Key) -> Option { self.attrs.get(&key).cloned() } - - /// Encoded attributes - #[cfg(feature = "metrics")] - #[cfg_attr(docsrs, doc(cfg(feature = "metrics")))] - pub fn encoded(&self, encoder: &dyn attributes::Encoder) -> String { - encoder.encode(&mut self.into_iter()) - } } /// An owned iterator over the entries of a `Resource`. diff --git a/opentelemetry-sdk/src/testing/metric.rs b/opentelemetry-sdk/src/testing/metric.rs deleted file mode 100644 index 37e526bfbf..0000000000 --- a/opentelemetry-sdk/src/testing/metric.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::sync::Arc; - -use opentelemetry_api::metrics::Result; - -use crate::{ - export::metrics::{AggregatorSelector, Checkpointer, LockedCheckpointer, Processor}, - metrics::{aggregators::Aggregator, sdk_api::Descriptor}, -}; - -#[derive(Debug)] -struct NoopAggregatorSelector; - -impl AggregatorSelector for NoopAggregatorSelector { - fn aggregator_for( - &self, - _descriptor: &Descriptor, - ) -> Option> { - None - } -} - -#[derive(Debug)] -pub struct NoopCheckpointer; - -impl Processor for NoopCheckpointer { - fn aggregator_selector(&self) -> &dyn AggregatorSelector { - &NoopAggregatorSelector - } -} - -impl Checkpointer for NoopCheckpointer { - fn checkpoint( - &self, - _f: &mut dyn FnMut(&mut dyn LockedCheckpointer) -> Result<()>, - ) -> Result<()> { - Ok(()) - } -} diff --git a/opentelemetry-sdk/src/testing/mod.rs b/opentelemetry-sdk/src/testing/mod.rs index 3cfc7e3850..ef7f16bad8 100644 --- a/opentelemetry-sdk/src/testing/mod.rs +++ b/opentelemetry-sdk/src/testing/mod.rs @@ -1,2 +1,2 @@ -pub mod metric; +#[cfg(all(feature = "testing", feature = "trace"))] pub mod trace; diff --git a/opentelemetry-sdk/src/testing/trace.rs b/opentelemetry-sdk/src/testing/trace.rs index 20a582cc6d..b8d6f42507 100644 --- a/opentelemetry-sdk/src/testing/trace.rs +++ b/opentelemetry-sdk/src/testing/trace.rs @@ -131,6 +131,7 @@ impl Display for TestExportError { } } +#[cfg(any(feature = "rt-tokio", feature = "rt-tokio-current-thread"))] impl From> for TestExportError { fn from(err: tokio::sync::mpsc::error::SendError) -> Self { TestExportError(err.to_string()) diff --git a/opentelemetry-sdk/tests/metrics.rs b/opentelemetry-sdk/tests/metrics.rs deleted file mode 100644 index 0bfc86abb2..0000000000 --- a/opentelemetry-sdk/tests/metrics.rs +++ /dev/null @@ -1,203 +0,0 @@ -#[cfg(test)] -#[cfg(feature = "metrics")] -mod metrics { - use std::time::Duration; - - use opentelemetry_api::metrics::{Counter, MeterProvider, UpDownCounter}; - use opentelemetry_api::Context; - use opentelemetry_sdk::export::metrics::aggregation::{ - cumulative_temporality_selector, delta_temporality_selector, LastValue, Sum, - TemporalitySelector, - }; - use opentelemetry_sdk::export::metrics::InstrumentationLibraryReader; - use opentelemetry_sdk::metrics::aggregators::{LastValueAggregator, SumAggregator}; - use opentelemetry_sdk::metrics::controllers::BasicController; - use opentelemetry_sdk::metrics::sdk_api::NumberKind; - use opentelemetry_sdk::metrics::{controllers, processors, selectors}; - - #[test] - fn test_temporality() { - struct TestSuite - where - T: TemporalitySelector + Clone + Send + Sync + 'static, - F: Fn() -> T, - { - temporality: F, - controller: BasicController, - context: Context, - - gauge_tx: crossbeam_channel::Sender, - counter: Counter, - up_down_counter: UpDownCounter, - - results: Vec>, - } - - impl TestSuite - where - F: Fn() -> T, - T: TemporalitySelector + Clone + Send + Sync + 'static, - { - fn setup(f: F) -> Self { - let controller = controllers::basic(processors::factory( - selectors::simple::inexpensive(), // basically give us Sum aggregation except for gauge, which is LastValue aggregation - f(), - )) - .with_collect_period(Duration::ZERO) // require manual collection - .build(); - let meter = controller.versioned_meter("test", None, None); - let (gauge_tx, gauge_rx) = crossbeam_channel::bounded(10); - let gauge = meter.i64_observable_gauge("gauge").init(); - meter - .register_callback(move |cx| { - if let Ok(val) = gauge_rx.try_recv() { - gauge.observe(cx, val, &[]); - } - }) - .expect("failed to register callback"); - TestSuite { - controller, - temporality: f, - counter: meter.f64_counter("counter").init(), - up_down_counter: meter.f64_up_down_counter("up_down_counter").init(), - context: Context::new(), - gauge_tx, - results: Vec::new(), - } - } - - fn add_counter(&mut self, val: f64) { - self.counter.add(&self.context, val, &[]); - } - - fn change_up_down_counter(&mut self, val: f64) { - self.up_down_counter.add(&self.context, val, &[]); - } - - fn change_gauge(&mut self, val: i64) { - self.gauge_tx.send(val).unwrap(); - } - - fn collect_and_save(&mut self) { - self.controller.collect(&self.context).unwrap(); - - let temporality = (self.temporality)(); - let mut result_per_round = Vec::new(); - self.controller - .try_for_each(&mut |_library, reader| { - reader.try_for_each(&temporality, &mut |record| { - if let Some(sum_agg) = record - .aggregator() - .unwrap() - .as_any() - .downcast_ref::() - { - result_per_round.push(( - record.descriptor().name().to_owned(), - sum_agg.sum().unwrap().to_f64(&NumberKind::F64), - )); - } - - if let Some(last_value_agg) = record - .aggregator() - .unwrap() - .as_any() - .downcast_ref::() - { - result_per_round.push(( - record.descriptor().name().to_owned(), - last_value_agg - .last_value() - .unwrap() - .0 - .to_f64(&NumberKind::I64), - )) - } - - Ok(()) - })?; - Ok(()) - }) - .expect("no error expected"); - // sort result per round stablely so we have deterministic results - result_per_round.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); - self.results.push(result_per_round); - } - - fn expect(self, expected: Vec, f64)>>) { - assert_eq!( - self.results, - expected - .into_iter() - .map(|v| v - .into_iter() - .map(|(k, v)| (k.into(), v)) - .collect::>()) - .collect::>() - ); - } - } - - let mut cumulative = TestSuite::setup(cumulative_temporality_selector); - // round 1 - cumulative.add_counter(10.0); - cumulative.add_counter(5.3); - cumulative.collect_and_save(); - // round 2 - cumulative.collect_and_save(); - // round 3 - cumulative.add_counter(0.0); - cumulative.change_up_down_counter(-1.0); - cumulative.change_gauge(-1); - cumulative.collect_and_save(); - // round 4 - cumulative.change_up_down_counter(1.0); - cumulative.add_counter(10.0); - cumulative.collect_and_save(); - // round 5 - cumulative.change_gauge(1); - cumulative.collect_and_save(); - // assert - cumulative.expect(vec![ - vec![("counter", 15.3)], - vec![("counter", 15.3)], - vec![ - ("counter", 15.3), - ("gauge", -1.0), - ("up_down_counter", -1.0), - ], - vec![("counter", 25.3), ("gauge", -1.0), ("up_down_counter", 0.0)], - vec![("counter", 25.3), ("gauge", 1.0), ("up_down_counter", 0.0)], - ]); - - let mut delta = TestSuite::setup(delta_temporality_selector); - // round 1 - delta.add_counter(10.0); - delta.add_counter(5.3); - delta.collect_and_save(); - // round 2 - delta.collect_and_save(); - // round 3 - delta.add_counter(10.0); - delta.collect_and_save(); - // round 4 - delta.add_counter(0.0); - delta.collect_and_save(); - // round 5 - delta.change_up_down_counter(-1.0); - delta.change_gauge(-1); - delta.collect_and_save(); - // round 6 - delta.change_up_down_counter(1.0); - delta.collect_and_save(); - // assert - delta.expect(vec![ - vec![("counter", 15.3)], - vec![], // no change and no data exported - vec![("counter", 10.0)], - vec![("counter", 0.0)], - vec![("gauge", -1.0), ("up_down_counter", -1.0)], - vec![("up_down_counter", 1.0)], - ]) - } -}