diff --git a/.vscode/settings.json b/.vscode/settings.json index 869dfb9e9..1aafb6542 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,11 +14,13 @@ "asciicast", "attribs", "Bderivebuilder", + "Bincode", "BITALIC", "bitflags", "Blackbox", "BOOKM", "boop", + "CBOR", "chrono", "cicd", "CLICOLOR", diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d6448f54..ca8bd9ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -680,6 +680,14 @@ reflect how the functionality is used in the real world. place is because it was easier to develop them there, but since then, lots of other consumers of this functionality have emerged, including crates that are created by "3rd party developers" (people not R3BL and not part of `r3bl-open-core` repo). + - Move the `kv.rs` module into `storage` from the + `nazmulidris/rust-scratch/tcp-api-server` repo. This provides an in-memory / + in-process key value store that is built on top of `sled`. This eliminates the need to + use files to save / load data. + - Move the `miette_setup_global_report_handler.rs` from the + `nazmulidris/rust-scratch/tcp-api-server` repo. This allows customization of the + miette global report handler at the process level. Useful for apps that need to + override the default report handler formatting. ### v0.9.16 (2024-09-12) @@ -914,8 +922,8 @@ modules in this crate. - Deleted: - Move the Jaeger tracing module to the `tcp-api-server` crate in the - `https://github.comnazmulidris/rust-scratch/` repo. This wasn't really used - anywhere else. + `https://github.comnazmulidris/rust-scratch/` repo. This wasn't really used anywhere + else. Also remove all the OpenTelemetry related dependencies from this crate. - Move the tracing module into the `core` folder (`r3bl_rs_utils_core` crate) in the mono repo. diff --git a/Cargo.lock b/Cargo.lock index 549eae40b..f07c6d2ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,12 +99,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "anyhow" -version = "1.0.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" - [[package]] name = "assert_cmd" version = "2.0.16" @@ -166,51 +160,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 0.1.2", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - [[package]] name = "backtrace" version = "0.3.74" @@ -235,12 +184,6 @@ dependencies = [ "backtrace", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -480,6 +423,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -496,7 +448,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio 0.8.11", - "parking_lot", + "parking_lot 0.12.3", "serde", "signal-hook", "signal-hook-mio", @@ -513,7 +465,7 @@ dependencies = [ "crossterm_winapi", "futures-core", "mio 1.0.2", - "parking_lot", + "parking_lot 0.12.3", "rustix", "signal-hook", "signal-hook-mio", @@ -709,6 +661,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.30" @@ -798,6 +760,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -835,31 +806,6 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.5.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.6" @@ -871,20 +817,14 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", - "indexmap 2.5.0", + "http", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.5" @@ -912,17 +852,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.1.0" @@ -934,17 +863,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -952,7 +870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http", ] [[package]] @@ -963,8 +881,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -974,36 +892,6 @@ version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.4.1" @@ -1013,9 +901,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", - "http 1.1.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", "itoa", "pin-project-lite", @@ -1031,8 +919,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.4.1", + "http", + "hyper", "hyper-util", "rustls", "rustls-pki-types", @@ -1041,18 +929,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper 0.14.30", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "hyper-tls" version = "0.6.0" @@ -1061,7 +937,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.4.1", + "hyper", "hyper-util", "native-tls", "tokio", @@ -1078,9 +954,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "hyper 1.4.1", + "http", + "http-body", + "hyper", "pin-project-lite", "socket2", "tokio", @@ -1124,22 +1000,21 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ - "autocfg", - "hashbrown 0.12.3", + "equivalent", + "hashbrown", ] [[package]] -name = "indexmap" -version = "2.5.0" +name = "instant" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "equivalent", - "hashbrown 0.14.5", + "cfg-if", ] [[package]] @@ -1203,6 +1078,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kv" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "620727085ac39ee9650b373fe6d8073a0aee6f99e52a9c72b25f7671078039ab" +dependencies = [ + "bincode", + "pin-project-lite", + "serde", + "serde_json", + "sled", + "thiserror", + "toml", +] + [[package]] name = "lazy-bytes-cast" version = "5.0.1" @@ -1278,12 +1168,6 @@ dependencies = [ "libc", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "memchr" version = "2.7.4" @@ -1579,95 +1463,12 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "opentelemetry" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900d57987be3f2aeb70d385fff9b27fb74c5723cc9a52d904d4f9c807a0667bf" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - -[[package]] -name = "opentelemetry-otlp" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb" -dependencies = [ - "async-trait", - "futures-core", - "http 0.2.12", - "opentelemetry", - "opentelemetry-proto", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", - "prost", - "thiserror", - "tokio", - "tonic", -] - -[[package]] -name = "opentelemetry-proto" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4" -dependencies = [ - "opentelemetry", - "opentelemetry_sdk", - "prost", - "tonic", -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9ab5bd6c42fb9349dcf28af2ba9a0667f697f9bdcca045d39f2cec5543e2910" - -[[package]] -name = "opentelemetry_sdk" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e90c7113be649e31e9a0f8b5ee24ed7a16923b322c3c5ab6367469c049d6b7e" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "once_cell", - "opentelemetry", - "ordered-float", - "percent-encoding", - "rand", - "thiserror", - "tokio", - "tokio-stream", -] - [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-float" -version = "4.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" -dependencies = [ - "num-traits", -] - [[package]] name = "overload" version = "0.1.1" @@ -1680,6 +1481,17 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -1687,7 +1499,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -1698,7 +1524,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.4", "smallvec", "windows-targets 0.52.6", ] @@ -1807,8 +1633,8 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42cf17e9a1800f5f396bc67d193dc9411b59012a5876445ef450d449881e1016" dependencies = [ - "base64 0.22.1", - "indexmap 2.5.0", + "base64", + "indexmap", "quick-xml", "serde", "time", @@ -1875,29 +1701,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.77", -] - [[package]] name = "quick-xml" version = "0.32.0" @@ -1993,10 +1796,12 @@ name = "r3bl_rs_utils_core" version = "0.9.16" dependencies = [ "assert_cmd", + "bincode", "chrono", "colorgrad", "crossterm 0.28.1", "futures-util", + "kv", "log", "miette", "nom", @@ -2010,6 +1815,8 @@ dependencies = [ "sha2", "size-of", "strip-ansi", + "strum", + "strum_macros", "tempfile", "thiserror", "time", @@ -2053,10 +1860,6 @@ dependencies = [ "futures-core", "futures-util", "miette", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", "pretty_assertions", "r3bl_rs_utils_core", "r3bl_test_fixtures", @@ -2069,7 +1872,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-core", - "tracing-opentelemetry", "tracing-subscriber", "unicode-segmentation", "unicode-width", @@ -2168,6 +1970,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.4" @@ -2243,16 +2054,16 @@ version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.4.6", - "http 1.1.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.4.1", + "hyper", "hyper-rustls", "hyper-tls", "hyper-util", @@ -2268,7 +2079,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -2333,7 +2144,7 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64 0.22.1", + "base64", "rustls-pki-types", ] @@ -2487,7 +2298,7 @@ dependencies = [ "futures", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "scc", "serial_test_derive", ] @@ -2595,6 +2406,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -2743,12 +2570,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.1" @@ -2969,7 +2790,7 @@ dependencies = [ "bytes", "libc", "mio 1.0.2", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2978,16 +2799,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.4.0" @@ -3020,17 +2831,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.12" @@ -3045,30 +2845,12 @@ dependencies = [ ] [[package]] -name = "tonic" -version = "0.11.0" +name = "toml" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.21.7", - "bytes", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", + "serde", ] [[package]] @@ -3079,16 +2861,11 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", - "slab", "tokio", - "tokio-util", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -3158,24 +2935,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9be14ba1bbe4ab79e9229f7f89fab8d120b865859f10527f31c033e599d2284" -dependencies = [ - "js-sys", - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time", -] - [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -3258,12 +3017,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf8parse" version = "0.2.2" @@ -3514,16 +3267,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "which" version = "4.4.2" diff --git a/core/Cargo.toml b/core/Cargo.toml index b665b49d4..4a3fe3c63 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -75,6 +75,17 @@ thiserror = "1.0.63" miette = { version = "7.2.0", features = ["fancy"] } pretty_assertions = "1.4.0" +# Enum to string generation. +strum = "0.26.3" +strum_macros = "0.26.4" + +# Convert Rust structs to and from binary representation. +bincode = { version = "1.3.3" } + +# Key Value store that can be used to store JSON or bincode encoded values. `kv` is built +# on top of `sled` which currently does not support access across multiple processes. +kv = { version = "0.24.0", features = ["json-value", "bincode-value"] } + [dev-dependencies] # for assert_eq! macro pretty_assertions = "1.4.0" diff --git a/core/src/bin/tracing_test_bin.rs b/core/src/bin/tracing_test_bin.rs index 07f49f05b..0f17fdf26 100644 --- a/core/src/bin/tracing_test_bin.rs +++ b/core/src/bin/tracing_test_bin.rs @@ -15,7 +15,10 @@ * limitations under the License. */ -use r3bl_rs_utils_core::{init_tracing, DisplayPreference, TracingConfig, WriterConfig}; +use r3bl_rs_utils_core::{init_tracing_thread_local, + DisplayPreference, + TracingConfig, + WriterConfig}; /// `assert_cmd` : /// @@ -34,7 +37,7 @@ fn main() { }; // Create a new tracing layer with stdout. - init_tracing(TracingConfig { + let default_guard = init_tracing_thread_local(TracingConfig { writer_config: WriterConfig::Display(display_preference), level: tracing::Level::DEBUG, }) @@ -46,4 +49,6 @@ fn main() { tracing::info!("info"); tracing::debug!("debug"); tracing::trace!("trace"); + + drop(default_guard); } diff --git a/core/src/common/miette_setup_global_report_handler.rs b/core/src/common/miette_setup_global_report_handler.rs new file mode 100644 index 000000000..32c0f724b --- /dev/null +++ b/core/src/common/miette_setup_global_report_handler.rs @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 Nazmul Idris + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! This module is standalone, you can use it any project that uses +//! [miette](https://docs.rs/miette/latest/miette/index.html) for error handling. +//! +//! If you want to change the default global report handler you can do the following: +//! 1. Customize the default global implementation of the `ReportHandler` trait. (Easy). +//! 2. Register a custom error report handler of your own. (Difficult). +//! +//! Background information on miette's architecture: +//! - Miette allows customization how the report is +//! [`Report`](https://docs.rs/miette/latest/miette/struct.Report.html) displayed to +//! terminal output (stdout, stderr), when the global hook is activated, due to a +//! program "erroring out" or "crashing", when the top-level miette handler in `main() +//! -> miette::Result<_>` is activated. This hook is only activated at the time that the +//! error is displayed to terminal output, not when it is registered, it is lazy. So it +//! is possible to detect the terminal width just before the output is generated to the +//! terminal output (stdout, stderr). +//! - The global default implementation of the [`ReportHandler` +//! trait](https://docs.rs/miette/latest/miette/trait.ReportHandler.html) is done by +//! [`MietteHandler` +//! struct](https://docs.rs/miette/latest/miette/struct.MietteHandler.html). +//! - Using the [`MietteHandlerOpts` +//! struct](https://docs.rs/miette/latest/miette/struct.MietteHandlerOpts.html) you can +//! configure the default `MietteHandler`. Under the hood, `build()` produces a +//! [`GraphicalReportHandler` +//! struct](https://docs.rs/miette/latest/miette/struct.GraphicalReportHandler.html) +//! which is the "real" handler struct. +//! - The [miette::set_hook] function is used to register a custom error report handler. Here's +//! an [example in a +//! test](https://github.com/zkat/miette/blob/6ea86a2248854acf88df345814b6c97d31b8b4d9/tests/test_location.rs#L39) +//! which registers a custom hook / report handler. + +use miette::MietteHandlerOpts; +use tracing::debug; + +/// The [miette::ErrorHook] is lazily evaluated, so the terminal width will be calculated +/// just at the time of the global error handler being used. So if an error never occurs, +/// then the terminal width will never be calculated. This is the desired behavior. +pub fn setup_default_miette_global_report_handler(issues_url: &'static str) { + miette::set_hook(Box::new(|_report| { + let terminal_width = { + let it = crossterm::terminal::size() + .map(|(columns, _rows)| columns) + .unwrap_or(80) as usize; + debug!("miette::set_hook -> terminal_width: {}", it); + it + }; + Box::new( + MietteHandlerOpts::new() + .width(terminal_width) + .wrap_lines(true) + .force_graphical(true) + .rgb_colors(miette::RgbColors::Always) + .terminal_links(true) + .unicode(true) + .context_lines(3) + .tab_width(4) + .break_words(true) + .with_cause_chain() + .footer(issues_url.to_string()) + .build(), + ) + })) + .ok(); +} diff --git a/core/src/common/mod.rs b/core/src/common/mod.rs index 434f8e682..cd73afe8e 100644 --- a/core/src/common/mod.rs +++ b/core/src/common/mod.rs @@ -19,8 +19,10 @@ pub mod common_enums; pub mod common_math; pub mod common_result_and_error; +pub mod miette_setup_global_report_handler; // Re-export. pub use common_enums::*; pub use common_math::*; pub use common_result_and_error::*; +pub use miette_setup_global_report_handler::*; diff --git a/core/src/lib.rs b/core/src/lib.rs index 599a7877d..5acb30804 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -134,6 +134,7 @@ pub mod misc; pub mod terminal_io; pub mod tracing_logging; pub mod tui_core; +pub mod storage; // Re-export. pub use common::*; @@ -143,3 +144,4 @@ pub use misc::*; pub use terminal_io::*; pub use tracing_logging::*; pub use tui_core::*; +pub use storage::*; \ No newline at end of file diff --git a/core/src/storage/kv.rs b/core/src/storage/kv.rs new file mode 100644 index 000000000..4db5f3aee --- /dev/null +++ b/core/src/storage/kv.rs @@ -0,0 +1,587 @@ +/* + * Copyright (c) 2024 Nazmul Idris + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Use module is standalone, you can use it in any project that needs to create an +//! embedded key/value store that stores keys that are of whatever type you choose, and +//! values that are whatever type you choose. +//! - It is a wrapper around the [kv] crate, to make it trivially simple to use. There are +//! only 4 functions that allow you access to the capabilities of the key/value embedded +//! store. +//! - [load_or_create_store] +//! - [load_or_create_bucket_from_store] +//! - [insert_into_bucket] +//! - [get_from_bucket] +//! - [remove_from_bucket] +//! - [is_key_contained_in_bucket] +//! - And provide lots of really fine grained errors, using [miette] and [thiserror] (see +//! [kv_error]). +//! +//! 1. The values are serialized to [Bincode] (from Rust struct) before they are saved. +//! 2. The values are deserialized from [Bincode] (to Rust struct) after they are loaded. +//! +//! See the tests in this module for an example of how to use this module. +//! +//! [Bincode] is like [`CBOR`](https://en.wikipedia.org/wiki/CBOR), except that it isn't +//! standards based, but it is faster. It also has full support of [serde] just like [kv] +//! does. +//! - [More info comparing [`CBOR`](https://en.wikipedia.org/wiki/CBOR) with +//! [`Bincode`](https://gemini.google.com/share/0684553f3d57) +//! +//! The [kv] crate works really well, even with multiple processes accessing the same +//! database on disk. Even though [sled](https://github.com/spacejam/sled), which the [kv] +//! crate itself wraps, is not multi-process safe. +//! +//! In my testing, I've run multiple processes that write to the key/value store at the +//! same time, and it works as expected. Even with multiple processes writing to the +//! store, the iterator [Bucket::iter] can be used to read the current state of the db, as +//! expected. + +use std::fmt::{Debug, Display}; + +use crossterm::style::Stylize; +use kv::*; +use miette::{Context, IntoDiagnostic}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, instrument}; + +/// Convenience type alias for the [kv::Bucket] type. +/// 1. A [Bucket] is created from a [Store]. +/// 2. A [Bucket] is given a name, and there may be many [Bucket]s in a [Store]. +/// 3. A [Bucket] provides typed access to a section of the key/value store [kv]. +/// +/// The [Bucket] stores the following key/value pairs. +/// - `KeyT`: The generic type ``. This will not be serialized or deserialized. This +/// also has a trait bound on [Key]. See [insert_into_bucket] for an example of this. +/// - `ValueT`: This type makes it concrete that [Bincode] will be used to serialize and +/// deserialize the data from the generic type ``, which has trait bounds on +/// [Serialize], [Deserialize]. See [insert_into_bucket] for an example of this. +pub type KVBucket<'a, KeyT, ValueT> = kv::Bucket<'a, KeyT, Bincode>; + +mod default_settings { + use super::*; + + #[derive(Debug, strum_macros::EnumString, Hash, PartialEq, Eq, Clone, Copy)] + pub enum Keys { + /// Your [Store] folder path name. [kv] uses this folder to save your key/value store. + /// It is your database persistence folder. + StoreFolderPath, + /// Your [Bucket] name that is used to store the key/value pairs. + /// - [Bincode] is used to serialize/deserialize the value stored in the key/value + /// pair. + /// - A [Bucket] provides typed access to a section of the key/value store [kv]. + BucketName, + } + + pub fn get(key: Keys) -> String { + match key { + Keys::StoreFolderPath => "kv_folder".to_string(), + Keys::BucketName => "my_bucket".to_string(), + } + } +} + +/// Create the db folder if it doesn't exit. Otherwise load it from the folder on disk. +/// Note there are no lifetime annotations on this function. All the other functions below +/// do have lifetime annotations, since they are all tied to the lifetime of the returned +/// [Store]. +#[instrument] +pub fn load_or_create_store( + maybe_db_folder_path: Option<&String>, +) -> miette::Result { + // Configure the database folder location. + let db_folder_path = maybe_db_folder_path.cloned().unwrap_or_else(|| { + default_settings::get(default_settings::Keys::StoreFolderPath) + }); + + let cfg = Config::new(db_folder_path.clone()); + + // Open the key/store store using the Config. + let store = + Store::new(cfg) + .into_diagnostic() + .wrap_err(KvErrorCouldNot::CreateDbFolder { + db_folder_path: db_folder_path.clone(), + })?; + + debug!( + "๐Ÿ“‘ {}", + format!( + "{}{}", + "load or create a store: ", + /*.blue() */ db_folder_path /*.bold().cyan() */ + ) + ); + + Ok(store) +} + +/// A [Bucket] provides typed access to a section of the key/value [Store]. It has a +/// lifetime, since the [Bucket] is created from a [Store]. +#[instrument(fields(store = ?store.path(), buckets = ?store.buckets()))] +pub fn load_or_create_bucket_from_store< + 'a, + KeyT: for<'k> kv::Key<'k>, + ValueT: Serialize + for<'d> Deserialize<'d>, +>( + store: &Store, + maybe_bucket_name: Option<&String>, +) -> miette::Result> { + let bucket_name = maybe_bucket_name + .cloned() + .unwrap_or_else(|| default_settings::get(default_settings::Keys::BucketName)); + + let my_payload_bucket: KVBucket = store + .bucket(Some(&bucket_name)) + .into_diagnostic() + .wrap_err(KvErrorCouldNot::CreateBucketFromStore { + bucket_name: bucket_name.clone(), + })?; + + debug!( + "๐Ÿ“ฆ {}", + format!( + "{}{}", + "Load or create bucket from store, and instantiate: ", /*.blue() */ + bucket_name, /*.bold().cyan() */ + ) + ); + + Ok(my_payload_bucket) +} + +/// The value is serialized using [Bincode] prior to saving it to the key/value store. +#[instrument(skip(bucket))] +pub fn insert_into_bucket< + 'a, + KeyT: Debug + Display + for<'k> kv::Key<'k>, + ValueT: Debug + Serialize + for<'d> Deserialize<'d>, +>( + bucket: &'a KVBucket<'a, KeyT, ValueT>, + key: KeyT, + value: ValueT, +) -> miette::Result<()> { + let value_str = format!("{:?}", value).bold().cyan(); + + // Serialize the Rust struct into a binary payload. + bucket + .set(&key, &Bincode(value)) + .into_diagnostic() + .wrap_err(KvErrorCouldNot::SaveKeyValuePairToBucket)?; + + debug!( + "๐Ÿ”ฝ {}", + format!( + "{}: {}: {}", + "Save key / value pair to bucket", /*.green() */ + key.to_string(), /*.bold().cyan() */ + value_str + ) + ); + + Ok(()) +} + +/// The value in the key/value store is serialized using [Bincode]. Upon loading that +/// value it is deserialized and returned by this function. +#[instrument(skip(bucket))] +pub fn get_from_bucket< + 'a, + KeyT: Debug + Display + for<'k> kv::Key<'k>, + ValueT: Debug + Serialize + for<'d> Deserialize<'d>, +>( + bucket: &KVBucket<'a, KeyT, ValueT>, + key: KeyT, +) -> miette::Result> { + let maybe_value: Option> = bucket + .get(&key) + .into_diagnostic() + .wrap_err(KvErrorCouldNot::LoadKeyValuePairFromBucket)?; + + let it = match maybe_value { + // Deserialize the binary payload into a Rust struct. + Some(Bincode(payload)) => Ok(Some(payload)), + _ => Ok(None), + }; + + debug!( + "๐Ÿ”ผ {}", + format!( + "{}: {}: {}", + "Load key / value pair from bucket", /*.green() */ + key.to_string(), /*.bold().cyan() */ + format!("{:?}", it) /*.bold().cyan() */ + ) + ); + + it +} + +#[instrument(skip(bucket))] +pub fn remove_from_bucket< + 'a, + KeyT: Debug + Display + for<'k> kv::Key<'k>, + ValueT: Debug + Serialize + for<'d> Deserialize<'d>, +>( + bucket: &KVBucket<'a, KeyT, ValueT>, + key: KeyT, +) -> miette::Result> { + let maybe_value: Option> = bucket + .remove(&key) + .into_diagnostic() + .wrap_err(KvErrorCouldNot::RemoveKeyValuePairFromBucket)?; + + let it = match maybe_value { + // Deserialize the binary payload into a Rust struct. + Some(Bincode(payload)) => Ok(Some(payload)), + _ => Ok(None), + }; + + debug!( + "โŒ {}", + format!( + "{}: {}: {}", + "Delete key / value pair from bucket", /*.green() */ + key.to_string(), /*.bold().cyan() */ + format!("{:?}", it) /*.bold().cyan() */ + ) + ); + + it +} + +#[instrument(skip(bucket))] +pub fn is_key_contained_in_bucket< + 'a, + KeyT: Debug + Display + for<'k> kv::Key<'k>, + ValueT: Debug + Serialize + for<'d> Deserialize<'d>, +>( + bucket: &KVBucket<'a, KeyT, ValueT>, + key: KeyT, +) -> miette::Result { + let it = bucket + .contains(&key) + .into_diagnostic() + .wrap_err(KvErrorCouldNot::LoadKeyValuePairFromBucket)?; + + debug!( + "๐Ÿ”ผ {}", + format!( + "{}: {}: {}", + "Check if key is contained in bucket", /*.green() */ + key.to_string(), /*.bold().cyan() */ + match it { + true => "true", /*.to_string().green() */ + false => "false", /*.to_string().red() */ + } + ) + ); + + Ok(it) +} + +pub fn iterate_bucket< + 'a, + KeyT: Debug + Display + for<'k> kv::Key<'k>, + ValueT: Debug + Serialize + for<'d> Deserialize<'d>, +>( + bucket: &KVBucket<'a, KeyT, ValueT>, + mut fn_to_apply: impl FnMut(KeyT, ValueT), +) { + for item in /* keep only the Ok variants */ bucket.iter().flatten() { + let Ok(key) = item.key::().into_diagnostic() else { + continue; + }; + let Ok(encoded_value) = item.value::>().into_diagnostic() else { + continue; + }; + let Bincode(value) = encoded_value; /* decode the value */ + fn_to_apply(key, value); + } +} + +pub mod kv_error { + #[allow(dead_code)] + #[derive(thiserror::Error, Debug, miette::Diagnostic)] + pub enum KvErrorCouldNot { + #[error("๐Ÿ“‘ Could not create db folder: '{db_folder_path}' on disk")] + CreateDbFolder { db_folder_path: String }, + + #[error("๐Ÿ“ฆ Could not create bucket from store: '{bucket_name}'")] + CreateBucketFromStore { bucket_name: String }, + + #[error("๐Ÿ”ฝ Could not save key/value pair to bucket")] + SaveKeyValuePairToBucket, + + #[error("๐Ÿ”ผ Could not load key/value pair from bucket")] + LoadKeyValuePairFromBucket, + + #[error("โŒ Could not remove key/value pair from bucket")] + RemoveKeyValuePairFromBucket, + + #[error("๐Ÿ” Could not get item from iterator from bucket")] + GetItemFromIteratorFromBucket, + + #[error("๐Ÿ” Could not get key from item from iterator from bucket")] + GetKeyFromItemFromIteratorFromBucket, + + #[error("๐Ÿ” Could not get value from item from iterator from bucket")] + GetValueFromItemFromIteratorFromBucket, + + #[error("โšก Could not execute transaction")] + ExecuteTransaction, + } +} +use kv_error::*; + +#[cfg(test)] +mod kv_tests { + use std::{collections::HashMap, + path::{Path, PathBuf}}; + + use serial_test::serial; + use tempfile::tempdir; + use tracing::{instrument, Level}; + + use super::*; + + fn check_folder_exists(path: &Path) -> bool { path.exists() && path.is_dir() } + + fn join_path_with_str(path: &Path, str: &str) -> PathBuf { path.join(str) } + + fn setup_tracing() { + let _ = tracing_subscriber::fmt() + .with_max_level(Level::INFO) + .compact() + .pretty() + .with_ansi(true) + .with_line_number(false) + .with_file(false) + .without_time() + .try_init(); + } + + fn get_path(dir: &tempfile::TempDir, folder_name: &str) -> PathBuf { + join_path_with_str(dir.path(), folder_name) + } + + fn create_temp_folder() -> tempfile::TempDir { + tempdir().expect("Failed to create temp dir") + } + + #[instrument] + fn perform_db_operations() -> miette::Result<()> { + let bucket_name = "bucket".to_string(); + + // Setup temp folder. + let dir = create_temp_folder(); + let path_buf = get_path(&dir, "db_folder"); + + setup_tracing(); + + // Create the key/value store. + let path_str = path_buf.as_path().to_string_lossy().to_string(); + let store = load_or_create_store(Some(&path_str))?; + + // Check that the key/value store folder exists. + assert!(check_folder_exists(path_buf.as_path())); + + // A bucket provides typed access to a section of the key/value store. + let bucket = load_or_create_bucket_from_store(&store, Some(&bucket_name))?; + + // Check if "foo" is contained in the bucket. + assert!(!(is_key_contained_in_bucket(&bucket, "foo".to_string())?)); + + // Nothing to iterate (empty bucket). + let mut count = 0; + for _ in bucket.iter() { + count += 1; + } + assert_eq!(count, 0); + + // Save to bucket. + insert_into_bucket(&bucket, "foo".to_string(), "bar".to_string())?; + + // Check if "foo" is contained in the bucket. + assert!(is_key_contained_in_bucket(&bucket, "foo".to_string())?); + + // Load from bucket. + assert_eq!( + get_from_bucket(&bucket, "foo".to_string())?, + Some("bar".to_string()) + ); + + // Enumerate contents of bucket. + let mut map = HashMap::new(); + for result_item in bucket.iter() { + let item = result_item + .into_diagnostic() + .wrap_err(KvErrorCouldNot::GetItemFromIteratorFromBucket)?; + + let key = item + .key::() + .into_diagnostic() + .wrap_err(KvErrorCouldNot::GetKeyFromItemFromIteratorFromBucket)?; + + // Deserialize the binary payload into a Rust struct. + let Bincode(payload) = item + .value::>() + .into_diagnostic() + .wrap_err(KvErrorCouldNot::GetValueFromItemFromIteratorFromBucket)?; + + map.insert(key.to_string(), payload); + } + + assert_eq!(map.get("foo"), Some(&"bar".to_string())); + + // Remove from bucket. + assert_eq!( + remove_from_bucket(&bucket, "foo".to_string())?, + Some("bar".to_string()) + ); + + // Check if "foo" is contained in the bucket. + assert!(!(is_key_contained_in_bucket(&bucket, "foo".to_string())?)); + + // Remove from bucket. + assert_eq!(remove_from_bucket(&bucket, "foo".to_string())?, None); + + Ok(()) + } + + #[instrument] + fn perform_db_operations_error_conditions() -> miette::Result<()> { + let bucket_name = "bucket".to_string(); + + // Setup temp folder. + let dir = create_temp_folder(); + let path_buf = get_path(&dir, "db_folder"); + + setup_tracing(); + + // Create the key/value store. + let path_str = path_buf.as_path().to_string_lossy().to_string(); + let store = load_or_create_store(Some(&path_str))?; + + // Check that the kv store folder exists. + assert!(check_folder_exists(path_buf.as_path())); + + // A bucket provides typed access to a section of the key/value store. + let bucket = load_or_create_bucket_from_store(&store, Some(&bucket_name))?; + + // Insert key/value pair into bucket. + insert_into_bucket(&bucket, "foo".to_string(), "bar".to_string())?; + + // Check for errors. The following line will induce errors, since we are + // intentionally trying to access a bucket that doesn't exist. + store.drop_bucket(bucket_name).into_diagnostic()?; + + // Insert into bucket. + let result = insert_into_bucket(&bucket, "foo".to_string(), "bar".to_string()); + match result { + Err(e) => { + assert_eq!(e.to_string(), "๐Ÿ”ฝ Could not save key/value pair to bucket"); + } + _ => { + panic!("Expected an error, but got None"); + } + } + + // Get from bucket. Take a deeper look in the chain of miette errors. + let result = get_from_bucket(&bucket, "foo".to_string()); + match result { + Err(e) => { + let mut iter = e.chain(); + // First. + assert_eq!( + iter.next().map(|it| it.to_string()).unwrap(), + "๐Ÿ”ผ Could not load key/value pair from bucket" + ); + + // Second. + let second = iter.next().map(|it| it.to_string()).unwrap(); + assert!(second.contains("Error in Sled: Collection")); + assert!(second.contains("does not exist")); + + // Third. + let third = iter.next().map(|it| it.to_string()).unwrap(); + assert!(third.contains("Collection")); + assert!(third.contains("does not exist")); + } + _ => { + panic!("Expected an error, but got None"); + } + } + + // Remove from bucket. + let result = remove_from_bucket(&bucket, "foo".to_string()); + match result { + Err(e) => { + assert_eq!( + e.to_string(), + "โŒ Could not remove key/value pair from bucket" + ); + } + _ => { + panic!("Expected an error, but got None"); + } + } + + // Check if key exists in bucket. + let result = is_key_contained_in_bucket(&bucket, "foo".to_string()); + match result { + Err(e) => { + assert_eq!( + e.to_string(), + "๐Ÿ”ผ Could not load key/value pair from bucket" + ); + } + _ => { + panic!("Expected an error, but got None"); + } + } + + // Enumerate contents of bucket. + let result = bucket.iter().next(); + match result { + Some(Err(e)) => { + assert!(e.to_string().contains("Error in Sled")); + assert!(e.to_string().contains("does not exist")); + } + _ => { + panic!("Expected an error, but got None"); + } + } + + Ok(()) + } + + /// Run this test in serial, not parallel. + #[serial] + #[test] + fn test_kv_operations() { + let result = perform_db_operations(); + assert!(result.is_ok()); + } + + /// Run this test in serial, not parallel. + #[serial] + #[test] + fn test_kv_errors() { + let result = perform_db_operations_error_conditions(); + assert!(result.is_ok()); + } +} diff --git a/terminal_async/src/tracing_jaeger/mod.rs b/core/src/storage/mod.rs similarity index 81% rename from terminal_async/src/tracing_jaeger/mod.rs rename to core/src/storage/mod.rs index fda5725fc..67a08c35c 100644 --- a/terminal_async/src/tracing_jaeger/mod.rs +++ b/core/src/storage/mod.rs @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 Nazmul Idris + * Copyright (c) 2024 R3BL LLC * All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,9 +16,7 @@ */ // Attach sources. -pub mod jaeger_setup; -pub mod port_availability; +pub mod kv; // Re-export. -pub use jaeger_setup::*; -pub use port_availability::*; +pub use kv::*; \ No newline at end of file diff --git a/core/src/tracing_logging/init_tracing.rs b/core/src/tracing_logging/init_tracing.rs index 554036af0..4aa7d2a73 100644 --- a/core/src/tracing_logging/init_tracing.rs +++ b/core/src/tracing_logging/init_tracing.rs @@ -25,6 +25,7 @@ //! logging are enabled. You can also customize the log level, and the file path and //! prefix for the log file. +use tracing::dispatcher; use tracing_core::LevelFilter; use tracing_subscriber::{layer::SubscriberExt, registry::LookupSpan, @@ -53,12 +54,30 @@ macro_rules! create_fmt { /// Type alias for a boxed layer. pub type DynLayer = dyn Layer + Send + Sync + 'static; -/// Simply initialize the tracing system with the provided [TracingConfig]. +/// Simply initialize the tracing system with the provided [TracingConfig]. This will set +/// the global default subscriber, which once set, can't be unset or changed. +/// +/// Documentation: +/// - [Global default tracing +/// subscriber](https://docs.rs/tracing/latest/tracing/subscriber/fn.set_global_default.html) pub fn init_tracing(tracing_config: TracingConfig) -> miette::Result<()> { try_create_layers(tracing_config) .map(|layers| tracing_subscriber::registry().with(layers).init()) } +/// Use this in tests, since this will not set the global default subscriber. This is a +/// thread local subscriber. +/// +/// Documentation: +/// - [Thread local tracing +/// subscriber](https://docs.rs/tracing/latest/tracing/subscriber/fn.set_default.html) +pub fn init_tracing_thread_local( + tracing_config: TracingConfig, +) -> miette::Result { + try_create_layers(tracing_config) + .map(|layers| tracing_subscriber::registry().with(layers).set_default()) +} + /// Returns the layers. This does not initialize the tracing system. Don't forget to do /// this manually, by calling `init` on the returned layers. /// @@ -279,7 +298,7 @@ mod test_tracing_shared_writer_output { // Create a new tracing layer with stdout. let display_pref = DisplayPreference::SharedWriter(SharedWriter::new(sender)); - init_tracing(TracingConfig { + let default_guard = init_tracing_thread_local(TracingConfig { writer_config: WriterConfig::Display(display_pref), level: tracing::Level::DEBUG, }) @@ -302,8 +321,12 @@ mod test_tracing_shared_writer_output { } let output = output.join("\n"); + println!("output: {}", output); + for it in EXPECTED.iter() { assert!(output.contains(it)); } + + drop(default_guard); } } diff --git a/terminal_async/Cargo.toml b/terminal_async/Cargo.toml index 4c48ba3a8..b859a0ef9 100644 --- a/terminal_async/Cargo.toml +++ b/terminal_async/Cargo.toml @@ -50,26 +50,6 @@ tracing-subscriber = "0.3.18" tracing-appender = "0.2.3" tracing-core = "0.1.32" -# โš ๏ธ NOTE โš ๏ธ -# DO NOT UPDATE TO THE LATEST VERSION OF THE FOLLOWING CRATES YET THERE ARE -# SOME INCOMPATIBILITIES WITH THE LATEST VERSIONS (0.24.0, etc) - -# OTel, jaeger, tracing. Implements the types defined in the Otel spec -# Run: `docker run -d -p16686:16686 -p4317:4317 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:latest` -# Open: http://localhost:16686/search -# Example: https://github.com/open-telemetry/opentelemetry-rust/blob/main/examples/tracing-jaeger/src/main.rs -# OpenTelemetryโ€™s API-level view of tracing, spans, etc. -opentelemetry = { version = "0.22.0" } -# Implements the OpenTelemetry APIs. -opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"] } -# The OTel protocol (OTLP) implementation to export data to Jaeger or some other backend. -# tonic is a gRPC crate. -opentelemetry-otlp = { version = "0.15.0", features = ["tonic"] } -# Integration between the tracing crate and the opentelemetry crate. -tracing-opentelemetry = "0.23.0" -# Standardized naming patterns for OpenTelemetry things. -opentelemetry-semantic-conventions = "0.14.0" - # Async stream for DI and testing. futures-core = "0.3.30" async-stream = "0.3.5" diff --git a/terminal_async/src/lib.rs b/terminal_async/src/lib.rs index 6276082ec..d4bd112ec 100644 --- a/terminal_async/src/lib.rs +++ b/terminal_async/src/lib.rs @@ -458,13 +458,11 @@ pub mod public_api; pub mod readline_impl; pub mod spinner_impl; -pub mod tracing_jaeger; // Re-export the public API. pub use public_api::*; pub use readline_impl::*; pub use spinner_impl::*; -pub use tracing_jaeger::*; // External crates. use std::{collections::VecDeque, io::Error, pin::Pin, sync::Arc}; diff --git a/terminal_async/src/tracing_jaeger/jaeger_setup.rs b/terminal_async/src/tracing_jaeger/jaeger_setup.rs deleted file mode 100644 index da2b5c73f..000000000 --- a/terminal_async/src/tracing_jaeger/jaeger_setup.rs +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2024 R3BL LLC - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use std::str::FromStr; - -use miette::IntoDiagnostic; -use opentelemetry::{global, trace::TraceError, KeyValue}; -use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::{runtime::Tokio, trace as sdktrace, Resource}; -use opentelemetry_semantic_conventions::resource::SERVICE_NAME; -use tracing::Subscriber; -use tracing_subscriber::registry::LookupSpan; - -use crate::port_availability; - -/// Hostname for Jaeger. -const HOST: &str = "127.0.0.1"; -/// gRPC port for Jaeger. -const PORT: u16 = 4317; - -pub fn tcp_addr() -> String { format!("{}:{}", HOST, PORT) } - -pub fn url() -> String { format!("http://{}:{}", HOST, PORT) } - -pub fn get_socket_addr( - maybe_otel_collector_endpoint: Option, -) -> miette::Result { - let addr = match maybe_otel_collector_endpoint { - Some(it) => it.to_string(), - None => tcp_addr(), - }; - - let (host, port) = { - let it = addr.splitn(2, ':').collect::>(); - if it.len() != 2 { - return Err(miette::miette!("Invalid address")); - } - let host = std::net::IpAddr::from_str(it[0]).into_diagnostic()?; - let port = it[1].parse::().into_diagnostic()?; - (host, port) - }; - - let socket_addr = std::net::SocketAddr::new(host, port); - Ok(socket_addr) -} - -#[test] -fn test_get_socket_addr() -> miette::Result<()> { - // Check defaults. - let addr = get_socket_addr(None)?; - assert_eq!(addr.port(), PORT); - assert_eq!( - addr.ip(), - std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)) - ); - - // Check custom address. - let ip: std::net::SocketAddr = "127.0.0.1:12".parse().into_diagnostic()?; - let addr = get_socket_addr(Some(ip))?; - assert_eq!(addr.port(), 12); - assert_eq!( - addr.ip(), - std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)) - ); - - Ok(()) -} - -/// Check whether port 4317 is open for Jaeger. -pub async fn is_jaeger_up( - maybe_otel_collector_endpoint: Option, -) -> miette::Result { - let socket_addr = get_socket_addr(maybe_otel_collector_endpoint)?; - match port_availability::check(socket_addr).await? { - port_availability::Status::Occupied => Ok(true), - port_availability::Status::Free => Ok(false), - } -} - -/// The OTLP (OpenTelemetry protocol) Exporter supports exporting logs, metrics and traces -/// in the OTLP format to the OpenTelemetry collector or other compatible backend. -/// -/// The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, -/// process, and export telemetry data. In addition, it removes the need to run, operate, -/// and maintain multiple agents/collectors in order to support open-source telemetry data -/// formats (e.g. Jaeger, Prometheus, etc.) sending to multiple open-source or commercial -/// back-ends. -/// -/// Currently, this crate only support sending tracing data or metrics in OTLP via grpc -/// and http (in binary format). Supports for other format and protocol will be added in -/// the future. The details of whatโ€™s currently offering in this crate can be found in -/// this doc. -/// -/// More info: -/// 1. -/// 2. -/// 3. -/// 4. -fn try_init_exporter( - service_name: &str, -) -> Result { - let exporter = opentelemetry_otlp::new_exporter() - .tonic() - .with_endpoint(url()); - - let trace_config = - sdktrace::config().with_resource(Resource::new(vec![KeyValue::new( - SERVICE_NAME, - service_name.to_string(), - )])); - - let runtime = Tokio; - - opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(exporter) - .with_trace_config(trace_config) - .install_batch(runtime) -} - -/// 1. This will try and create a Jaeger OTel layer, and if successful, it will return it. -/// 2. If Jaeger is not up, it will return [Option::None]. -/// 3. If there is a problem with detecting whether Jaeger is up, it will return an error. -pub async fn try_get_otel_layer( - service_name: &str, - maybe_otel_collector_endpoint: Option, -) -> miette::Result< - std::option::Option<( - tracing_opentelemetry::OpenTelemetryLayer, - DropTracer, - )>, -> -where - S: Subscriber + for<'span> LookupSpan<'span>, -{ - if is_jaeger_up(maybe_otel_collector_endpoint).await? { - if let Ok(tracer) = try_init_exporter(service_name) { - let it = tracing_opentelemetry::layer().with_tracer(tracer); - return Ok(Some((it, DropTracer))); - } - } - Ok(None) -} - -pub struct DropTracer; - -impl Drop for DropTracer { - fn drop(&mut self) { global::shutdown_tracer_provider(); } -} diff --git a/terminal_async/src/tracing_jaeger/port_availability.rs b/terminal_async/src/tracing_jaeger/port_availability.rs deleted file mode 100644 index 495cbcc9b..000000000 --- a/terminal_async/src/tracing_jaeger/port_availability.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2024 R3BL LLC - * All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use miette::{Context, IntoDiagnostic}; - -pub fn find_first_available_port_in_range( - maybe_range: Option>, - host: &str, -) -> Option { - let mut range = maybe_range.unwrap_or(8000..9000); - range.find(|port| std::net::TcpListener::bind((host, *port)).is_ok()) -} - -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum Status { - Free, - Occupied, -} - -/// Check whether a port is available. Returns an error if there is an issue creating a -/// TCP socket. Does not return an error, if the port is not available. -pub async fn check(addr: std::net::SocketAddr) -> miette::Result { - let socket = tokio::net::TcpSocket::new_v4() - .into_diagnostic() - .with_context(|| "Failed to create a new TCP socket".to_string())?; - let result_socket_connect = socket.connect(addr).await; - match result_socket_connect { - Ok(_) => Ok(Status::Occupied), - Err(_) => Ok(Status::Free), - } -} - -#[tokio::test] -async fn test_is_port_available() { - // Find a free port. Check if it's available. - let host = "127.0.0.1"; - let free_port = find_first_available_port_in_range(None, host).unwrap(); - let addr_str = format!("{}:{}", host, free_port); - - use std::str::FromStr; - let addr = std::net::SocketAddr::from_str(&addr_str).unwrap(); - - assert_eq!(check(addr).await.unwrap(), Status::Free); - - // Occupy that port. Check if it's not available. - let _listener = std::net::TcpListener::bind(addr).unwrap(); - assert_eq!(check(addr).await.unwrap(), Status::Occupied); -}