diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10b9490b..6a63b09f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,6 @@ on: push: branches: [main] pull_request: - branches: [main] workflow_dispatch: jobs: @@ -39,7 +38,7 @@ jobs: - uses: moonrepo/setup-rust@v1 - run: cargo test --all - compat-integration-test: + compat-integration-test-instrumentation: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -58,3 +57,21 @@ jobs: with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} + + compat-integration-test-walltime: + runs-on: codspeed-macro + steps: + - uses: actions/checkout@v4 + - uses: moonrepo/setup-rust@v1 + with: + cache-target: release + + - run: cargo install --path crates/cargo-codspeed --locked + + - run: cargo codspeed build -p codspeed-divan-compat + + - name: Run the benchmarks + uses: CodSpeedHQ/action@main + with: + run: cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.vscode/settings.json b/.vscode/settings.json index 08556dab..869f00c0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { - "rust-analyzer.cargo.features": "all", - "editor.formatOnSave": true, "rust-analyzer.checkOnSave": true, - "rust-analyzer.check.command": "clippy" + "rust-analyzer.check.command": "clippy", + "rust-analyzer.cargo.features": "all", + "rust-analyzer.cargo.cfgs": ["debug_assertions", "miri", "codspeed"] } diff --git a/Cargo.lock b/Cargo.lock index 4fc47fe3..50725511 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,15 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "assert_cmd" version = "2.0.15" @@ -133,7 +142,7 @@ checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7" dependencies = [ "async-task", "concurrent-queue", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-lite 2.3.0", "slab", ] @@ -419,6 +428,18 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytemuck" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "camino" version = "1.1.9" @@ -437,9 +458,14 @@ dependencies = [ "assert_cmd", "cargo_metadata", "clap", + "codspeed 2.7.2", "fs_extra", + "glob", "itertools 0.13.0", "predicates", + "serde", + "serde_json", + "statrs", "termcolor", "uuid", ] @@ -532,6 +558,7 @@ dependencies = [ "anstyle", "clap_lex", "strsim", + "terminal_size", ] [[package]] @@ -558,8 +585,22 @@ version = "2.7.2" dependencies = [ "colored", "libc", + "serde", "serde_json", "tempfile", + "uuid", +] + +[[package]] +name = "codspeed" +version = "2.7.2" +source = "git+https://github.com/CodSpeedHQ/codspeed-rust?branch=cod-526-build-and-find-walltime-entrypoint-with-divan#209374e1bc7e49221879f3348a364365992ae065" +dependencies = [ + "colored", + "libc", + "serde", + "serde_json", + "uuid", ] [[package]] @@ -567,7 +608,7 @@ name = "codspeed-bencher-compat" version = "2.7.2" dependencies = [ "bencher", - "codspeed", + "codspeed 2.7.2", ] [[package]] @@ -575,7 +616,7 @@ name = "codspeed-criterion-compat" version = "2.7.2" dependencies = [ "async-std", - "codspeed", + "codspeed 2.7.2", "colored", "criterion", "futures", @@ -583,6 +624,25 @@ dependencies = [ "tokio", ] +[[package]] +name = "codspeed-divan-compat" +version = "2.7.2" +dependencies = [ + "codspeed 2.7.2", + "codspeed-divan-compat-macros", + "divan", +] + +[[package]] +name = "codspeed-divan-compat-macros" +version = "2.7.2" +dependencies = [ + "divan-macros 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "colorchoice" version = "1.0.2" @@ -608,6 +668,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "criterion" version = "0.5.1" @@ -707,6 +773,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "divan" +version = "0.1.17" +source = "git+https://github.com/CodSpeedHQ/divan#e605bf0c971aeb08bc55867abecc56bafbbdc3a0" +dependencies = [ + "cfg-if", + "clap", + "codspeed 2.7.2 (git+https://github.com/CodSpeedHQ/codspeed-rust?branch=cod-526-build-and-find-walltime-entrypoint-with-divan)", + "condtype", + "divan-macros 0.1.17 (git+https://github.com/CodSpeedHQ/divan)", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "git+https://github.com/CodSpeedHQ/divan#e605bf0c971aeb08bc55867abecc56bafbbdc3a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -778,9 +879,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "float-cmp" @@ -866,7 +967,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -919,6 +1020,12 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -1050,6 +1157,12 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -1071,6 +1184,16 @@ dependencies = [ "value-bag", ] +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1086,12 +1209,68 @@ dependencies = [ "adler", ] +[[package]] +name = "nalgebra" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b" +dependencies = [ + "approx", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "rand", + "rand_distr", + "simba", + "typenum", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1099,6 +1278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1128,6 +1308,12 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1147,7 +1333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-io", ] @@ -1210,6 +1396,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "predicates" version = "3.1.2" @@ -1258,6 +1453,52 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.10.0" @@ -1301,6 +1542,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -1346,6 +1593,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1366,18 +1622,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -1386,9 +1642,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.137" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -1405,6 +1661,19 @@ dependencies = [ "libc", ] +[[package]] +name = "simba" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a386a501cd104797982c15ae17aafe8b9261315b5d07e3ec803f2ea26be0fa" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", + "wide", +] + [[package]] name = "slab" version = "0.4.9" @@ -1458,6 +1727,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "nalgebra", + "num-traits", + "rand", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1482,7 +1763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand 2.3.0", "rustix 0.38.34", "windows-sys 0.52.0", ] @@ -1496,6 +1777,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix 0.38.34", + "windows-sys 0.48.0", +] + [[package]] name = "termtree" version = "0.4.1" @@ -1558,6 +1849,12 @@ version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -1572,9 +1869,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ "getrandom", ] @@ -1692,6 +1989,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wide" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b5576b9a81633f3e8df296ce0063042a73507636cbe956c61133dd7034ab22" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1861,3 +2168,24 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 91684ae7..a7ca6d1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,11 @@ members = [ "crates/bencher_compat", "crates/criterion_compat", "crates/cargo-codspeed", + "crates/divan_compat", + "crates/divan_compat/macros", ] resolver = "2" + +[workspace.dependencies] +serde = { version = "1.0.217", features = ["derive"] } +serde_json = "1.0.138" diff --git a/README.md b/README.md index 0f3b004c..cdb6b719 100644 --- a/README.md +++ b/README.md @@ -13,4 +13,5 @@ This mono-repo contains the integration crates for using CodSpeed in Rust: - [`cargo-codspeed`](./crates/cargo-codspeed/): A cargo subcommand for running CodSpeed on your project - [`codspeed-criterion-compat`](./crates/criterion_compat/): Criterion.rs compatibility layer for CodSpeed - [`codspeed-bencher-compat`](./crates/bencher_compat/): Bencher compatibility layer for CodSpeed +- [`codspeed-divan-compat`](./crates/divan_compat/): Divan compatibility layer for CodSpeed - [`codspeed`](./crates/codspeed/): The core library used to integrate with Codspeed runners diff --git a/crates/cargo-codspeed/Cargo.toml b/crates/cargo-codspeed/Cargo.toml index 48f36fa4..cc1a61a2 100644 --- a/crates/cargo-codspeed/Cargo.toml +++ b/crates/cargo-codspeed/Cargo.toml @@ -19,11 +19,16 @@ keywords = ["codspeed", "benchmark", "cargo"] [dependencies] cargo_metadata = "0.19.1" -clap = { version = "=4.5.17", features = ["derive"] } +clap = { version = "=4.5.17", features = ["derive", "env"] } termcolor = "1.4" anyhow = "1.0.86" itertools = "0.13.0" anstyle = "1.0.8" +serde = { workspace = true } +serde_json = { workspace = true } +codspeed = { path = "../codspeed", version = "=2.7.2" } +glob = "0.3.2" +statrs = "0.18.0" [dev-dependencies] assert_cmd = "2.0.15" diff --git a/crates/cargo-codspeed/src/app.rs b/crates/cargo-codspeed/src/app.rs index 334270df..d2c9cda4 100644 --- a/crates/cargo-codspeed/src/app.rs +++ b/crates/cargo-codspeed/src/app.rs @@ -1,4 +1,4 @@ -use crate::{prelude::*, run::run_benches}; +use crate::{measurement_mode::MeasurementMode, prelude::*, run::run_benches}; use cargo_metadata::MetadataCommand; use clap::{Args, Parser, Subcommand}; use std::{ffi::OsString, process::exit}; @@ -12,6 +12,13 @@ struct Cli { #[arg(short, long, global = true)] quiet: bool, + /// The measurement tool to use for measuring performance. + /// Automatically set to `walltime` on macro runners + // This is an Option even if MeasurementMode has a default because + // the default is dynamic and this would mislead the user + #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE")] + measurement_mode: Option, + #[command(subcommand)] command: Commands, } @@ -63,6 +70,9 @@ pub fn run(args: impl Iterator) -> Result<()> { let metadata = MetadataCommand::new().exec()?; let cli = Cli::try_parse_from(args)?; + let measurement_mode = cli.measurement_mode.unwrap_or_default(); + eprintln!("[cargo-codspeed] Measurement mode: {measurement_mode:?}\n"); + let res = match cli.command { Commands::Build { filters, @@ -71,9 +81,16 @@ pub fn run(args: impl Iterator) -> Result<()> { } => { let features = features.map(|f| f.split([' ', ',']).map(|s| s.to_string()).collect_vec()); - build_benches(&metadata, filters, features, profile, cli.quiet) + build_benches( + &metadata, + filters, + features, + profile, + cli.quiet, + measurement_mode, + ) } - Commands::Run { filters } => run_benches(&metadata, filters), + Commands::Run { filters } => run_benches(&metadata, filters, measurement_mode), }; if let Err(e) = res { diff --git a/crates/cargo-codspeed/src/build.rs b/crates/cargo-codspeed/src/build.rs index 6be68fa8..c01a2530 100644 --- a/crates/cargo-codspeed/src/build.rs +++ b/crates/cargo-codspeed/src/build.rs @@ -1,6 +1,7 @@ use crate::{ app::{Filters, PackageFilters}, helpers::{clear_dir, get_codspeed_target_dir}, + measurement_mode::MeasurementMode, prelude::*, }; use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind}; @@ -21,10 +22,15 @@ struct BuiltBench { impl BuildOptions<'_> { /// Builds the benchmarks by invoking cargo /// Returns a list of built benchmarks, with path to associated executables - fn build(&self, metadata: &Metadata, quiet: bool) -> Result> { + fn build( + &self, + metadata: &Metadata, + quiet: bool, + measurement_mode: MeasurementMode, + ) -> Result> { let workspace_packages = metadata.workspace_packages(); - let mut cargo = self.build_command(); + let mut cargo = self.build_command(measurement_mode); if quiet { cargo.arg("--quiet"); } @@ -43,10 +49,17 @@ impl BuildOptions<'_> { let mut built_benches = Vec::new(); for message in Message::parse_stream(reader) { - if let Message::CompilerArtifact(artifact) = - message.expect("Failed to parse compiler message") - { - if artifact.target.is_kind(TargetKind::Bench) { + match message.expect("Failed to parse message") { + // Those messages will include build errors and warnings even if stderr also contain some of them + Message::CompilerMessage(msg) => { + println!("{}", &msg.message); + } + Message::TextLine(line) => { + println!("{}", line); + } + Message::CompilerArtifact(artifact) + if artifact.target.is_kind(TargetKind::Bench) => + { let package = workspace_packages .iter() .find(|p| p.id == artifact.package_id) @@ -71,6 +84,7 @@ impl BuildOptions<'_> { }); } } + _ => {} } } @@ -91,18 +105,20 @@ impl BuildOptions<'_> { } /// Generates a subcommand to build the benchmarks by invoking cargo and forwarding the filters - /// This command explicitely ignores the `self.benches`: all benches are built - fn build_command(&self) -> Command { + /// This command explicitly ignores the `self.benches`: all benches are built + fn build_command(&self, measurement_mode: MeasurementMode) -> Command { let mut cargo = Command::new("cargo"); cargo.args(["build", "--benches"]); - cargo.env( - "RUSTFLAGS", - format!( - "{} -g --cfg codspeed", - std::env::var("RUSTFLAGS").unwrap_or_else(|_| "".into()) - ), - ); + let mut rust_flags = std::env::var("RUSTFLAGS").unwrap_or_else(|_| "".into()); + // Add debug info (equivalent to -g) + rust_flags.push_str(" -C debuginfo=2"); + + // Add the codspeed cfg flag if instrumentation mode is enabled + if measurement_mode == MeasurementMode::Instrumentation { + rust_flags.push_str(" --cfg codspeed"); + } + cargo.env("RUSTFLAGS", rust_flags); if let Some(features) = self.features { cargo.arg("--features").arg(features.join(",")); @@ -142,13 +158,14 @@ pub fn build_benches( features: Option>, profile: String, quiet: bool, + measurement_mode: MeasurementMode, ) -> Result<()> { let built_benches = BuildOptions { filters, features: &features, profile: &profile, } - .build(metadata, quiet)?; + .build(metadata, quiet, measurement_mode)?; if built_benches.is_empty() { bail!( diff --git a/crates/cargo-codspeed/src/main.rs b/crates/cargo-codspeed/src/main.rs index d0920a16..82f1e573 100644 --- a/crates/cargo-codspeed/src/main.rs +++ b/crates/cargo-codspeed/src/main.rs @@ -1,8 +1,10 @@ mod app; mod build; mod helpers; +mod measurement_mode; mod prelude; mod run; +mod walltime_results; use crate::prelude::*; use std::{env::args_os, process::exit}; diff --git a/crates/cargo-codspeed/src/measurement_mode.rs b/crates/cargo-codspeed/src/measurement_mode.rs new file mode 100644 index 00000000..7170a9a2 --- /dev/null +++ b/crates/cargo-codspeed/src/measurement_mode.rs @@ -0,0 +1,20 @@ +use clap::ValueEnum; +use serde::Serialize; +use std::env; + +#[derive(Debug, Clone, ValueEnum, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum MeasurementMode { + Walltime, + Instrumentation, +} + +impl Default for MeasurementMode { + fn default() -> Self { + if env::var("CODSPEED_ENV").is_ok() { + MeasurementMode::Instrumentation + } else { + MeasurementMode::Walltime + } + } +} diff --git a/crates/cargo-codspeed/src/run.rs b/crates/cargo-codspeed/src/run.rs index 46cfa2fd..0d9c3946 100644 --- a/crates/cargo-codspeed/src/run.rs +++ b/crates/cargo-codspeed/src/run.rs @@ -1,10 +1,18 @@ use crate::{ app::{Filters, PackageFilters}, helpers::get_codspeed_target_dir, + measurement_mode::MeasurementMode, prelude::*, + walltime_results::{WalltimeBenchmark, WalltimeResults}, }; +use anyhow::Context; use cargo_metadata::{Metadata, Package}; -use std::{io, path::PathBuf}; +use codspeed::walltime::get_raw_result_dir_from_workspace_root; +use glob::glob; +use std::{ + io::{self, Write}, + path::{Path, PathBuf}, +}; struct BenchToRun { bench_path: PathBuf, @@ -86,8 +94,16 @@ impl PackageFilters { } } -pub fn run_benches(metadata: &Metadata, filters: Filters) -> Result<()> { +pub fn run_benches( + metadata: &Metadata, + filters: Filters, + measurement_mode: MeasurementMode, +) -> Result<()> { let codspeed_target_dir = get_codspeed_target_dir(metadata); + let workspace_root = metadata.workspace_root.as_std_path(); + if measurement_mode == MeasurementMode::Walltime { + clear_raw_walltime_data(workspace_root)?; + } let benches = filters.benches_to_run(codspeed_target_dir, metadata)?; if benches.is_empty() { bail!("No benchmarks found. Run `cargo codspeed build` first."); @@ -125,9 +141,16 @@ pub fn run_benches(metadata: &Metadata, filters: Filters) -> Result<()> { // while CARGO_MANIFEST_DIR returns the path to the sub package let workspace_root = metadata.workspace_root.clone(); eprintln!("Running {} {}", &bench.package_name, bench_name); - std::process::Command::new(&bench.bench_path) + let mut command = std::process::Command::new(&bench.bench_path); + command .env("CODSPEED_CARGO_WORKSPACE_ROOT", workspace_root) - .current_dir(&bench.working_directory) + .current_dir(&bench.working_directory); + + if measurement_mode == MeasurementMode::Walltime { + command.arg("--bench"); // Walltime targets need this additional argument (inherited from running them with `cargo bench`) + } + + command .status() .map_err(|e| anyhow!("failed to execute the benchmark process: {}", e)) .and_then(|status| { @@ -143,5 +166,55 @@ pub fn run_benches(metadata: &Metadata, filters: Filters) -> Result<()> { eprintln!("Done running {}", bench_name); } eprintln!("Finished running {} benchmark suite(s)", to_run.len()); + + if measurement_mode == MeasurementMode::Walltime { + aggregate_raw_walltime_data(workspace_root)?; + } + + Ok(()) +} + +fn clear_raw_walltime_data(workspace_root: &Path) -> Result<()> { + let raw_results_dir = get_raw_result_dir_from_workspace_root(workspace_root); + std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist + std::fs::create_dir_all(&raw_results_dir).context("Failed to create raw_results directory")?; + Ok(()) +} + +fn aggregate_raw_walltime_data(workspace_root: &Path) -> Result<()> { + // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json + let walltime_benchmarks = glob(&format!( + "{}/**/*.json", + get_raw_result_dir_from_workspace_root(workspace_root) + .to_str() + .unwrap(), + ))? + .map(|sample| { + let sample = sample?; + let raw_walltime_data: codspeed::walltime::RawWallTimeData = + serde_json::from_reader(std::fs::File::open(&sample)?)?; + Ok(WalltimeBenchmark::from(raw_walltime_data)) + }) + .collect::>>()?; + + if walltime_benchmarks.is_empty() { + eprintln!("No walltime benchmarks found"); + return Ok(()); + } + + let results_folder = std::env::var("CODSPEED_PROFILE_FOLDER") + .map(PathBuf::from) + .unwrap_or_else(|_| workspace_root.join("target/codspeed/profiles")) + .join("results"); + std::fs::create_dir_all(&results_folder).context("Failed to create results folder")?; + + let results = WalltimeResults::from_benchmarks(walltime_benchmarks); + let results_path = results_folder.join(format!("{}.json", std::process::id())); + let mut results_file = + std::fs::File::create(&results_path).context("Failed to create results file")?; + serde_json::to_writer_pretty(&results_file, &results)?; + results_file + .flush() + .context("Failed to flush results file")?; Ok(()) } diff --git a/crates/cargo-codspeed/src/walltime_results.rs b/crates/cargo-codspeed/src/walltime_results.rs new file mode 100644 index 00000000..0f60fd48 --- /dev/null +++ b/crates/cargo-codspeed/src/walltime_results.rs @@ -0,0 +1,144 @@ +use codspeed::walltime::{BenchmarkMetadata, RawWallTimeData}; +use serde::{Deserialize, Serialize}; +use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; + +const IQR_OUTLIER_FACTOR: f64 = 1.5; +const STDEV_OUTLIER_FACTOR: f64 = 3.0; + +#[derive(Debug, Serialize, Deserialize)] +struct BenchmarkStats { + min_ns: f64, + max_ns: f64, + mean_ns: f64, + stdev_ns: f64, + + q1_ns: f64, + median_ns: f64, + q3_ns: f64, + + rounds: u64, + total_time: f64, + iqr_outlier_rounds: u64, + stdev_outlier_rounds: u64, + iter_per_round: u64, + warmup_iters: u64, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +struct BenchmarkConfig { + warmup_time_ns: Option, + min_round_time_ns: Option, + max_time_ns: Option, + max_rounds: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WalltimeBenchmark { + #[serde(flatten)] + metadata: BenchmarkMetadata, + + config: BenchmarkConfig, + stats: BenchmarkStats, +} + +impl From for WalltimeBenchmark { + fn from(value: RawWallTimeData) -> Self { + let times_ns: Vec = value.times_ns.iter().map(|&t| t as f64).collect(); + let mut data = Data::new(times_ns.clone()); + let rounds = data.len() as u64; + let total_time = times_ns.iter().sum::() / 1_000_000_000.0; + + let mean_ns = data.mean().unwrap(); + let stdev_ns = data.std_dev().unwrap(); + + let q1_ns = data.quantile(0.25); + let median_ns = data.median(); + let q3_ns = data.quantile(0.75); + + let iqr_ns = q3_ns - q1_ns; + let iqr_outlier_rounds = data + .iter() + .filter(|&&t| { + t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns + }) + .count() as u64; + + let stdev_outlier_rounds = data + .iter() + .filter(|&&t| { + t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns + || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns + }) + .count() as u64; + + let min_ns = data.min(); + let max_ns = data.max(); + + let iter_per_round = value.iter_per_round as u64; + let warmup_iters = 0; // FIXME: add warmup detection + + let stats = BenchmarkStats { + min_ns, + max_ns, + mean_ns, + stdev_ns, + q1_ns, + median_ns, + q3_ns, + rounds, + total_time, + iqr_outlier_rounds, + stdev_outlier_rounds, + iter_per_round, + warmup_iters, + }; + + WalltimeBenchmark { + metadata: BenchmarkMetadata { + name: value.metadata.name, + uri: value.metadata.uri, + }, + config: BenchmarkConfig { + max_time_ns: value.max_time_ns.map(|t| t as f64), + ..Default::default() + }, + stats, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Instrument { + #[serde(rename = "type")] + type_: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Creator { + name: String, + version: String, + pid: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WalltimeResults { + creator: Creator, + instrument: Instrument, + benchmarks: Vec, +} + +impl WalltimeResults { + pub fn from_benchmarks(benchmarks: Vec) -> Self { + WalltimeResults { + instrument: Instrument { + type_: "walltime".to_string(), + }, + creator: Creator { + name: "codspeed-rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + pid: std::process::id(), + }, + benchmarks, + } + } +} diff --git a/crates/codspeed/Cargo.toml b/crates/codspeed/Cargo.toml index 4f172ef7..515a4219 100644 --- a/crates/codspeed/Cargo.toml +++ b/crates/codspeed/Cargo.toml @@ -20,7 +20,9 @@ keywords = ["codspeed", "benchmark"] [dependencies] colored = "2.0.0" libc = "^0.2" -serde_json = "1.0.120" +serde = { workspace = true } +serde_json = { workspace = true } +uuid = { version = "1.12.1", features = ["v4"] } [[bench]] name = "native" diff --git a/crates/codspeed/src/lib.rs b/crates/codspeed/src/lib.rs index 42ffe4be..bbac6486 100644 --- a/crates/codspeed/src/lib.rs +++ b/crates/codspeed/src/lib.rs @@ -3,3 +3,4 @@ mod macros; mod measurement; mod request; pub mod utils; +pub mod walltime; diff --git a/crates/codspeed/src/walltime.rs b/crates/codspeed/src/walltime.rs new file mode 100644 index 00000000..7be00fb4 --- /dev/null +++ b/crates/codspeed/src/walltime.rs @@ -0,0 +1,73 @@ +use std::{ + io::Write, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkMetadata { + pub name: String, + pub uri: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct RawWallTimeData { + #[serde(flatten)] + pub metadata: BenchmarkMetadata, + pub iter_per_round: u32, + pub max_time_ns: Option, + pub times_ns: Vec, +} + +impl RawWallTimeData { + fn from_runtime_data( + name: String, + uri: String, + iter_per_round: u32, + max_time_ns: Option, + times_ns: Vec, + ) -> Self { + RawWallTimeData { + metadata: BenchmarkMetadata { name, uri }, + iter_per_round, + max_time_ns, + times_ns, + } + } + + fn dump_to_results(&self, workspace_root: &Path, scope: &str) { + let output_dir = get_raw_result_dir_from_workspace_root(workspace_root).join(scope); + std::fs::create_dir_all(&output_dir).unwrap(); + let bench_id = uuid::Uuid::new_v4().to_string(); + let output_path = output_dir.join(format!("{}.json", bench_id)); + let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); + serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); + writer.flush().expect("Failed to flush the writer"); + } +} + +/// Entry point called in patched integration to harvest raw walltime data +pub fn collect_raw_walltime_results( + scope: &str, + name: String, + uri: String, + iter_per_round: u32, + max_time_ns: Option, + times_ns: Vec, +) { + let workspace_root = std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT").map(PathBuf::from); + let Ok(workspace_root) = workspace_root else { + eprintln!("codspeed failed to get workspace root. skipping"); + return; + }; + let data = RawWallTimeData::from_runtime_data(name, uri, iter_per_round, max_time_ns, times_ns); + data.dump_to_results(&workspace_root, scope); +} + +pub fn get_raw_result_dir_from_workspace_root(workspace_root: &Path) -> PathBuf { + workspace_root + .join("target") + .join("codspeed") + .join("raw_results") +} diff --git a/crates/divan_compat/Cargo.toml b/crates/divan_compat/Cargo.toml new file mode 100644 index 00000000..6333aa6a --- /dev/null +++ b/crates/divan_compat/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "codspeed-divan-compat" +version = "2.7.2" +rust-version = "1.74" # MSRV TODO: Check versioning +edition = "2021" +description = "Bencher compatibility layer for CodSpeed" +authors = ["Arthur Pastel "] +documentation = "https://docs.codspeed.io" +readme = "README.md" +repository = "https://github.com/CodSpeedHQ/codspeed-rust" +homepage = "https://codspeed.io" +license = "MIT OR Apache-2.0" +categories = [ + "development-tools", + "development-tools::profiling", + "development-tools::testing", +] +keywords = ["codspeed", "benchmark", "divan"] + +[dependencies] +codspeed = { path = "../codspeed", version = "=2.7.2" } +divan = { git = "https://github.com/CodSpeedHQ/divan" } +codspeed-divan-compat-macros = { version = "=2.7.2", path = './macros' } + +[[bench]] +name = "basic_example" +harness = false diff --git a/crates/divan_compat/README.md b/crates/divan_compat/README.md new file mode 100644 index 00000000..1333ed77 --- /dev/null +++ b/crates/divan_compat/README.md @@ -0,0 +1 @@ +TODO diff --git a/crates/divan_compat/benches/basic_example.rs b/crates/divan_compat/benches/basic_example.rs new file mode 100644 index 00000000..77822dba --- /dev/null +++ b/crates/divan_compat/benches/basic_example.rs @@ -0,0 +1,29 @@ +use codspeed_divan_compat as divan; + +fn fibo(n: i32) -> i32 { + let mut a = 0; + let mut b = 1; + + for _ in 0..n { + let tmp = a; + a = b; + b += tmp; + } + + a +} + +#[divan::bench] +fn fibo_500() -> i32 { + divan::black_box(fibo(500)) +} + +#[divan::bench] +fn fibo_100() -> i32 { + divan::black_box(fibo(10)) +} + +fn main() { + // Run `add` benchmark: + divan::main(); +} diff --git a/crates/divan_compat/build.rs b/crates/divan_compat/build.rs new file mode 100644 index 00000000..688a004f --- /dev/null +++ b/crates/divan_compat/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rustc-check-cfg=cfg(codspeed)"); +} diff --git a/crates/divan_compat/macros/Cargo.toml b/crates/divan_compat/macros/Cargo.toml new file mode 100644 index 00000000..307a6d63 --- /dev/null +++ b/crates/divan_compat/macros/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "codspeed-divan-compat-macros" +version = "2.7.2" +rust-version = "1.74" # MSRV TODO: Check versioning +edition = "2021" +description = "Bencher compatibility layer for CodSpeed" +authors = ["Arthur Pastel "] +documentation = "https://docs.codspeed.io" +readme = "../README.md" +repository = "https://github.com/CodSpeedHQ/codspeed-rust" +homepage = "https://codspeed.io" +license = "MIT OR Apache-2.0" +categories = [ + "development-tools", + "development-tools::profiling", + "development-tools::testing", +] +keywords = ["codspeed", "benchmark", "divan"] + +[lib] +proc-macro = true + +[dependencies] +divan-macros = { version = "=0.1.17" } +proc-macro2 = "1" +quote = { version = "1", default-features = false } +# Versions prior to *.18 fail to parse empty attribute metadata. +syn = { version = "^2.0.18", default-features = false, features = ["full", "clone-impls", "parsing", "printing", "proc-macro"] } diff --git a/crates/divan_compat/macros/src/lib.rs b/crates/divan_compat/macros/src/lib.rs new file mode 100644 index 00000000..cacc43ed --- /dev/null +++ b/crates/divan_compat/macros/src/lib.rs @@ -0,0 +1,47 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{ + parse::Parse, parse_macro_input, punctuated::Punctuated, ItemFn, Meta, MetaNameValue, Token, +}; + +struct MyBenchArgs { + args: Punctuated, +} + +impl Parse for MyBenchArgs { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + Ok(Self { + args: Punctuated::parse_terminated(input)?, + }) + } +} + +#[proc_macro_attribute] +pub fn bench_compat(attr: TokenStream, item: TokenStream) -> TokenStream { + let parsed_args = parse_macro_input!(attr as MyBenchArgs); + let input = parse_macro_input!(item as ItemFn); + + let mut filtered_args = Vec::new(); + + for arg in parsed_args.args { + match &arg { + Meta::NameValue(MetaNameValue { path, .. }) if path.is_ident("crate") => { + return quote! { + compile_error!("crate argument is not supported with codspeed_divan_compat"); + } + .into(); + } + _ => filtered_args.push(arg), + } + } + + filtered_args.push(syn::parse_quote!(crate = ::codspeed_divan_compat)); + + // Important: keep macro name in sync with re-exported macro name in divan-compat lib + let expanded = quote! { + #[::codspeed_divan_compat::bench_original(#(#filtered_args),*)] + #input + }; + + TokenStream::from(expanded) +} diff --git a/crates/divan_compat/src/compat/bench.rs b/crates/divan_compat/src/compat/bench.rs new file mode 100644 index 00000000..21872451 --- /dev/null +++ b/crates/divan_compat/src/compat/bench.rs @@ -0,0 +1,44 @@ +//! Handpicked stubs from [divan::entry](https://github.com/nvzqz/divan/blob/main/src/entry/mod.rs) +//! Minimally reimplemented in an API compatible way to run the benches using codspeed intrumentation +use codspeed::codspeed::CodSpeed; +use std::{cell::RefCell, rc::Rc}; + +/// Benchmarking options set directly by the user in `#[divan::bench]` and +/// `#[divan::bench_group]`. +/// +/// Changes to fields must be reflected in the "Options" sections of the docs +/// for `#[divan::bench]` and `#[divan::bench_group]`. +#[derive(Default)] +pub struct BenchOptions<'a> { + pub(crate) _marker: std::marker::PhantomData<&'a ()>, +} + +pub struct Bencher<'a, 'b> { + pub(crate) codspeed: Rc>, + pub(crate) uri: String, + pub(crate) _marker: std::marker::PhantomData<&'a &'b ()>, +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, 'b> Bencher<'a, 'b> { + pub(crate) fn new(uri: String) -> Self { + Self { + codspeed: Rc::new(RefCell::new(CodSpeed::new())), + uri, + _marker: std::marker::PhantomData, + } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, 'b> Bencher<'a, 'b> { + pub fn bench(&self, benched: B) + where + B: Fn() -> O + Sync, + { + let mut codspeed = self.codspeed.borrow_mut(); + codspeed.start_benchmark(self.uri.as_str()); + divan::black_box(benched()); + codspeed.end_benchmark(); + } +} diff --git a/crates/divan_compat/src/compat/entry.rs b/crates/divan_compat/src/compat/entry.rs new file mode 100644 index 00000000..4df183a4 --- /dev/null +++ b/crates/divan_compat/src/compat/entry.rs @@ -0,0 +1,144 @@ +//! Handpicked stubs from [divan::entry](https://github.com/nvzqz/divan/blob/main/src/entry/mod.rs) +//! Necessary to be able to use the [divan::bench](https://docs.rs/divan/0.1.17/divan/attr.bench.html) macro without changing it too much +use std::{ + ptr, + sync::{ + atomic::{AtomicPtr, Ordering as AtomicOrdering}, + LazyLock, + }, +}; + +use super::bench::{BenchOptions, Bencher}; + +/// Benchmark entries generated by `#[divan::bench]`. +/// +/// Note: generic-type benchmark entries are instead stored in `GROUP_ENTRIES` +/// in `generic_benches`. +pub static BENCH_ENTRIES: EntryList = EntryList::root(); + +/// Determines how the benchmark entry is run. +#[derive(Clone, Copy)] +pub enum BenchEntryRunner { + /// Benchmark without arguments. + Plain(fn(Bencher)), + // /// Benchmark with runtime arguments. + // Args(fn() -> BenchArgsRunner), +} + +/// Compile-time entry for a benchmark, generated by `#[divan::bench]`. +pub struct BenchEntry { + /// Entry metadata. + pub meta: EntryMeta, + + /// The benchmarking function. + pub bench: BenchEntryRunner, +} + +/// Metadata common to `#[divan::bench]` and `#[divan::bench_group]`. +pub struct EntryMeta { + /// The entry's display name. + pub display_name: &'static str, + + /// The entry's original name. + /// + /// This is used to find a `GroupEntry` for a `BenchEntry`. + pub raw_name: &'static str, + + /// The entry's raw `module_path!()`. + pub module_path: &'static str, + + /// Where the entry was defined. + pub location: EntryLocation, + /// Configures the benchmarker via attribute options. + pub bench_options: Option>>, +} + +/// Where an entry is located. +#[derive(Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +#[allow(missing_docs)] +pub struct EntryLocation { + pub file: &'static str, + pub line: u32, + pub col: u32, +} + +/// Linked list of entries. +/// +/// This is implemented in a thread-safe way despite the fact that constructors +/// are run single-threaded. +pub struct EntryList { + entry: Option<&'static T>, + next: AtomicPtr, +} + +impl EntryList { + pub(crate) const fn root() -> Self { + Self { + entry: None, + next: AtomicPtr::new(ptr::null_mut()), + } + } + + /// Dereferences the `next` pointer. + #[inline] + fn next(&self) -> Option<&Self> { + // SAFETY: `next` is only assigned by `push`, which always receives a + // 'static lifetime. + unsafe { self.next.load(AtomicOrdering::Relaxed).as_ref() } + } +} + +// Externally used by macros or tests. +#[allow(missing_docs)] +impl EntryList { + #[inline] + pub const fn new(entry: &'static T) -> Self { + Self { + entry: Some(entry), + next: AtomicPtr::new(ptr::null_mut()), + } + } + + /// Creates an iterator over entries in `self`. + #[inline] + pub fn iter(&self) -> impl Iterator { + let mut list = Some(self); + std::iter::from_fn(move || -> Option> { + let current = list?; + list = current.next(); + Some(current.entry.as_ref().copied()) + }) + .flatten() + } + + /// Inserts `other` to the front of the list. + /// + /// # Safety + /// + /// This function must be safe to call before `main`. + #[inline] + pub fn push(&'static self, other: &'static Self) { + let mut old_next = self.next.load(AtomicOrdering::Relaxed); + loop { + // Each publicly-created instance has `list.next` be null, so we can + // simply store `self.next` there. + other.next.store(old_next, AtomicOrdering::Release); + + // SAFETY: The content of `other` can already be seen, so we don't + // need to strongly order reads into it. + let other = other as *const Self as *mut Self; + match self.next.compare_exchange_weak( + old_next, + other, + AtomicOrdering::AcqRel, + AtomicOrdering::Acquire, + ) { + // Successfully wrote our thread's value to the list. + Ok(_) => return, + + // Lost the race, store winner's value in `other.next`. + Err(new) => old_next = new, + } + } + } +} diff --git a/crates/divan_compat/src/compat/mod.rs b/crates/divan_compat/src/compat/mod.rs new file mode 100644 index 00000000..88a1102a --- /dev/null +++ b/crates/divan_compat/src/compat/mod.rs @@ -0,0 +1,40 @@ +// Used by generated code. Not public API and thus not subject to SemVer. +#[doc(hidden)] +#[path = "private.rs"] +pub mod __private; + +mod bench; +mod entry; + +pub fn main() { + // 1. Get registered entries + // TODO: Manage bench groups + + // TODO: remove when releasing divan with instrumentation mode + todo!("Instrumentation mode with divan is not yet available."); + #[allow(unreachable_code)] + let bench_entries = &entry::BENCH_ENTRIES; + + // 2. Build an execution tree + // TODO: + + // 3. Filter the tree then sort it (drop sort?) + // TODO: + + // 4. Scan the tree and execute benchmarks + // TODO: + + for entry in bench_entries.iter() { + match entry.bench { + entry::BenchEntryRunner::Plain(bench_fn) => { + bench_fn(bench::Bencher::new(format!( + "{}:{}::{}::{}", + entry.meta.location.file, + entry.meta.location.line, + entry.meta.module_path, + entry.meta.display_name + ))); + } + } + } +} diff --git a/crates/divan_compat/src/compat/private.rs b/crates/divan_compat/src/compat/private.rs new file mode 100644 index 00000000..fd5dfa61 --- /dev/null +++ b/crates/divan_compat/src/compat/private.rs @@ -0,0 +1,3 @@ +pub use super::entry::{ + BenchEntry, BenchEntryRunner, EntryList, EntryLocation, EntryMeta, BENCH_ENTRIES, +}; diff --git a/crates/divan_compat/src/lib.rs b/crates/divan_compat/src/lib.rs new file mode 100644 index 00000000..d9b5c09c --- /dev/null +++ b/crates/divan_compat/src/lib.rs @@ -0,0 +1,21 @@ +pub use codspeed::codspeed_uri; + +#[cfg(not(codspeed))] +mod compat_divan { + pub use divan::*; +} + +#[cfg(codspeed)] +#[path = "."] +mod compat_divan { + pub use divan::black_box; + + pub use codspeed_divan_compat_macros::bench_compat as bench; + // Important: Keep in sync with the name used in the compat macro + pub use divan::bench as bench_original; + + mod compat; + pub use compat::*; +} + +pub use compat_divan::*;