From c8ff0b642a536b1977bbd73b9fa6b047db2bd0b2 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Mon, 6 Mar 2023 16:21:58 -0800 Subject: [PATCH 01/29] document SDK makefile targets --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 389a08d99..6c788a05d 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,15 @@ DEPS-FLAGS=--check-pkg-deps --unused-pkg-deps help: @echo "install - install package along with dependencies" + @echo "install-sdk - install the SDK which includes developer tools" @echo "remove - remove package" + @echo "remove-sdk - remove SDK; this will not remove SDK dependencies" @echo "build - Compile libraries" @echo "build-docs - Build docs" @echo "build-standalone-docs - Build self-contained docs that could be hosted somewhere" @echo "build-all - Compile libraries, build docs, and check dependencies" @echo "clean - remove all build artifacts" + @echo "clean-sdk - remove all build artifacts in SDK paths" @echo "check-deps - check dependencies" @echo "test - run tests" @echo "test-with-errortrace - run tests with error tracing" From dd7051dbb20cdcd53dd8c6662570e5f13a22c286 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 11:41:58 -0800 Subject: [PATCH 02/29] makefile target for performance regression report --- Makefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6c788a05d..3a94763a8 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,11 @@ help: @echo "profile-competitive - Run competitive benchmarks" @echo "profile-forms - Run benchmarks for individual Qi forms" @echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)" - @echo "form-performance-report - Run benchmarks for Qi forms and produce results for use in CI" + @echo "form-performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression" + @echo " For use in regression: make form-performance-report > /path/to/before.json" + @echo "performance-regression-report - Run benchmarks for Qi forms against a reference report." + @echo " make performance-regression-report REF=/path/to/before.json" + # Primarily for use by CI. # Installs dependencies as well as linking this as a package. @@ -181,4 +185,7 @@ profile: profile-competitive profile-forms form-performance-report: @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile form-performance-report +performance-regression-report: + @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF) + +.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile form-performance-report performance-regression-report From 32116d8188c2e87a75fbd551b4ae437525dfe3b0 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 18:17:35 -0800 Subject: [PATCH 03/29] Refactor benchmarks to unify form-related ones --- .../{forms-base.rkt => forms/base.rkt} | 6 +- .../{forms.rkt => forms/benchmarks.rkt} | 315 ++++-------------- qi-sdk/profile/{ => forms}/loadlib.rkt | 0 qi-sdk/profile/{ => forms}/regression.rkt | 0 qi-sdk/profile/forms/report.rkt | 246 ++++++++++++++ qi-sdk/profile/report.rkt | 199 ----------- 6 files changed, 309 insertions(+), 457 deletions(-) rename qi-sdk/profile/{forms-base.rkt => forms/base.rkt} (69%) rename qi-sdk/profile/{forms.rkt => forms/benchmarks.rkt} (63%) rename qi-sdk/profile/{ => forms}/loadlib.rkt (100%) rename qi-sdk/profile/{ => forms}/regression.rkt (100%) create mode 100755 qi-sdk/profile/forms/report.rkt delete mode 100755 qi-sdk/profile/report.rkt diff --git a/qi-sdk/profile/forms-base.rkt b/qi-sdk/profile/forms/base.rkt similarity index 69% rename from qi-sdk/profile/forms-base.rkt rename to qi-sdk/profile/forms/base.rkt index 707bc19a3..7431b112f 100644 --- a/qi-sdk/profile/forms-base.rkt +++ b/qi-sdk/profile/forms/base.rkt @@ -2,11 +2,9 @@ (provide (all-from-out racket/base) (all-from-out qi) - (all-from-out "util.rkt") + (all-from-out "../util.rkt") sqr) (require qi - "util.rkt" + "../util.rkt" (only-in math sqr)) - - diff --git a/qi-sdk/profile/forms.rkt b/qi-sdk/profile/forms/benchmarks.rkt similarity index 63% rename from qi-sdk/profile/forms.rkt rename to qi-sdk/profile/forms/benchmarks.rkt index e747637a3..27fc22865 100755 --- a/qi-sdk/profile/forms.rkt +++ b/qi-sdk/profile/forms/benchmarks.rkt @@ -1,32 +1,7 @@ #!/usr/bin/env racket #lang racket/base -#| -To add a benchmark for a new form: - -1. Add a submodule for it which provides a `run` function taking no -arguments. This function will be expected to exercise the new form and -return a time taken. The `run` function typically uses one of the -utility macros `run-benchmark` or `run-summary-benchmark`, and -provides it one of the helper functions `check-value` (to invoke the -form with a single value each time during benchmarking) or -`check-values` (to invoke the form with multiple values each time -during benchmarking). Note that at the moment, as a hack for convenience, -`run-benchmark` expects a function with the name of the form being -benchmarked _prefixed with tilde_. This is to avoid name collisions -between this function and the Qi form with the same name. Basically, -just follow one of the numerous examples in this module to see what -this is referring to. - -2. Require the submodule in the `main` submodule with an appropriate -prefix (see other examples) - -3. Add the required `run` function to the `env` hash in the main -submodule. This will ensure that it gets picked up when the benchmarks -for the forms are run. -|# - -(module one-of? "forms-base.rkt" +(module one-of? "base.rkt" (provide run) (define (~one-of? v) @@ -38,7 +13,7 @@ for the forms are run. check-value 100000))) -(module and "forms-base.rkt" +(module and "base.rkt" (provide run) (define (~and v) @@ -50,7 +25,7 @@ for the forms are run. check-value 200000))) -(module or "forms-base.rkt" +(module or "base.rkt" (provide run) (define (~or v) @@ -62,7 +37,7 @@ for the forms are run. check-value 200000))) -(module not "forms-base.rkt" +(module not "base.rkt" (provide run) (define (~not v) @@ -74,7 +49,7 @@ for the forms are run. check-value 200000))) -(module and% "forms-base.rkt" +(module and% "base.rkt" (provide run) (define (~and% a b) @@ -86,7 +61,7 @@ for the forms are run. check-two-values 200000))) -(module or% "forms-base.rkt" +(module or% "base.rkt" (provide run) (define (~or% a b) @@ -98,7 +73,7 @@ for the forms are run. check-two-values 200000))) -(module group "forms-base.rkt" +(module group "base.rkt" (provide run) (define (~group . vs) @@ -114,7 +89,7 @@ for the forms are run. check-values 200000))) -(module count "forms-base.rkt" +(module count "base.rkt" (provide run) (define (~count . vs) @@ -127,7 +102,7 @@ for the forms are run. check-values 1000000))) -(module relay "forms-base.rkt" +(module relay "base.rkt" (provide run) (define (~relay . vs) @@ -149,7 +124,7 @@ for the forms are run. check-values 50000))) -(module relay* "forms-base.rkt" +(module relay* "base.rkt" (provide run) (define (~relay* . vs) @@ -165,7 +140,7 @@ for the forms are run. check-values 50000))) -(module amp "forms-base.rkt" +(module amp "base.rkt" (provide run) (define (~amp . vs) @@ -178,7 +153,7 @@ for the forms are run. check-values 300000))) -(module ground "forms-base.rkt" +(module ground "base.rkt" (provide run) (define (~ground . vs) @@ -191,7 +166,7 @@ for the forms are run. check-values 200000))) -(module thread "forms-base.rkt" +(module thread "base.rkt" (provide run) (define (~thread . vs) @@ -214,7 +189,7 @@ for the forms are run. check-values 200000))) -(module thread-right "forms-base.rkt" +(module thread-right "base.rkt" (provide run) (define (~thread-right . vs) @@ -237,7 +212,7 @@ for the forms are run. check-values 200000))) -(module crossover "forms-base.rkt" +(module crossover "base.rkt" (provide run) (define (~crossover . vs) @@ -250,7 +225,7 @@ for the forms are run. check-values 200000))) -(module all "forms-base.rkt" +(module all "base.rkt" (provide run) (define (~all . vs) @@ -263,7 +238,7 @@ for the forms are run. check-values 200000))) -(module any "forms-base.rkt" +(module any "base.rkt" (provide run) (define (~any . vs) @@ -276,7 +251,7 @@ for the forms are run. check-values 200000))) -(module none "forms-base.rkt" +(module none "base.rkt" (provide run) (define (~none . vs) @@ -289,7 +264,7 @@ for the forms are run. check-values 200000))) -(module all? "forms-base.rkt" +(module all? "base.rkt" (provide run) (define (~all? . vs) @@ -302,7 +277,7 @@ for the forms are run. check-values 200000))) -(module any? "forms-base.rkt" +(module any? "base.rkt" (provide run) (define (~any? . vs) @@ -315,7 +290,7 @@ for the forms are run. check-values 200000))) -(module none? "forms-base.rkt" +(module none? "base.rkt" (provide run) (define (~none? . vs) @@ -328,7 +303,7 @@ for the forms are run. check-values 200000))) -(module collect "forms-base.rkt" +(module collect "base.rkt" (provide run) (define (~collect . vs) @@ -341,7 +316,7 @@ for the forms are run. check-values 1000000))) -(module sep "forms-base.rkt" +(module sep "base.rkt" (provide run) (define (~sep v) @@ -353,7 +328,7 @@ for the forms are run. check-list 1000000))) -(module gen "forms-base.rkt" +(module gen "base.rkt" (provide run) (define (~gen . vs) @@ -366,7 +341,7 @@ for the forms are run. check-values 1000000))) -(module esc "forms-base.rkt" +(module esc "base.rkt" (provide run) (define (~esc . vs) @@ -379,7 +354,7 @@ for the forms are run. check-values 1000000))) -(module AND "forms-base.rkt" +(module AND "base.rkt" (provide run) (define (~AND . vs) @@ -392,7 +367,7 @@ for the forms are run. check-values 200000))) -(module OR "forms-base.rkt" +(module OR "base.rkt" (provide run) (define (~OR . vs) @@ -405,7 +380,7 @@ for the forms are run. check-values 200000))) -(module NOT "forms-base.rkt" +(module NOT "base.rkt" (provide run) (define (~NOT v) @@ -417,7 +392,7 @@ for the forms are run. check-value 200000))) -(module NAND "forms-base.rkt" +(module NAND "base.rkt" (provide run) (define (~NAND . vs) @@ -430,7 +405,7 @@ for the forms are run. check-values 200000))) -(module NOR "forms-base.rkt" +(module NOR "base.rkt" (provide run) (define (~NOR . vs) @@ -443,7 +418,7 @@ for the forms are run. check-values 200000))) -(module XOR "forms-base.rkt" +(module XOR "base.rkt" (provide run) (define (~XOR . vs) @@ -456,7 +431,7 @@ for the forms are run. check-values 200000))) -(module XNOR "forms-base.rkt" +(module XNOR "base.rkt" (provide run) (define (~XNOR . vs) @@ -469,7 +444,7 @@ for the forms are run. check-values 200000))) -(module tee "forms-base.rkt" +(module tee "base.rkt" (provide run) (define (~tee v) @@ -481,7 +456,7 @@ for the forms are run. check-value 200000))) -(module try "forms-base.rkt" +(module try "base.rkt" (provide run) (define (try-happy . vs) @@ -504,7 +479,7 @@ for the forms are run. (try-happy check-values 20000) (try-error check-values 20000)))) -(module currying "forms-base.rkt" +(module currying "base.rkt" (provide run) (define (currying . vs) @@ -515,7 +490,7 @@ for the forms are run. check-values 200000))) -(module template "forms-base.rkt" +(module template "base.rkt" (provide run) (define (template . vs) @@ -526,7 +501,7 @@ for the forms are run. check-values 200000))) -(module catchall-template "forms-base.rkt" +(module catchall-template "base.rkt" (provide run) (define (catchall-template . vs) @@ -537,7 +512,7 @@ for the forms are run. check-values 200000))) -(module if "forms-base.rkt" +(module if "base.rkt" (provide run) (define (~if . vs) @@ -549,7 +524,7 @@ for the forms are run. check-values 500000))) -(module when "forms-base.rkt" +(module when "base.rkt" (provide run) (define (~when . vs) @@ -561,7 +536,7 @@ for the forms are run. check-values 500000))) -(module unless "forms-base.rkt" +(module unless "base.rkt" (provide run) (define (~unless . vs) @@ -573,7 +548,7 @@ for the forms are run. check-values 500000))) -(module switch "forms-base.rkt" +(module switch "base.rkt" (provide run) (define (switch-basic . vs) @@ -601,7 +576,7 @@ for the forms are run. (switch-else check-values 200000) (switch-divert check-values 200000)))) -(module sieve "forms-base.rkt" +(module sieve "base.rkt" (provide run) (define (~sieve . vs) @@ -613,7 +588,7 @@ for the forms are run. check-values 100000))) -(module partition "forms-base.rkt" +(module partition "base.rkt" (provide run) (define (~partition . vs) (apply (flow (partition [negative? *] @@ -623,7 +598,7 @@ for the forms are run. (define (run) (run-benchmark ~partition check-values 100000))) -(module gate "forms-base.rkt" +(module gate "base.rkt" (provide run) (define (~gate . vs) @@ -635,7 +610,7 @@ for the forms are run. check-values 500000))) -(module input-aliases "forms-base.rkt" +(module input-aliases "base.rkt" (provide run) (define (input-alias-1 . vs) @@ -663,7 +638,7 @@ for the forms are run. check-values 100000)))) -(module fanout "forms-base.rkt" +(module fanout "base.rkt" (provide run) (define (fanout-small-n . vs) @@ -684,7 +659,7 @@ for the forms are run. check-values 20000)))) -(module inverter "forms-base.rkt" +(module inverter "base.rkt" (provide run) (define (~inverter . vs) @@ -696,7 +671,7 @@ for the forms are run. check-values 200000))) -(module feedback "forms-base.rkt" +(module feedback "base.rkt" (provide run) (define (feedback-number . vs) @@ -725,7 +700,7 @@ for the forms are run. check-value 70000)))) -(module select "forms-base.rkt" +(module select "base.rkt" (provide run) (define (~select . vs) @@ -737,7 +712,7 @@ for the forms are run. check-values 20000))) -(module block "forms-base.rkt" +(module block "base.rkt" (provide run) (define (~block . vs) @@ -749,7 +724,7 @@ for the forms are run. check-values 20000))) -(module bundle "forms-base.rkt" +(module bundle "base.rkt" (provide run) (define (~bundle . vs) @@ -761,7 +736,7 @@ for the forms are run. check-values 20000))) -(module effect "forms-base.rkt" +(module effect "base.rkt" (provide run) (define (~effect . vs) @@ -773,7 +748,7 @@ for the forms are run. check-values 200000))) -(module live? "forms-base.rkt" +(module live? "base.rkt" (provide run) (define (~live? . vs) @@ -785,7 +760,7 @@ for the forms are run. check-values 500000))) -(module rectify "forms-base.rkt" +(module rectify "base.rkt" (provide run) (define (~rectify . vs) @@ -797,7 +772,7 @@ for the forms are run. check-values 500000))) -(module pass "forms-base.rkt" +(module pass "base.rkt" (provide run) (define (~pass . vs) @@ -809,7 +784,7 @@ for the forms are run. check-values 200000))) -(module foldl "forms-base.rkt" +(module foldl "base.rkt" (provide run) (define (~foldl . vs) @@ -821,7 +796,7 @@ for the forms are run. check-values 200000))) -(module foldr "forms-base.rkt" +(module foldr "base.rkt" (provide run) (define (~foldr . vs) @@ -833,7 +808,7 @@ for the forms are run. check-values 200000))) -(module loop "forms-base.rkt" +(module loop "base.rkt" (provide run) (define (~loop . vs) @@ -845,7 +820,7 @@ for the forms are run. check-values 100000))) -(module loop2 "forms-base.rkt" +(module loop2 "base.rkt" (provide run) (define (~loop2 . vs) @@ -860,7 +835,7 @@ for the forms are run. check-values 100000))) -(module apply "forms-base.rkt" +(module apply "base.rkt" (provide run) (require (only-in racket/base @@ -875,7 +850,7 @@ for the forms are run. check-values 300000))) -(module clos "forms-base.rkt" +(module clos "base.rkt" (provide run) ;; TODO: this uses a lot of other things besides `clos` and is @@ -889,171 +864,3 @@ for the forms are run. (run-benchmark ~clos check-values 100000))) - -;; To run benchmarks for a form interactively, use e.g.: -;; (require (submod "." fanout)) -;; (run) - -(module* main cli - - (require - (prefix-in one-of?: (submod ".." one-of?)) - (prefix-in and: (submod ".." and)) - (prefix-in or: (submod ".." or)) - (prefix-in not: (submod ".." not)) - (prefix-in and%: (submod ".." and%)) - (prefix-in or%: (submod ".." or%)) - (prefix-in group: (submod ".." group)) - (prefix-in count: (submod ".." count)) - (prefix-in relay: (submod ".." relay)) - (prefix-in relay*: (submod ".." relay*)) - (prefix-in amp: (submod ".." amp)) - (prefix-in ground: (submod ".." ground)) - (prefix-in thread: (submod ".." thread)) - (prefix-in thread-right: (submod ".." thread-right)) - (prefix-in crossover: (submod ".." crossover)) - (prefix-in all: (submod ".." all)) - (prefix-in any: (submod ".." any)) - (prefix-in none: (submod ".." none)) - (prefix-in all?: (submod ".." all?)) - (prefix-in any?: (submod ".." any?)) - (prefix-in none?: (submod ".." none?)) - (prefix-in collect: (submod ".." collect)) - (prefix-in sep: (submod ".." sep)) - (prefix-in gen: (submod ".." gen)) - (prefix-in esc: (submod ".." esc)) - (prefix-in AND: (submod ".." AND)) - (prefix-in OR: (submod ".." OR)) - (prefix-in NOT: (submod ".." NOT)) - (prefix-in NAND: (submod ".." NAND)) - (prefix-in NOR: (submod ".." NOR)) - (prefix-in XOR: (submod ".." XOR)) - (prefix-in XNOR: (submod ".." XNOR)) - (prefix-in tee: (submod ".." tee)) - (prefix-in try: (submod ".." try)) - (prefix-in currying: (submod ".." currying)) - (prefix-in template: (submod ".." template)) - (prefix-in catchall-template: (submod ".." catchall-template)) - (prefix-in if: (submod ".." if)) - (prefix-in when: (submod ".." when)) - (prefix-in unless: (submod ".." unless)) - (prefix-in switch: (submod ".." switch)) - (prefix-in sieve: (submod ".." sieve)) - (prefix-in partition: (submod ".." partition)) - (prefix-in gate: (submod ".." gate)) - (prefix-in input-aliases: (submod ".." input-aliases)) - (prefix-in fanout: (submod ".." fanout)) - (prefix-in inverter: (submod ".." inverter)) - (prefix-in feedback: (submod ".." feedback)) - (prefix-in select: (submod ".." select)) - (prefix-in block: (submod ".." block)) - (prefix-in bundle: (submod ".." bundle)) - (prefix-in effect: (submod ".." effect)) - (prefix-in live?: (submod ".." live?)) - (prefix-in rectify: (submod ".." rectify)) - (prefix-in pass: (submod ".." pass)) - (prefix-in foldl: (submod ".." foldl)) - (prefix-in foldr: (submod ".." foldr)) - (prefix-in loop: (submod ".." loop)) - (prefix-in loop2: (submod ".." loop2)) - (prefix-in apply: (submod ".." apply)) - (prefix-in clos: (submod ".." clos))) - - (require racket/match - racket/format - relation - qi - (only-in "util.rkt" - only-if - for/call)) - - ;; It would be great if we could get the value of a variable - ;; by using its (string) name, but (eval (string->symbol name)) - ;; doesn't find it. So instead, we reify the "lexical environment" - ;; here manually, so that the values can be looked up at runtime - ;; based on the string names (note that the value is always the key - ;; + ":" + "run") - (define env - (hash - "one-of?" one-of?:run - "and" and:run - "or" or:run - "not" not:run - "and%" and%:run - "or%" or%:run - "group" group:run - "count" count:run - "relay" relay:run - "relay*" relay*:run - "amp" amp:run - "ground" ground:run - "thread" thread:run - "thread-right" thread-right:run - "crossover" crossover:run - "all" all:run - "any" any:run - "none" none:run - "all?" all?:run - "any?" any?:run - "none?" none?:run - "collect" collect:run - "sep" sep:run - "gen" gen:run - "esc" esc:run - "AND" AND:run - "OR" OR:run - "NOT" NOT:run - "NAND" NAND:run - "NOR" NOR:run - "XOR" XOR:run - "XNOR" XNOR:run - "tee" tee:run - "try" try:run - "currying" currying:run - "template" template:run - "catchall-template" catchall-template:run - "if" if:run - "when" when:run - "unless" unless:run - "switch" switch:run - "sieve" sieve:run - "partition" partition:run - "gate" gate:run - "input-aliases" input-aliases:run - "fanout" fanout:run - "inverter" inverter:run - "feedback" feedback:run - "select" select:run - "block" block:run - "bundle" bundle:run - "effect" effect:run - "live?" live?:run - "rectify" rectify:run - "pass" pass:run - "foldl" foldl:run - "foldr" foldr:run - "loop" loop:run - "loop2" loop2:run - "apply" apply:run - "clos" clos:run)) - - (flag (forms #:param [forms null] name) - ("-f" "--form" "Forms to benchmark") - (forms (cons name (forms)))) - - (constraint (multi forms)) - - (help - (usage (~a "Run benchmarks for individual Qi forms " - "(by default, all of them)."))) - - (program (main) - (let ([fs (~>> ((forms)) - (only-if null? - (gen (hash-keys env))) - (sort <))]) - (for ([f fs]) - (match-let ([(list name ms) ((hash-ref env f))]) - (displayln (~a name ": " ms " ms")))))) - - (run main)) diff --git a/qi-sdk/profile/loadlib.rkt b/qi-sdk/profile/forms/loadlib.rkt similarity index 100% rename from qi-sdk/profile/loadlib.rkt rename to qi-sdk/profile/forms/loadlib.rkt diff --git a/qi-sdk/profile/regression.rkt b/qi-sdk/profile/forms/regression.rkt similarity index 100% rename from qi-sdk/profile/regression.rkt rename to qi-sdk/profile/forms/regression.rkt diff --git a/qi-sdk/profile/forms/report.rkt b/qi-sdk/profile/forms/report.rkt new file mode 100755 index 000000000..e32073740 --- /dev/null +++ b/qi-sdk/profile/forms/report.rkt @@ -0,0 +1,246 @@ +#!/usr/bin/env racket +#lang cli + +#| +To add a benchmark for a new form: + +1. Add a submodule for it in benchmarks.rkt which provides a `run` +function taking no arguments. This function will be expected to +exercise the new form and return a time taken. The `run` function +typically uses one of the utility macros `run-benchmark` or +`run-summary-benchmark`, and provides it one of the helper functions +`check-value` (to invoke the form with a single value each time during +benchmarking) or `check-values` (to invoke the form with multiple +values each time during benchmarking). Note that at the moment, as a +hack for convenience, `run-benchmark` expects a function with the name +of the form being benchmarked _prefixed with tilde_. This is to avoid +name collisions between this function and the Qi form with the same +name. Basically, just follow one of the numerous examples in this +module to see what this is referring to. + +2. Require the submodule in the present module with an appropriate +prefix (see other examples) + +3. Add the required `run` function to the `env` hash below. This will +ensure that it gets picked up when the benchmarks for the forms are +run. +|# + +(require + (prefix-in one-of?: (submod "benchmarks.rkt" one-of?)) + (prefix-in and: (submod "benchmarks.rkt" and)) + (prefix-in or: (submod "benchmarks.rkt" or)) + (prefix-in not: (submod "benchmarks.rkt" not)) + (prefix-in and%: (submod "benchmarks.rkt" and%)) + (prefix-in or%: (submod "benchmarks.rkt" or%)) + (prefix-in group: (submod "benchmarks.rkt" group)) + (prefix-in count: (submod "benchmarks.rkt" count)) + (prefix-in relay: (submod "benchmarks.rkt" relay)) + (prefix-in relay*: (submod "benchmarks.rkt" relay*)) + (prefix-in amp: (submod "benchmarks.rkt" amp)) + (prefix-in ground: (submod "benchmarks.rkt" ground)) + (prefix-in thread: (submod "benchmarks.rkt" thread)) + (prefix-in thread-right: (submod "benchmarks.rkt" thread-right)) + (prefix-in crossover: (submod "benchmarks.rkt" crossover)) + (prefix-in all: (submod "benchmarks.rkt" all)) + (prefix-in any: (submod "benchmarks.rkt" any)) + (prefix-in none: (submod "benchmarks.rkt" none)) + (prefix-in all?: (submod "benchmarks.rkt" all?)) + (prefix-in any?: (submod "benchmarks.rkt" any?)) + (prefix-in none?: (submod "benchmarks.rkt" none?)) + (prefix-in collect: (submod "benchmarks.rkt" collect)) + (prefix-in sep: (submod "benchmarks.rkt" sep)) + (prefix-in gen: (submod "benchmarks.rkt" gen)) + (prefix-in esc: (submod "benchmarks.rkt" esc)) + (prefix-in AND: (submod "benchmarks.rkt" AND)) + (prefix-in OR: (submod "benchmarks.rkt" OR)) + (prefix-in NOT: (submod "benchmarks.rkt" NOT)) + (prefix-in NAND: (submod "benchmarks.rkt" NAND)) + (prefix-in NOR: (submod "benchmarks.rkt" NOR)) + (prefix-in XOR: (submod "benchmarks.rkt" XOR)) + (prefix-in XNOR: (submod "benchmarks.rkt" XNOR)) + (prefix-in tee: (submod "benchmarks.rkt" tee)) + (prefix-in try: (submod "benchmarks.rkt" try)) + (prefix-in currying: (submod "benchmarks.rkt" currying)) + (prefix-in template: (submod "benchmarks.rkt" template)) + (prefix-in catchall-template: (submod "benchmarks.rkt" catchall-template)) + (prefix-in if: (submod "benchmarks.rkt" if)) + (prefix-in when: (submod "benchmarks.rkt" when)) + (prefix-in unless: (submod "benchmarks.rkt" unless)) + (prefix-in switch: (submod "benchmarks.rkt" switch)) + (prefix-in sieve: (submod "benchmarks.rkt" sieve)) + (prefix-in partition: (submod "benchmarks.rkt" partition)) + (prefix-in gate: (submod "benchmarks.rkt" gate)) + (prefix-in input-aliases: (submod "benchmarks.rkt" input-aliases)) + (prefix-in fanout: (submod "benchmarks.rkt" fanout)) + (prefix-in inverter: (submod "benchmarks.rkt" inverter)) + (prefix-in feedback: (submod "benchmarks.rkt" feedback)) + (prefix-in select: (submod "benchmarks.rkt" select)) + (prefix-in block: (submod "benchmarks.rkt" block)) + (prefix-in bundle: (submod "benchmarks.rkt" bundle)) + (prefix-in effect: (submod "benchmarks.rkt" effect)) + (prefix-in live?: (submod "benchmarks.rkt" live?)) + (prefix-in rectify: (submod "benchmarks.rkt" rectify)) + (prefix-in pass: (submod "benchmarks.rkt" pass)) + (prefix-in foldl: (submod "benchmarks.rkt" foldl)) + (prefix-in foldr: (submod "benchmarks.rkt" foldr)) + (prefix-in loop: (submod "benchmarks.rkt" loop)) + (prefix-in loop2: (submod "benchmarks.rkt" loop2)) + (prefix-in apply: (submod "benchmarks.rkt" apply)) + (prefix-in clos: (submod "benchmarks.rkt" clos))) + +(require "loadlib.rkt" + "regression.rkt") + +(require racket/match + racket/format + relation + qi + json + csv-writing + (only-in "../util.rkt" + only-if + for/call)) + +;; It would be great if we could get the value of a variable +;; by using its (string) name, but (eval (string->symbol name)) +;; doesn't find it. So instead, we reify the "lexical environment" +;; here manually, so that the values can be looked up at runtime +;; based on the string names (note that the value is always the key +;; + ":" + "run") +(define env + (hash + "one-of?" one-of?:run + "and" and:run + "or" or:run + "not" not:run + "and%" and%:run + "or%" or%:run + "group" group:run + "count" count:run + "relay" relay:run + "relay*" relay*:run + "amp" amp:run + "ground" ground:run + "thread" thread:run + "thread-right" thread-right:run + "crossover" crossover:run + "all" all:run + "any" any:run + "none" none:run + "all?" all?:run + "any?" any?:run + "none?" none?:run + "collect" collect:run + "sep" sep:run + "gen" gen:run + "esc" esc:run + "AND" AND:run + "OR" OR:run + "NOT" NOT:run + "NAND" NAND:run + "NOR" NOR:run + "XOR" XOR:run + "XNOR" XNOR:run + "tee" tee:run + "try" try:run + "currying" currying:run + "template" template:run + "catchall-template" catchall-template:run + "if" if:run + "when" when:run + "unless" unless:run + "switch" switch:run + "sieve" sieve:run + "partition" partition:run + "gate" gate:run + "input-aliases" input-aliases:run + "fanout" fanout:run + "inverter" inverter:run + "feedback" feedback:run + "select" select:run + "block" block:run + "bundle" bundle:run + "effect" effect:run + "live?" live?:run + "rectify" rectify:run + "pass" pass:run + "foldl" foldl:run + "foldr" foldr:run + "loop" loop:run + "loop2" loop2:run + "apply" apply:run + "clos" clos:run)) + +(define (write-csv data) + (~> (data) + △ + (>< (~> (-< (hash-ref 'name) + (hash-ref 'unit) + (hash-ref 'value)) + ▽)) + (-< '(name unit value) + _) + ▽ + display-table)) + +(flag (forms #:param [forms null] name) + ("-f" "--form" "Forms to benchmark") + (forms (cons name (forms)))) + +(constraint (multi forms)) + +(help + (usage + (~a "Run benchmarks for individual Qi forms " + "(by default, all of them), reporting the results " + "in a configurable output format."))) + +(flag (output-format #:param [output-format ""] fmt) + ("-o" + "--format" + "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") + (output-format fmt)) + +(flag (regression-file #:param [regression-file #f] reg-file) + ("-r" "--regression" "'Before' data to compute regression against") + (regression-file reg-file)) + +(define (format-output output) + ;; Note: this is a case where declaring "constraints" on the CLI args + ;; would be useful, instead of using the ad hoc fallback `else` check here + ;; https://github.com/countvajhula/cli/issues/6 + (cond + [(equal? (output-format) "json") (write-json output)] + [(equal? (output-format) "csv") (write-csv output)] + [(equal? (output-format) "") (values)] + [else (error (~a "Unrecognized format: " (output-format) "!"))])) + +(program (main) + (define fs (~>> ((forms)) + (only-if null? + (gen (hash-keys env))) + (sort <))) + (define forms-data (for/list ([f (in-list fs)]) + (match-let ([(list name ms) ((hash-ref env f))]) + ;; Print results "live" to STDERR, with + ;; only the actual output (if desired) + ;; going to STDOUT at the end. + (displayln (~a name ": " ms " ms") + (current-error-port)) + (hash 'name name 'unit "ms" 'value ms)))) + (define require-data (list (hash 'name "(require qi)" + 'unit "ms" + 'value (time-module-ms "qi")))) + (let ([output (append forms-data require-data)]) + + (if (regression-file) + (let ([before (parse-benchmarks (parse-json-file (regression-file)))] + [after (parse-benchmarks output)]) + (compute-regression before after)) + (format-output output)))) + +;; To run benchmarks for a form interactively, use e.g.: +;; (run main #("-f" "fanout")) + +(run main) diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt deleted file mode 100755 index 214b1f4d2..000000000 --- a/qi-sdk/profile/report.rkt +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env racket -#lang cli - -(require - (prefix-in one-of?: (submod "forms.rkt" one-of?)) - (prefix-in and: (submod "forms.rkt" and)) - (prefix-in or: (submod "forms.rkt" or)) - (prefix-in not: (submod "forms.rkt" not)) - (prefix-in and%: (submod "forms.rkt" and%)) - (prefix-in or%: (submod "forms.rkt" or%)) - (prefix-in group: (submod "forms.rkt" group)) - (prefix-in count: (submod "forms.rkt" count)) - (prefix-in relay: (submod "forms.rkt" relay)) - (prefix-in relay*: (submod "forms.rkt" relay*)) - (prefix-in amp: (submod "forms.rkt" amp)) - (prefix-in ground: (submod "forms.rkt" ground)) - (prefix-in thread: (submod "forms.rkt" thread)) - (prefix-in thread-right: (submod "forms.rkt" thread-right)) - (prefix-in crossover: (submod "forms.rkt" crossover)) - (prefix-in all: (submod "forms.rkt" all)) - (prefix-in any: (submod "forms.rkt" any)) - (prefix-in none: (submod "forms.rkt" none)) - (prefix-in all?: (submod "forms.rkt" all?)) - (prefix-in any?: (submod "forms.rkt" any?)) - (prefix-in none?: (submod "forms.rkt" none?)) - (prefix-in collect: (submod "forms.rkt" collect)) - (prefix-in sep: (submod "forms.rkt" sep)) - (prefix-in gen: (submod "forms.rkt" gen)) - (prefix-in esc: (submod "forms.rkt" esc)) - (prefix-in AND: (submod "forms.rkt" AND)) - (prefix-in OR: (submod "forms.rkt" OR)) - (prefix-in NOT: (submod "forms.rkt" NOT)) - (prefix-in NAND: (submod "forms.rkt" NAND)) - (prefix-in NOR: (submod "forms.rkt" NOR)) - (prefix-in XOR: (submod "forms.rkt" XOR)) - (prefix-in XNOR: (submod "forms.rkt" XNOR)) - (prefix-in tee: (submod "forms.rkt" tee)) - (prefix-in try: (submod "forms.rkt" try)) - (prefix-in currying: (submod "forms.rkt" currying)) - (prefix-in template: (submod "forms.rkt" template)) - (prefix-in catchall-template: (submod "forms.rkt" catchall-template)) - (prefix-in if: (submod "forms.rkt" if)) - (prefix-in when: (submod "forms.rkt" when)) - (prefix-in unless: (submod "forms.rkt" unless)) - (prefix-in switch: (submod "forms.rkt" switch)) - (prefix-in sieve: (submod "forms.rkt" sieve)) - (prefix-in partition: (submod "forms.rkt" partition)) - (prefix-in gate: (submod "forms.rkt" gate)) - (prefix-in input-aliases: (submod "forms.rkt" input-aliases)) - (prefix-in fanout: (submod "forms.rkt" fanout)) - (prefix-in inverter: (submod "forms.rkt" inverter)) - (prefix-in feedback: (submod "forms.rkt" feedback)) - (prefix-in select: (submod "forms.rkt" select)) - (prefix-in block: (submod "forms.rkt" block)) - (prefix-in bundle: (submod "forms.rkt" bundle)) - (prefix-in effect: (submod "forms.rkt" effect)) - (prefix-in live?: (submod "forms.rkt" live?)) - (prefix-in rectify: (submod "forms.rkt" rectify)) - (prefix-in pass: (submod "forms.rkt" pass)) - (prefix-in foldl: (submod "forms.rkt" foldl)) - (prefix-in foldr: (submod "forms.rkt" foldr)) - (prefix-in loop: (submod "forms.rkt" loop)) - (prefix-in loop2: (submod "forms.rkt" loop2)) - (prefix-in apply: (submod "forms.rkt" apply)) - (prefix-in clos: (submod "forms.rkt" clos))) - -(require "loadlib.rkt" - "regression.rkt") - -(require racket/match - racket/format - relation - qi - json - csv-writing - (only-in "util.rkt" - only-if - for/call)) - -;; It would be great if we could get the value of a variable -;; by using its (string) name, but (eval (string->symbol name)) -;; doesn't find it. So instead, we reify the "lexical environment" -;; here manually, so that the values can be looked up at runtime -;; based on the string names (note that the value is always the key -;; + ":" + "run") -(define env - (hash - "one-of?" one-of?:run - "and" and:run - "or" or:run - "not" not:run - "and%" and%:run - "or%" or%:run - "group" group:run - "count" count:run - "relay" relay:run - "relay*" relay*:run - "amp" amp:run - "ground" ground:run - "thread" thread:run - "thread-right" thread-right:run - "crossover" crossover:run - "all" all:run - "any" any:run - "none" none:run - "all?" all?:run - "any?" any?:run - "none?" none?:run - "collect" collect:run - "sep" sep:run - "gen" gen:run - "esc" esc:run - "AND" AND:run - "OR" OR:run - "NOT" NOT:run - "NAND" NAND:run - "NOR" NOR:run - "XOR" XOR:run - "XNOR" XNOR:run - "tee" tee:run - "try" try:run - "currying" currying:run - "template" template:run - "catchall-template" catchall-template:run - "if" if:run - "when" when:run - "unless" unless:run - "switch" switch:run - "sieve" sieve:run - "partition" partition:run - "gate" gate:run - "input-aliases" input-aliases:run - "fanout" fanout:run - "inverter" inverter:run - "feedback" feedback:run - "select" select:run - "block" block:run - "bundle" bundle:run - "effect" effect:run - "live?" live?:run - "rectify" rectify:run - "pass" pass:run - "foldl" foldl:run - "foldr" foldr:run - "loop" loop:run - "loop2" loop2:run - "apply" apply:run - "clos" clos:run)) - -(define (write-csv data) - (~> (data) - △ - (>< (~> (-< (hash-ref 'name) - (hash-ref 'unit) - (hash-ref 'value)) - ▽)) - (-< '(name unit value) - _) - ▽ - display-table)) - -(help - (usage (~a "Report on the performance of all of the forms " - "of the language, in a configurable output format."))) - -(flag (output-format #:param [output-format "json"] fmt) - ("-f" "--format" "Output format to use, either 'json' or 'csv'") - (output-format fmt)) - -(flag (regression-file #:param [regression-file #f] reg-file) - ("-r" "--regression" "'Before' data to compute regression against") - (regression-file reg-file)) - -(define (format-output output) - ;; Note: this is a case where declaring "constraints" on the CLI args - ;; would be useful, instead of using the ad hoc fallback `else` check here - ;; https://github.com/countvajhula/cli/issues/6 - (cond - [(equal? (output-format) "json") (write-json output)] - [(equal? (output-format) "csv") (write-csv output)] - [else (error (~a "Unrecognized format: " (output-format) "!"))])) - -(program (main) - (define fs (hash-keys env #t)) - (define forms-data (for/list ([f (in-list fs)]) - (match-let ([(list name ms) ((hash-ref env f))]) - (hash 'name name 'unit "ms" 'value ms)))) - (define require-data (list (hash 'name "(require qi)" - 'unit "ms" - 'value (time-module-ms "qi")))) - (let ([output (append forms-data require-data)]) - - (if (regression-file) - (let ([before (parse-benchmarks (parse-json-file (regression-file)))] - [after (parse-benchmarks output)]) - (compute-regression before after)) - (format-output output)))) - -(run main) From 7c955e82355816279b9c3dd03a0bdf8a9a1c819f Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 18:47:34 -0800 Subject: [PATCH 04/29] start to separate form benchmarks from other benchmarks --- qi-sdk/profile/forms/benchmarks.rkt | 186 +++++++++++++ qi-sdk/profile/forms/report.rkt | 246 ------------------ qi-sdk/profile/{forms => loading}/loadlib.rkt | 0 qi-sdk/profile/report.rkt | 79 ++++++ 4 files changed, 265 insertions(+), 246 deletions(-) delete mode 100755 qi-sdk/profile/forms/report.rkt rename qi-sdk/profile/{forms => loading}/loadlib.rkt (100%) create mode 100755 qi-sdk/profile/report.rkt diff --git a/qi-sdk/profile/forms/benchmarks.rkt b/qi-sdk/profile/forms/benchmarks.rkt index 27fc22865..84e8d0713 100755 --- a/qi-sdk/profile/forms/benchmarks.rkt +++ b/qi-sdk/profile/forms/benchmarks.rkt @@ -1,6 +1,31 @@ #!/usr/bin/env racket #lang racket/base +#| +To add a benchmark for a new form: + +1. Add a submodule for it which provides a `run` function taking no +arguments. This function will be expected to exercise the new form and +return a time taken. The `run` function typically uses one of the +utility macros `run-benchmark` or `run-summary-benchmark`, and +provides it one of the helper functions `check-value` (to invoke the +form with a single value each time during benchmarking) or +`check-values` (to invoke the form with multiple values each time +during benchmarking). Note that at the moment, as a hack for +convenience, `run-benchmark` expects a function with the name of the +form being benchmarked _prefixed with tilde_. This is to avoid name +collisions between this function and the Qi form with the same +name. Basically, just follow one of the numerous examples in this +module to see what this is referring to. + +2. Require the submodule in the `main` submodule with an appropriate +prefix (see other examples) + +3. Add the required `run` function to the `env` hash in the main +submodule. This will ensure that it gets picked up when the benchmarks +for the forms are run. +|# + (module one-of? "base.rkt" (provide run) @@ -864,3 +889,164 @@ (run-benchmark ~clos check-values 100000))) + +(module main racket/base + + (provide benchmark) + + (require racket/match + racket/format + relation + qi + json + csv-writing + (only-in "../util.rkt" + only-if + for/call)) + (require + (prefix-in one-of?: (submod ".." one-of?)) + (prefix-in and: (submod ".." and)) + (prefix-in or: (submod ".." or)) + (prefix-in not: (submod ".." not)) + (prefix-in and%: (submod ".." and%)) + (prefix-in or%: (submod ".." or%)) + (prefix-in group: (submod ".." group)) + (prefix-in count: (submod ".." count)) + (prefix-in relay: (submod ".." relay)) + (prefix-in relay*: (submod ".." relay*)) + (prefix-in amp: (submod ".." amp)) + (prefix-in ground: (submod ".." ground)) + (prefix-in thread: (submod ".." thread)) + (prefix-in thread-right: (submod ".." thread-right)) + (prefix-in crossover: (submod ".." crossover)) + (prefix-in all: (submod ".." all)) + (prefix-in any: (submod ".." any)) + (prefix-in none: (submod ".." none)) + (prefix-in all?: (submod ".." all?)) + (prefix-in any?: (submod ".." any?)) + (prefix-in none?: (submod ".." none?)) + (prefix-in collect: (submod ".." collect)) + (prefix-in sep: (submod ".." sep)) + (prefix-in gen: (submod ".." gen)) + (prefix-in esc: (submod ".." esc)) + (prefix-in AND: (submod ".." AND)) + (prefix-in OR: (submod ".." OR)) + (prefix-in NOT: (submod ".." NOT)) + (prefix-in NAND: (submod ".." NAND)) + (prefix-in NOR: (submod ".." NOR)) + (prefix-in XOR: (submod ".." XOR)) + (prefix-in XNOR: (submod ".." XNOR)) + (prefix-in tee: (submod ".." tee)) + (prefix-in try: (submod ".." try)) + (prefix-in currying: (submod ".." currying)) + (prefix-in template: (submod ".." template)) + (prefix-in catchall-template: (submod ".." catchall-template)) + (prefix-in if: (submod ".." if)) + (prefix-in when: (submod ".." when)) + (prefix-in unless: (submod ".." unless)) + (prefix-in switch: (submod ".." switch)) + (prefix-in sieve: (submod ".." sieve)) + (prefix-in partition: (submod ".." partition)) + (prefix-in gate: (submod ".." gate)) + (prefix-in input-aliases: (submod ".." input-aliases)) + (prefix-in fanout: (submod ".." fanout)) + (prefix-in inverter: (submod ".." inverter)) + (prefix-in feedback: (submod ".." feedback)) + (prefix-in select: (submod ".." select)) + (prefix-in block: (submod ".." block)) + (prefix-in bundle: (submod ".." bundle)) + (prefix-in effect: (submod ".." effect)) + (prefix-in live?: (submod ".." live?)) + (prefix-in rectify: (submod ".." rectify)) + (prefix-in pass: (submod ".." pass)) + (prefix-in foldl: (submod ".." foldl)) + (prefix-in foldr: (submod ".." foldr)) + (prefix-in loop: (submod ".." loop)) + (prefix-in loop2: (submod ".." loop2)) + (prefix-in apply: (submod ".." apply)) + (prefix-in clos: (submod ".." clos))) + + ;; It would be great if we could get the value of a variable + ;; by using its (string) name, but (eval (string->symbol name)) + ;; doesn't find it. So instead, we reify the "lexical environment" + ;; here manually, so that the values can be looked up at runtime + ;; based on the string names (note that the value is always the key + ;; + ":" + "run") + (define env + (hash + "one-of?" one-of?:run + "and" and:run + "or" or:run + "not" not:run + "and%" and%:run + "or%" or%:run + "group" group:run + "count" count:run + "relay" relay:run + "relay*" relay*:run + "amp" amp:run + "ground" ground:run + "thread" thread:run + "thread-right" thread-right:run + "crossover" crossover:run + "all" all:run + "any" any:run + "none" none:run + "all?" all?:run + "any?" any?:run + "none?" none?:run + "collect" collect:run + "sep" sep:run + "gen" gen:run + "esc" esc:run + "AND" AND:run + "OR" OR:run + "NOT" NOT:run + "NAND" NAND:run + "NOR" NOR:run + "XOR" XOR:run + "XNOR" XNOR:run + "tee" tee:run + "try" try:run + "currying" currying:run + "template" template:run + "catchall-template" catchall-template:run + "if" if:run + "when" when:run + "unless" unless:run + "switch" switch:run + "sieve" sieve:run + "partition" partition:run + "gate" gate:run + "input-aliases" input-aliases:run + "fanout" fanout:run + "inverter" inverter:run + "feedback" feedback:run + "select" select:run + "block" block:run + "bundle" bundle:run + "effect" effect:run + "live?" live?:run + "rectify" rectify:run + "pass" pass:run + "foldl" foldl:run + "foldr" foldr:run + "loop" loop:run + "loop2" loop2:run + "apply" apply:run + "clos" clos:run)) + + (define (benchmark forms) + (define fs (~>> (forms) + (only-if null? + (gen (hash-keys env))) + (sort <))) + (define forms-data (for/list ([f (in-list fs)]) + (match-let ([(list name ms) ((hash-ref env f))]) + ;; Print results "live" to STDERR, with + ;; only the actual output (if desired) + ;; going to STDOUT at the end. + (displayln (~a name ": " ms " ms") + (current-error-port)) + (hash 'name name 'unit "ms" 'value ms)))) + forms-data)) diff --git a/qi-sdk/profile/forms/report.rkt b/qi-sdk/profile/forms/report.rkt deleted file mode 100755 index e32073740..000000000 --- a/qi-sdk/profile/forms/report.rkt +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env racket -#lang cli - -#| -To add a benchmark for a new form: - -1. Add a submodule for it in benchmarks.rkt which provides a `run` -function taking no arguments. This function will be expected to -exercise the new form and return a time taken. The `run` function -typically uses one of the utility macros `run-benchmark` or -`run-summary-benchmark`, and provides it one of the helper functions -`check-value` (to invoke the form with a single value each time during -benchmarking) or `check-values` (to invoke the form with multiple -values each time during benchmarking). Note that at the moment, as a -hack for convenience, `run-benchmark` expects a function with the name -of the form being benchmarked _prefixed with tilde_. This is to avoid -name collisions between this function and the Qi form with the same -name. Basically, just follow one of the numerous examples in this -module to see what this is referring to. - -2. Require the submodule in the present module with an appropriate -prefix (see other examples) - -3. Add the required `run` function to the `env` hash below. This will -ensure that it gets picked up when the benchmarks for the forms are -run. -|# - -(require - (prefix-in one-of?: (submod "benchmarks.rkt" one-of?)) - (prefix-in and: (submod "benchmarks.rkt" and)) - (prefix-in or: (submod "benchmarks.rkt" or)) - (prefix-in not: (submod "benchmarks.rkt" not)) - (prefix-in and%: (submod "benchmarks.rkt" and%)) - (prefix-in or%: (submod "benchmarks.rkt" or%)) - (prefix-in group: (submod "benchmarks.rkt" group)) - (prefix-in count: (submod "benchmarks.rkt" count)) - (prefix-in relay: (submod "benchmarks.rkt" relay)) - (prefix-in relay*: (submod "benchmarks.rkt" relay*)) - (prefix-in amp: (submod "benchmarks.rkt" amp)) - (prefix-in ground: (submod "benchmarks.rkt" ground)) - (prefix-in thread: (submod "benchmarks.rkt" thread)) - (prefix-in thread-right: (submod "benchmarks.rkt" thread-right)) - (prefix-in crossover: (submod "benchmarks.rkt" crossover)) - (prefix-in all: (submod "benchmarks.rkt" all)) - (prefix-in any: (submod "benchmarks.rkt" any)) - (prefix-in none: (submod "benchmarks.rkt" none)) - (prefix-in all?: (submod "benchmarks.rkt" all?)) - (prefix-in any?: (submod "benchmarks.rkt" any?)) - (prefix-in none?: (submod "benchmarks.rkt" none?)) - (prefix-in collect: (submod "benchmarks.rkt" collect)) - (prefix-in sep: (submod "benchmarks.rkt" sep)) - (prefix-in gen: (submod "benchmarks.rkt" gen)) - (prefix-in esc: (submod "benchmarks.rkt" esc)) - (prefix-in AND: (submod "benchmarks.rkt" AND)) - (prefix-in OR: (submod "benchmarks.rkt" OR)) - (prefix-in NOT: (submod "benchmarks.rkt" NOT)) - (prefix-in NAND: (submod "benchmarks.rkt" NAND)) - (prefix-in NOR: (submod "benchmarks.rkt" NOR)) - (prefix-in XOR: (submod "benchmarks.rkt" XOR)) - (prefix-in XNOR: (submod "benchmarks.rkt" XNOR)) - (prefix-in tee: (submod "benchmarks.rkt" tee)) - (prefix-in try: (submod "benchmarks.rkt" try)) - (prefix-in currying: (submod "benchmarks.rkt" currying)) - (prefix-in template: (submod "benchmarks.rkt" template)) - (prefix-in catchall-template: (submod "benchmarks.rkt" catchall-template)) - (prefix-in if: (submod "benchmarks.rkt" if)) - (prefix-in when: (submod "benchmarks.rkt" when)) - (prefix-in unless: (submod "benchmarks.rkt" unless)) - (prefix-in switch: (submod "benchmarks.rkt" switch)) - (prefix-in sieve: (submod "benchmarks.rkt" sieve)) - (prefix-in partition: (submod "benchmarks.rkt" partition)) - (prefix-in gate: (submod "benchmarks.rkt" gate)) - (prefix-in input-aliases: (submod "benchmarks.rkt" input-aliases)) - (prefix-in fanout: (submod "benchmarks.rkt" fanout)) - (prefix-in inverter: (submod "benchmarks.rkt" inverter)) - (prefix-in feedback: (submod "benchmarks.rkt" feedback)) - (prefix-in select: (submod "benchmarks.rkt" select)) - (prefix-in block: (submod "benchmarks.rkt" block)) - (prefix-in bundle: (submod "benchmarks.rkt" bundle)) - (prefix-in effect: (submod "benchmarks.rkt" effect)) - (prefix-in live?: (submod "benchmarks.rkt" live?)) - (prefix-in rectify: (submod "benchmarks.rkt" rectify)) - (prefix-in pass: (submod "benchmarks.rkt" pass)) - (prefix-in foldl: (submod "benchmarks.rkt" foldl)) - (prefix-in foldr: (submod "benchmarks.rkt" foldr)) - (prefix-in loop: (submod "benchmarks.rkt" loop)) - (prefix-in loop2: (submod "benchmarks.rkt" loop2)) - (prefix-in apply: (submod "benchmarks.rkt" apply)) - (prefix-in clos: (submod "benchmarks.rkt" clos))) - -(require "loadlib.rkt" - "regression.rkt") - -(require racket/match - racket/format - relation - qi - json - csv-writing - (only-in "../util.rkt" - only-if - for/call)) - -;; It would be great if we could get the value of a variable -;; by using its (string) name, but (eval (string->symbol name)) -;; doesn't find it. So instead, we reify the "lexical environment" -;; here manually, so that the values can be looked up at runtime -;; based on the string names (note that the value is always the key -;; + ":" + "run") -(define env - (hash - "one-of?" one-of?:run - "and" and:run - "or" or:run - "not" not:run - "and%" and%:run - "or%" or%:run - "group" group:run - "count" count:run - "relay" relay:run - "relay*" relay*:run - "amp" amp:run - "ground" ground:run - "thread" thread:run - "thread-right" thread-right:run - "crossover" crossover:run - "all" all:run - "any" any:run - "none" none:run - "all?" all?:run - "any?" any?:run - "none?" none?:run - "collect" collect:run - "sep" sep:run - "gen" gen:run - "esc" esc:run - "AND" AND:run - "OR" OR:run - "NOT" NOT:run - "NAND" NAND:run - "NOR" NOR:run - "XOR" XOR:run - "XNOR" XNOR:run - "tee" tee:run - "try" try:run - "currying" currying:run - "template" template:run - "catchall-template" catchall-template:run - "if" if:run - "when" when:run - "unless" unless:run - "switch" switch:run - "sieve" sieve:run - "partition" partition:run - "gate" gate:run - "input-aliases" input-aliases:run - "fanout" fanout:run - "inverter" inverter:run - "feedback" feedback:run - "select" select:run - "block" block:run - "bundle" bundle:run - "effect" effect:run - "live?" live?:run - "rectify" rectify:run - "pass" pass:run - "foldl" foldl:run - "foldr" foldr:run - "loop" loop:run - "loop2" loop2:run - "apply" apply:run - "clos" clos:run)) - -(define (write-csv data) - (~> (data) - △ - (>< (~> (-< (hash-ref 'name) - (hash-ref 'unit) - (hash-ref 'value)) - ▽)) - (-< '(name unit value) - _) - ▽ - display-table)) - -(flag (forms #:param [forms null] name) - ("-f" "--form" "Forms to benchmark") - (forms (cons name (forms)))) - -(constraint (multi forms)) - -(help - (usage - (~a "Run benchmarks for individual Qi forms " - "(by default, all of them), reporting the results " - "in a configurable output format."))) - -(flag (output-format #:param [output-format ""] fmt) - ("-o" - "--format" - "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") - (output-format fmt)) - -(flag (regression-file #:param [regression-file #f] reg-file) - ("-r" "--regression" "'Before' data to compute regression against") - (regression-file reg-file)) - -(define (format-output output) - ;; Note: this is a case where declaring "constraints" on the CLI args - ;; would be useful, instead of using the ad hoc fallback `else` check here - ;; https://github.com/countvajhula/cli/issues/6 - (cond - [(equal? (output-format) "json") (write-json output)] - [(equal? (output-format) "csv") (write-csv output)] - [(equal? (output-format) "") (values)] - [else (error (~a "Unrecognized format: " (output-format) "!"))])) - -(program (main) - (define fs (~>> ((forms)) - (only-if null? - (gen (hash-keys env))) - (sort <))) - (define forms-data (for/list ([f (in-list fs)]) - (match-let ([(list name ms) ((hash-ref env f))]) - ;; Print results "live" to STDERR, with - ;; only the actual output (if desired) - ;; going to STDOUT at the end. - (displayln (~a name ": " ms " ms") - (current-error-port)) - (hash 'name name 'unit "ms" 'value ms)))) - (define require-data (list (hash 'name "(require qi)" - 'unit "ms" - 'value (time-module-ms "qi")))) - (let ([output (append forms-data require-data)]) - - (if (regression-file) - (let ([before (parse-benchmarks (parse-json-file (regression-file)))] - [after (parse-benchmarks output)]) - (compute-regression before after)) - (format-output output)))) - -;; To run benchmarks for a form interactively, use e.g.: -;; (run main #("-f" "fanout")) - -(run main) diff --git a/qi-sdk/profile/forms/loadlib.rkt b/qi-sdk/profile/loading/loadlib.rkt similarity index 100% rename from qi-sdk/profile/forms/loadlib.rkt rename to qi-sdk/profile/loading/loadlib.rkt diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt new file mode 100755 index 000000000..72a3e3edd --- /dev/null +++ b/qi-sdk/profile/report.rkt @@ -0,0 +1,79 @@ +#!/usr/bin/env racket +#lang cli + +(require "loading/loadlib.rkt" + "forms/regression.rkt") + +(require racket/match + racket/format + relation + qi + json + csv-writing + (only-in "util.rkt" + only-if + for/call)) +(require + (submod "forms/benchmarks.rkt" main)) + +(define (write-csv data) + (~> (data) + △ + (>< (~> (-< (hash-ref 'name) + (hash-ref 'unit) + (hash-ref 'value)) + ▽)) + (-< '(name unit value) + _) + ▽ + display-table)) + +(flag (forms #:param [forms null] name) + ("-f" "--form" "Forms to benchmark") + (forms (cons name (forms)))) + +(constraint (multi forms)) + +(help + (usage + (~a "Run benchmarks for individual Qi forms " + "(by default, all of them), reporting the results " + "in a configurable output format."))) + +(flag (output-format #:param [output-format ""] fmt) + ("-o" + "--format" + "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") + (output-format fmt)) + +(flag (regression-file #:param [regression-file #f] reg-file) + ("-r" "--regression" "'Before' data to compute regression against") + (regression-file reg-file)) + +(define (format-output output) + ;; Note: this is a case where declaring "constraints" on the CLI args + ;; would be useful, instead of using the ad hoc fallback `else` check here + ;; https://github.com/countvajhula/cli/issues/6 + (cond + [(equal? (output-format) "json") (write-json output)] + [(equal? (output-format) "csv") (write-csv output)] + [(equal? (output-format) "") (values)] + [else (error (~a "Unrecognized format: " (output-format) "!"))])) + +(program (main) + (define forms-data (benchmark (forms))) + (define require-data (list (hash 'name "(require qi)" + 'unit "ms" + 'value (time-module-ms "qi")))) + (let ([output (append forms-data require-data)]) + + (if (regression-file) + (let ([before (parse-benchmarks (parse-json-file (regression-file)))] + [after (parse-benchmarks output)]) + (compute-regression before after)) + (format-output output)))) + +;; To run benchmarks for a form interactively, use e.g.: +;; (run main #("-f" "fanout")) + +(run main) From 1a51450cc42040edf1a66eff844f0fd8ce6e99ab Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 20:05:40 -0800 Subject: [PATCH 05/29] Organize scripts to generate performance reports Have separate scripts to generate reports of different aspects, to preserve separation of concerns. This results in some duplication of configuration, though, which is probably best avoided via some improvements to the cli library. Follow the general pattern of echoing live results to STDERR and actual formatted output to STDOUT. --- .github/workflows/benchmarks.yml | 2 +- Makefile | 18 +++++--- qi-sdk/profile/forms/report.rkt | 51 +++++++++++++++++++++ qi-sdk/profile/loading/loadlib.rkt | 12 ++++- qi-sdk/profile/loading/report.rkt | 39 ++++++++++++++++ qi-sdk/profile/{forms => }/regression.rkt | 0 qi-sdk/profile/report.rkt | 55 +++++++++-------------- qi-sdk/profile/util.rkt | 29 +++++++++++- 8 files changed, 162 insertions(+), 44 deletions(-) create mode 100755 qi-sdk/profile/forms/report.rkt create mode 100755 qi-sdk/profile/loading/report.rkt rename qi-sdk/profile/{forms => }/regression.rkt (100%) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5d37d3451..53d10d1b7 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -25,7 +25,7 @@ jobs: run: make install-sdk - name: Run benchmark shell: 'bash --noprofile --norc -eo pipefail {0}' - run: make form-performance-report | tee benchmarks.txt + run: make performance-report | tee benchmarks.txt - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: diff --git a/Makefile b/Makefile index 3a94763a8..102bb2aab 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,8 @@ help: @echo "profile-competitive - Run competitive benchmarks" @echo "profile-forms - Run benchmarks for individual Qi forms" @echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)" - @echo "form-performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression" - @echo " For use in regression: make form-performance-report > /path/to/before.json" + @echo "performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression" + @echo " For use in regression: make performance-report > /path/to/before.json" @echo "performance-regression-report - Run benchmarks for Qi forms against a reference report." @echo " make performance-regression-report REF=/path/to/before.json" @@ -171,10 +171,14 @@ cover-coveralls: profile-forms: echo "Profiling forms..." - racket $(PACKAGE-NAME)-sdk/profile/forms.rkt + racket $(PACKAGE-NAME)-sdk/profile/forms/report.rkt + +profile-loading: + echo "Profiling module loading..." + racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt profile-selected-forms: - @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/forms.rkt' directly, with -f form-name for each form." + @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/forms/report.rkt' directly, with -f form-name for each form." profile-competitive: echo "Running competitive benchmarks..." @@ -182,10 +186,10 @@ profile-competitive: profile: profile-competitive profile-forms -form-performance-report: - @racket $(PACKAGE-NAME)-sdk/profile/report.rkt +performance-report: + @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -o json performance-regression-report: @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF) -.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile form-performance-report performance-regression-report +.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile performance-report performance-regression-report diff --git a/qi-sdk/profile/forms/report.rkt b/qi-sdk/profile/forms/report.rkt new file mode 100755 index 000000000..753a8166d --- /dev/null +++ b/qi-sdk/profile/forms/report.rkt @@ -0,0 +1,51 @@ +#!/usr/bin/env racket +#lang cli + +(require "../regression.rkt") + +(require racket/match + racket/format + relation + qi + (only-in "../util.rkt" + only-if + for/call + write-csv + format-output)) +(require + (submod "benchmarks.rkt" main)) + +(flag (forms #:param [forms null] name) + ("-f" "--form" "Forms to benchmark") + (forms (cons name (forms)))) + +(constraint (multi forms)) + +(help + (usage + (~a "Run benchmarks for individual Qi forms " + "(by default, all of them), reporting the results " + "in a configurable output format."))) + +(flag (output-format #:param [output-format ""] fmt) + ("-o" + "--format" + "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") + (output-format fmt)) + +(flag (regression-file #:param [regression-file #f] reg-file) + ("-r" "--regression" "'Before' data to compute regression against") + (regression-file reg-file)) + +(program (main) + (let ([output (benchmark (forms))]) + (if (regression-file) + (let ([before (parse-benchmarks (parse-json-file (regression-file)))] + [after (parse-benchmarks output)]) + (compute-regression before after)) + (format-output output (output-format))))) + +;; To run benchmarks for a form interactively, use e.g.: +;; (run main #("-f" "fanout")) + +(run main) diff --git a/qi-sdk/profile/loading/loadlib.rkt b/qi-sdk/profile/loading/loadlib.rkt index 4ebdaed6d..1541f04af 100755 --- a/qi-sdk/profile/loading/loadlib.rkt +++ b/qi-sdk/profile/loading/loadlib.rkt @@ -2,7 +2,8 @@ #lang cli (provide time-racket - time-module-ms) + time-module-ms + profile-load) (require racket/port racket/format) @@ -41,6 +42,15 @@ what remains is just the time contributed by requiring the specified module. (- (time-racket module-name) (time-racket)))) +(define (profile-load module-name) + (let ([name (~a "(require " module-name ")")] + [ms (time-module-ms module-name)]) + (displayln (~a name ": " ms " ms") + (current-error-port)) + (hash 'name name + 'unit "ms" + 'value ms))) + (program (time-require module-name) (displayln (~a (time-module-ms module-name) " ms"))) diff --git a/qi-sdk/profile/loading/report.rkt b/qi-sdk/profile/loading/report.rkt new file mode 100755 index 000000000..f6542cd10 --- /dev/null +++ b/qi-sdk/profile/loading/report.rkt @@ -0,0 +1,39 @@ +#!/usr/bin/env racket +#lang cli + +(require "../regression.rkt" + "loadlib.rkt") + +(require racket/match + racket/format + relation + qi + (only-in "../util.rkt" + only-if + for/call + write-csv + format-output)) + +(help + (usage + (~a "Measure module load time, i.e. the time taken by (require qi)."))) + +(flag (output-format #:param [output-format ""] fmt) + ("-o" + "--format" + "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") + (output-format fmt)) + +(flag (regression-file #:param [regression-file #f] reg-file) + ("-r" "--regression" "'Before' data to compute regression against") + (regression-file reg-file)) + +(program (main) + (let ([output (profile-load "qi")]) + (if (regression-file) + (let ([before (parse-benchmarks (parse-json-file (regression-file)))] + [after (parse-benchmarks output)]) + (compute-regression before after)) + (format-output output (output-format))))) + +(run main) diff --git a/qi-sdk/profile/forms/regression.rkt b/qi-sdk/profile/regression.rkt similarity index 100% rename from qi-sdk/profile/forms/regression.rkt rename to qi-sdk/profile/regression.rkt diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index 72a3e3edd..6150bb36a 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -2,32 +2,20 @@ #lang cli (require "loading/loadlib.rkt" - "forms/regression.rkt") + "regression.rkt") (require racket/match racket/format relation qi - json - csv-writing (only-in "util.rkt" only-if - for/call)) + for/call + write-csv + format-output)) (require (submod "forms/benchmarks.rkt" main)) -(define (write-csv data) - (~> (data) - △ - (>< (~> (-< (hash-ref 'name) - (hash-ref 'unit) - (hash-ref 'value)) - ▽)) - (-< '(name unit value) - _) - ▽ - display-table)) - (flag (forms #:param [forms null] name) ("-f" "--form" "Forms to benchmark") (forms (cons name (forms)))) @@ -46,32 +34,33 @@ "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) +(flag (type #:param [report-type "all"] typ) + ("-t" + "--type" + "Type of report, either `forms`, `loading` or `all` (default `all`)") + (report-type typ)) + (flag (regression-file #:param [regression-file #f] reg-file) ("-r" "--regression" "'Before' data to compute regression against") (regression-file reg-file)) -(define (format-output output) - ;; Note: this is a case where declaring "constraints" on the CLI args - ;; would be useful, instead of using the ad hoc fallback `else` check here - ;; https://github.com/countvajhula/cli/issues/6 - (cond - [(equal? (output-format) "json") (write-json output)] - [(equal? (output-format) "csv") (write-csv output)] - [(equal? (output-format) "") (values)] - [else (error (~a "Unrecognized format: " (output-format) "!"))])) - +;; Note: much of this file is duplicated across forms/report.rkt +;; and loading/report.rkt. It could be avoided if we had +;; "composition of commands", see: +;; https://github.com/countvajhula/cli/issues/3 (program (main) - (define forms-data (benchmark (forms))) - (define require-data (list (hash 'name "(require qi)" - 'unit "ms" - 'value (time-module-ms "qi")))) - (let ([output (append forms-data require-data)]) - + (let* ([forms-data (if (member? (report-type) (list "all" "forms")) + (benchmark (forms)) + null)] + [require-data (if (member? (report-type) (list "all" "loading")) + (list (profile-load "qi")) + null)] + [output (append forms-data require-data)]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) (compute-regression before after)) - (format-output output)))) + (format-output output (output-format))))) ;; To run benchmarks for a form interactively, use e.g.: ;; (run main #("-f" "fanout")) diff --git a/qi-sdk/profile/util.rkt b/qi-sdk/profile/util.rkt index 17fd1c805..40c8aa61b 100644 --- a/qi-sdk/profile/util.rkt +++ b/qi-sdk/profile/util.rkt @@ -10,7 +10,9 @@ run-summary-benchmark run-competitive-benchmark (for-space qi only-if) - for/call) + for/call + write-csv + format-output) (require (only-in racket/list range @@ -21,7 +23,8 @@ cycle take in) - racket/function + csv-writing + json racket/format syntax/parse/define (for-syntax racket/base @@ -136,3 +139,25 @@ [label (list "λ" "☯")]) (let ([ms (measure runner f n-times)]) (displayln (~a label ": " ms " ms")))))) + +(define (write-csv data) + (~> (data) + △ + (>< (~> (-< (hash-ref 'name) + (hash-ref 'unit) + (hash-ref 'value)) + ▽)) + (-< '(name unit value) + _) + ▽ + display-table)) + +(define (format-output output fmt) + ;; Note: this is a case where declaring "constraints" on the CLI args + ;; would be useful, instead of using the ad hoc fallback `else` check here + ;; https://github.com/countvajhula/cli/issues/6 + (cond + [(equal? fmt "json") (write-json output)] + [(equal? fmt "csv") (write-csv output)] + [(equal? fmt "") (values)] + [else (error (~a "Unrecognized format: " fmt "!"))])) From 49a827ab50b052d5c2f2c8665c85acf445267a61 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 20:31:45 -0800 Subject: [PATCH 06/29] Use require-latency library instead of measuring load time locally --- qi-sdk/info.rkt | 1 + qi-sdk/profile/loading/loadlib.rkt | 50 +++--------------------------- 2 files changed, 5 insertions(+), 46 deletions(-) diff --git a/qi-sdk/info.rkt b/qi-sdk/info.rkt index b79b9ef6c..ea3ccec2d 100644 --- a/qi-sdk/info.rkt +++ b/qi-sdk/info.rkt @@ -10,6 +10,7 @@ "collections-lib" "relation-lib" "csv-writing" + "require-latency" "cover" "cover-coveralls")) (define build-deps '()) diff --git a/qi-sdk/profile/loading/loadlib.rkt b/qi-sdk/profile/loading/loadlib.rkt index 1541f04af..b0dae806e 100755 --- a/qi-sdk/profile/loading/loadlib.rkt +++ b/qi-sdk/profile/loading/loadlib.rkt @@ -1,58 +1,16 @@ #!/usr/bin/env racket -#lang cli +#lang racket/base -(provide time-racket - time-module-ms - profile-load) +(provide profile-load) -(require racket/port +(require pkg/require-latency racket/format) -#| -This works by: -1. Running `racket -l ` and `racket -l racket/base` independently -2. Subtracting the latter from the former. -3. Printing that result in milliseconds. - -where is the argument you specified at the command line, -e.g. ./loadlib.rkt racket/list - -The idea is to subtract out the contribution from racket/base, so that -what remains is just the time contributed by requiring the specified module. -|# - -(define (time-racket [module-name "racket/base"]) - (define-values (sp out in err) - (subprocess #f #f #f (find-executable-path "time") "-p" (find-executable-path "racket") "-l" module-name)) - (define result (port->string err)) - (define seconds (string->number - (car - (regexp-match #px"[\\d|\\.]+" - (car - (regexp-match #rx"(?m:^real.*)" - result)))))) - (close-input-port out) - (close-output-port in) - (close-input-port err) - (subprocess-wait sp) - seconds) - -(define (time-module-ms module-name) - (* 1000 - (- (time-racket module-name) - (time-racket)))) - (define (profile-load module-name) (let ([name (~a "(require " module-name ")")] - [ms (time-module-ms module-name)]) + [ms (cdr (time-module-ms module-name))]) (displayln (~a name ": " ms " ms") (current-error-port)) (hash 'name name 'unit "ms" 'value ms))) - -(program (time-require module-name) - (displayln (~a (time-module-ms module-name) " ms"))) - -(module+ main - (run time-require)) From eaa26cfe28c94a674218229349a4906186044698 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 20:39:48 -0800 Subject: [PATCH 07/29] move competitive benchmarks into a separate folder --- Makefile | 2 +- qi-sdk/profile/{ => competitive}/builtin.rkt | 0 qi-sdk/profile/{ => competitive}/competitive.rkt | 2 +- qi-sdk/profile/{ => competitive}/qi.rkt | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename qi-sdk/profile/{ => competitive}/builtin.rkt (100%) rename qi-sdk/profile/{ => competitive}/competitive.rkt (98%) rename qi-sdk/profile/{ => competitive}/qi.rkt (100%) diff --git a/Makefile b/Makefile index 102bb2aab..f40cbd0b9 100644 --- a/Makefile +++ b/Makefile @@ -182,7 +182,7 @@ profile-selected-forms: profile-competitive: echo "Running competitive benchmarks..." - racket $(PACKAGE-NAME)-sdk/profile/competitive.rkt + racket $(PACKAGE-NAME)-sdk/profile/competitive/competitive.rkt profile: profile-competitive profile-forms diff --git a/qi-sdk/profile/builtin.rkt b/qi-sdk/profile/competitive/builtin.rkt similarity index 100% rename from qi-sdk/profile/builtin.rkt rename to qi-sdk/profile/competitive/builtin.rkt diff --git a/qi-sdk/profile/competitive.rkt b/qi-sdk/profile/competitive/competitive.rkt similarity index 98% rename from qi-sdk/profile/competitive.rkt rename to qi-sdk/profile/competitive/competitive.rkt index 833e5bf80..8547d11c0 100755 --- a/qi-sdk/profile/competitive.rkt +++ b/qi-sdk/profile/competitive/competitive.rkt @@ -12,7 +12,7 @@ (prefix-in q: "qi.rkt") (prefix-in b: "builtin.rkt")) -(require "util.rkt") +(require "../util.rkt") (displayln "\nRunning flat benchmarks...") diff --git a/qi-sdk/profile/qi.rkt b/qi-sdk/profile/competitive/qi.rkt similarity index 100% rename from qi-sdk/profile/qi.rkt rename to qi-sdk/profile/competitive/qi.rkt From 87fed46e1b6af9da78c8df63026eba203158504d Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 20:40:18 -0800 Subject: [PATCH 08/29] update phonies in makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f40cbd0b9..d96e20b05 100644 --- a/Makefile +++ b/Makefile @@ -192,4 +192,4 @@ performance-report: performance-regression-report: @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF) -.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile performance-report performance-regression-report +.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-loading profile-selected-forms profile-competitive profile performance-report performance-regression-report From 395832062c91b49951916bc7d644e783c47c1949 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 20:44:20 -0800 Subject: [PATCH 09/29] merge some require forms --- qi-sdk/profile/competitive/competitive.rkt | 5 ++--- qi-sdk/profile/forms/report.rkt | 8 +++----- qi-sdk/profile/loading/report.rkt | 7 +++---- qi-sdk/profile/report.rkt | 11 +++++------ 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/qi-sdk/profile/competitive/competitive.rkt b/qi-sdk/profile/competitive/competitive.rkt index 8547d11c0..9a74b39ec 100755 --- a/qi-sdk/profile/competitive/competitive.rkt +++ b/qi-sdk/profile/competitive/competitive.rkt @@ -10,9 +10,8 @@ (only-in racket/function curryr) (prefix-in q: "qi.rkt") - (prefix-in b: "builtin.rkt")) - -(require "../util.rkt") + (prefix-in b: "builtin.rkt") + "../util.rkt") (displayln "\nRunning flat benchmarks...") diff --git a/qi-sdk/profile/forms/report.rkt b/qi-sdk/profile/forms/report.rkt index 753a8166d..73e96eea9 100755 --- a/qi-sdk/profile/forms/report.rkt +++ b/qi-sdk/profile/forms/report.rkt @@ -1,8 +1,6 @@ #!/usr/bin/env racket #lang cli -(require "../regression.rkt") - (require racket/match racket/format relation @@ -11,9 +9,9 @@ only-if for/call write-csv - format-output)) -(require - (submod "benchmarks.rkt" main)) + format-output) + "../regression.rkt" + (submod "benchmarks.rkt" main)) (flag (forms #:param [forms null] name) ("-f" "--form" "Forms to benchmark") diff --git a/qi-sdk/profile/loading/report.rkt b/qi-sdk/profile/loading/report.rkt index f6542cd10..147b0d62b 100755 --- a/qi-sdk/profile/loading/report.rkt +++ b/qi-sdk/profile/loading/report.rkt @@ -1,9 +1,6 @@ #!/usr/bin/env racket #lang cli -(require "../regression.rkt" - "loadlib.rkt") - (require racket/match racket/format relation @@ -12,7 +9,9 @@ only-if for/call write-csv - format-output)) + format-output) + "../regression.rkt" + "loadlib.rkt") (help (usage diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index 6150bb36a..576a73a15 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -1,9 +1,6 @@ #!/usr/bin/env racket #lang cli -(require "loading/loadlib.rkt" - "regression.rkt") - (require racket/match racket/format relation @@ -12,9 +9,11 @@ only-if for/call write-csv - format-output)) -(require - (submod "forms/benchmarks.rkt" main)) + format-output) + + "loading/loadlib.rkt" + "regression.rkt" + (submod "forms/benchmarks.rkt" main)) (flag (forms #:param [forms null] name) ("-f" "--form" "Forms to benchmark") From 662a0bc44ab8325f8444c8b8f55d2ead23c2cffc Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 21:10:13 -0800 Subject: [PATCH 10/29] categorize performance modules into intrinsic vs competitive --- Makefile | 10 +++++----- qi-sdk/profile/{ => intrinsic}/forms/base.rkt | 4 ++-- qi-sdk/profile/{ => intrinsic}/forms/benchmarks.rkt | 2 +- qi-sdk/profile/{ => intrinsic}/forms/report.rkt | 2 +- qi-sdk/profile/{ => intrinsic}/loading/loadlib.rkt | 0 qi-sdk/profile/{ => intrinsic}/loading/report.rkt | 2 +- qi-sdk/profile/{ => intrinsic}/regression.rkt | 0 qi-sdk/profile/{ => intrinsic}/report.rkt | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) rename qi-sdk/profile/{ => intrinsic}/forms/base.rkt (67%) rename qi-sdk/profile/{ => intrinsic}/forms/benchmarks.rkt (99%) rename qi-sdk/profile/{ => intrinsic}/forms/report.rkt (97%) rename qi-sdk/profile/{ => intrinsic}/loading/loadlib.rkt (100%) rename qi-sdk/profile/{ => intrinsic}/loading/report.rkt (96%) rename qi-sdk/profile/{ => intrinsic}/regression.rkt (100%) rename qi-sdk/profile/{ => intrinsic}/report.rkt (98%) diff --git a/Makefile b/Makefile index d96e20b05..9014d2bc6 100644 --- a/Makefile +++ b/Makefile @@ -171,14 +171,14 @@ cover-coveralls: profile-forms: echo "Profiling forms..." - racket $(PACKAGE-NAME)-sdk/profile/forms/report.rkt + racket $(PACKAGE-NAME)-sdk/profile/intrinsic/forms/report.rkt profile-loading: echo "Profiling module loading..." - racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt + racket $(PACKAGE-NAME)-sdk/profile/intrinsic/loading/report.rkt profile-selected-forms: - @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/forms/report.rkt' directly, with -f form-name for each form." + @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/intrinsic/forms/report.rkt' directly, with -f form-name for each form." profile-competitive: echo "Running competitive benchmarks..." @@ -187,9 +187,9 @@ profile-competitive: profile: profile-competitive profile-forms performance-report: - @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -o json + @racket $(PACKAGE-NAME)-sdk/profile/intrinsic/report.rkt -o json performance-regression-report: - @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF) + @racket $(PACKAGE-NAME)-sdk/profile/intrinsic/report.rkt -r $(REF) .PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-loading profile-selected-forms profile-competitive profile performance-report performance-regression-report diff --git a/qi-sdk/profile/forms/base.rkt b/qi-sdk/profile/intrinsic/forms/base.rkt similarity index 67% rename from qi-sdk/profile/forms/base.rkt rename to qi-sdk/profile/intrinsic/forms/base.rkt index 7431b112f..a3ccd9fc7 100644 --- a/qi-sdk/profile/forms/base.rkt +++ b/qi-sdk/profile/intrinsic/forms/base.rkt @@ -2,9 +2,9 @@ (provide (all-from-out racket/base) (all-from-out qi) - (all-from-out "../util.rkt") + (all-from-out "../../util.rkt") sqr) (require qi - "../util.rkt" + "../../util.rkt" (only-in math sqr)) diff --git a/qi-sdk/profile/forms/benchmarks.rkt b/qi-sdk/profile/intrinsic/forms/benchmarks.rkt similarity index 99% rename from qi-sdk/profile/forms/benchmarks.rkt rename to qi-sdk/profile/intrinsic/forms/benchmarks.rkt index 84e8d0713..0decd7036 100755 --- a/qi-sdk/profile/forms/benchmarks.rkt +++ b/qi-sdk/profile/intrinsic/forms/benchmarks.rkt @@ -900,7 +900,7 @@ for the forms are run. qi json csv-writing - (only-in "../util.rkt" + (only-in "../../util.rkt" only-if for/call)) (require diff --git a/qi-sdk/profile/forms/report.rkt b/qi-sdk/profile/intrinsic/forms/report.rkt similarity index 97% rename from qi-sdk/profile/forms/report.rkt rename to qi-sdk/profile/intrinsic/forms/report.rkt index 73e96eea9..d55fc1a04 100755 --- a/qi-sdk/profile/forms/report.rkt +++ b/qi-sdk/profile/intrinsic/forms/report.rkt @@ -5,7 +5,7 @@ racket/format relation qi - (only-in "../util.rkt" + (only-in "../../util.rkt" only-if for/call write-csv diff --git a/qi-sdk/profile/loading/loadlib.rkt b/qi-sdk/profile/intrinsic/loading/loadlib.rkt similarity index 100% rename from qi-sdk/profile/loading/loadlib.rkt rename to qi-sdk/profile/intrinsic/loading/loadlib.rkt diff --git a/qi-sdk/profile/loading/report.rkt b/qi-sdk/profile/intrinsic/loading/report.rkt similarity index 96% rename from qi-sdk/profile/loading/report.rkt rename to qi-sdk/profile/intrinsic/loading/report.rkt index 147b0d62b..47cdf2411 100755 --- a/qi-sdk/profile/loading/report.rkt +++ b/qi-sdk/profile/intrinsic/loading/report.rkt @@ -5,7 +5,7 @@ racket/format relation qi - (only-in "../util.rkt" + (only-in "../../util.rkt" only-if for/call write-csv diff --git a/qi-sdk/profile/regression.rkt b/qi-sdk/profile/intrinsic/regression.rkt similarity index 100% rename from qi-sdk/profile/regression.rkt rename to qi-sdk/profile/intrinsic/regression.rkt diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/intrinsic/report.rkt similarity index 98% rename from qi-sdk/profile/report.rkt rename to qi-sdk/profile/intrinsic/report.rkt index 576a73a15..a8deeab89 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/intrinsic/report.rkt @@ -5,7 +5,7 @@ racket/format relation qi - (only-in "util.rkt" + (only-in "../util.rkt" only-if for/call write-csv From 25a267ac196e70262e75b0b3da817109352bef11 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 7 Mar 2023 21:46:41 -0800 Subject: [PATCH 11/29] rename a file for uniformity --- Makefile | 2 +- qi-sdk/profile/competitive/{competitive.rkt => report.rkt} | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) rename qi-sdk/profile/competitive/{competitive.rkt => report.rkt} (93%) diff --git a/Makefile b/Makefile index 9014d2bc6..f9ff618e2 100644 --- a/Makefile +++ b/Makefile @@ -182,7 +182,7 @@ profile-selected-forms: profile-competitive: echo "Running competitive benchmarks..." - racket $(PACKAGE-NAME)-sdk/profile/competitive/competitive.rkt + racket $(PACKAGE-NAME)-sdk/profile/competitive/report.rkt profile: profile-competitive profile-forms diff --git a/qi-sdk/profile/competitive/competitive.rkt b/qi-sdk/profile/competitive/report.rkt similarity index 93% rename from qi-sdk/profile/competitive/competitive.rkt rename to qi-sdk/profile/competitive/report.rkt index 9a74b39ec..711b21a7e 100755 --- a/qi-sdk/profile/competitive/competitive.rkt +++ b/qi-sdk/profile/competitive/report.rkt @@ -15,6 +15,10 @@ (displayln "\nRunning flat benchmarks...") +;; TODO: make these display the results as "side effects" +;; in STDERR like the intrinsic benchmarking scripts. +;; and configurable via CLI flags + (run-competitive-benchmark "Conditionals" check-value cond-fn From f82f1baa065cf940183771463771336fa4ad6bbe Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 14 Mar 2023 22:38:12 -0700 Subject: [PATCH 12/29] begin refactor of competitive benchmarks for uniformity/tractability --- Makefile | 2 +- qi-sdk/profile/competitive/intrinsic.rkt | 19 +++ .../competitive/{qi.rkt => qi/main.rkt} | 0 .../{builtin.rkt => racket/main.rkt} | 0 qi-sdk/profile/competitive/report.rkt | 122 ++++++------------ qi-sdk/profile/competitive/spec.rkt | 56 ++++++++ qi-sdk/profile/intrinsic/forms/report.rkt | 2 + qi-sdk/profile/util.rkt | 29 ++--- 8 files changed, 132 insertions(+), 98 deletions(-) create mode 100755 qi-sdk/profile/competitive/intrinsic.rkt rename qi-sdk/profile/competitive/{qi.rkt => qi/main.rkt} (100%) rename qi-sdk/profile/competitive/{builtin.rkt => racket/main.rkt} (100%) create mode 100644 qi-sdk/profile/competitive/spec.rkt diff --git a/Makefile b/Makefile index f9ff618e2..0f0de57e6 100644 --- a/Makefile +++ b/Makefile @@ -182,7 +182,7 @@ profile-selected-forms: profile-competitive: echo "Running competitive benchmarks..." - racket $(PACKAGE-NAME)-sdk/profile/competitive/report.rkt + cd $(PACKAGE-NAME)-sdk/profile/competitive; racket report.rkt profile: profile-competitive profile-forms diff --git a/qi-sdk/profile/competitive/intrinsic.rkt b/qi-sdk/profile/competitive/intrinsic.rkt new file mode 100755 index 000000000..5dd6b791e --- /dev/null +++ b/qi-sdk/profile/competitive/intrinsic.rkt @@ -0,0 +1,19 @@ +#!/usr/bin/env racket +#lang cli + +(provide benchmark) + +(require "../util.rkt" + "spec.rkt") + +(define (benchmark language) + (define namespace (make-base-namespace)) + (cond [(eq? 'qi language) (eval '(require "qi/main.rkt") namespace)] + [(eq? 'racket language) (eval '(require "racket/main.rkt") namespace)]) + + (for/list ([spec specs]) + (let ([name (bm-name spec)] + [exerciser (bm-exerciser spec)] + [f (eval (read (open-input-string (bm-target spec))) namespace)] + [n-times (bm-times spec)]) + (run-nonlocal-benchmark name exerciser f n-times)))) diff --git a/qi-sdk/profile/competitive/qi.rkt b/qi-sdk/profile/competitive/qi/main.rkt similarity index 100% rename from qi-sdk/profile/competitive/qi.rkt rename to qi-sdk/profile/competitive/qi/main.rkt diff --git a/qi-sdk/profile/competitive/builtin.rkt b/qi-sdk/profile/competitive/racket/main.rkt similarity index 100% rename from qi-sdk/profile/competitive/builtin.rkt rename to qi-sdk/profile/competitive/racket/main.rkt diff --git a/qi-sdk/profile/competitive/report.rkt b/qi-sdk/profile/competitive/report.rkt index 711b21a7e..575691144 100755 --- a/qi-sdk/profile/competitive/report.rkt +++ b/qi-sdk/profile/competitive/report.rkt @@ -1,80 +1,44 @@ #!/usr/bin/env racket -#lang racket/base - -(require (only-in data/collection - cycle - take - in) - (only-in racket/list - range) - (only-in racket/function - curryr) - (prefix-in q: "qi.rkt") - (prefix-in b: "builtin.rkt") - "../util.rkt") - -(displayln "\nRunning flat benchmarks...") - -;; TODO: make these display the results as "side effects" -;; in STDERR like the intrinsic benchmarking scripts. -;; and configurable via CLI flags - -(run-competitive-benchmark "Conditionals" - check-value - cond-fn - 300000) - -(run-competitive-benchmark "Composition" - check-value - compose-fn - 300000) - -(run-competitive-benchmark "Root Mean Square" - check-list - root-mean-square - 500000) - -(run-competitive-benchmark "Filter-map" - check-list - filter-map-fn - 500000) - -(run-competitive-benchmark "Filter-map values" - check-values - filter-map-values - 500000) - -(run-competitive-benchmark "Double list" - check-list - double-list - 500000) - -(run-competitive-benchmark "Double values" - check-values - double-values - 500000) - -(displayln "\nRunning Recursive benchmarks...") - -(run-competitive-benchmark "Factorial" - check-value - fact - 100000) - -(run-competitive-benchmark "Pingala" - check-value - ping - 10000) - -(define check-value-primes (curryr check-value #(100 200 300))) - -(run-competitive-benchmark "Eratosthenes" - check-value-primes - eratos - 100) - -;; See https://en.wikipedia.org/wiki/Collatz_conjecture -(run-competitive-benchmark "Collatz" - check-value - collatz - 10000) +#lang cli + +(require racket/match + racket/format + relation + qi + (only-in "../util.rkt" + only-if + for/call + write-csv + format-output) + "../intrinsic/regression.rkt" + "intrinsic.rkt") + +(help + (usage + (~a "Run competitive benchmarks between Qi and Racket, " + "reporting the results in a configurable output format."))) + +(flag (output-format #:param [output-format ""] fmt) + ("-o" + "--format" + "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") + (output-format fmt)) + +(flag (regression-file #:param [regression-file #f] reg-file) + ("-r" "--regression" "'Before' data to compute regression against") + (regression-file reg-file)) + +(program (main) + (displayln "\nRunning competitive benchmarks..." (current-error-port)) + + (let ([output (benchmark 'qi)]) + (if (regression-file) + (let ([before (parse-benchmarks (parse-json-file (regression-file)))] + [after (parse-benchmarks output)]) + (compute-regression before after)) + (format-output output (output-format))))) + +;; ;; To run benchmarks for a form interactively, use e.g.: +;; ;; (run main #("-f" "fanout")) + +(run main) diff --git a/qi-sdk/profile/competitive/spec.rkt b/qi-sdk/profile/competitive/spec.rkt new file mode 100644 index 000000000..4e589fd37 --- /dev/null +++ b/qi-sdk/profile/competitive/spec.rkt @@ -0,0 +1,56 @@ +#lang racket/base + +(provide specs + (struct-out bm)) + +(require "../util.rkt") + +(struct bm (name exerciser target times) + #:transparent) + +(define specs + (list (bm "Conditionals" + check-value + "cond-fn" + 300000) + (bm "Composition" + check-value + "compose-fn" + 300000) + (bm "Root Mean Square" + check-list + "root-mean-square" + 500000) + (bm "Filter-map" + check-list + "filter-map-fn" + 500000) + (bm "Filter-map values" + check-values + "filter-map-values" + 500000) + (bm "Double list" + check-list + "double-list" + 500000) + (bm "Double values" + check-values + "double-values" + 500000) + (bm "Factorial" + check-value + "fact" + 100000) + (bm "Pingala" + check-value + "ping" + 10000) + (bm "Eratosthenes" + check-value-primes + "eratos" + 100) + ;; See https://en.wikipedia.org/wiki/Collatz_conjecture + (bm "Collatz" + check-value + "collatz" + 10000))) diff --git a/qi-sdk/profile/intrinsic/forms/report.rkt b/qi-sdk/profile/intrinsic/forms/report.rkt index d55fc1a04..a21b33af7 100755 --- a/qi-sdk/profile/intrinsic/forms/report.rkt +++ b/qi-sdk/profile/intrinsic/forms/report.rkt @@ -38,6 +38,8 @@ (program (main) (let ([output (benchmark (forms))]) (if (regression-file) + ;; TODO: regression ignores any flags and is a parallel path + ;; it should be properly incorporated into the CLI (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) (compute-regression before after)) diff --git a/qi-sdk/profile/util.rkt b/qi-sdk/profile/util.rkt index 40c8aa61b..0ba2951b5 100644 --- a/qi-sdk/profile/util.rkt +++ b/qi-sdk/profile/util.rkt @@ -3,12 +3,13 @@ (provide average measure check-value + check-value-primes check-list check-values check-two-values run-benchmark run-summary-benchmark - run-competitive-benchmark + run-nonlocal-benchmark (for-space qi only-if) for/call write-csv @@ -17,6 +18,8 @@ (require (only-in racket/list range second) + (only-in racket/function + curryr) (only-in adjutor values->list) (only-in data/collection @@ -58,6 +61,8 @@ (set! i (remainder (add1 i) len)) (fn (vector-ref inputs i))))) +(define check-value-primes (curryr check-value #(100 200 300))) + ;; This uses the same list input each time. Not sure if that ;; may end up being cached at some level and thus obfuscate ;; the results? On the other hand, @@ -122,23 +127,11 @@ ;; Run different implementations of the same benchmark (e.g. a Racket vs a Qi ;; implementation) a specified number of times, and report the time taken ;; by each implementation. -(define-syntax-parse-rule (run-competitive-benchmark name runner f-name n-times) - #:with f-builtin (datum->syntax #'name - (string->symbol - (string-append "b:" - (symbol->string - (syntax->datum #'f-name))))) - #:with f-qi (datum->syntax #'name - (string->symbol - (string-append "q:" - (symbol->string - (syntax->datum #'f-name))))) - (begin - (displayln (~a name ":")) - (for ([f (list f-builtin f-qi)] - [label (list "λ" "☯")]) - (let ([ms (measure runner f n-times)]) - (displayln (~a label ": " ms " ms")))))) +(define (run-nonlocal-benchmark bm-name runner f n-times) + (displayln (~a bm-name ":") (current-error-port)) + (let ([ms (measure runner f n-times)]) + (displayln (~a ms " ms") (current-error-port)) + (hash 'name bm-name 'unit "ms" 'value ms))) (define (write-csv data) (~> (data) From e967c1ba867603872aa89d7ecac058c7e89942ea Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 14 Mar 2023 23:50:38 -0700 Subject: [PATCH 13/29] label a todo so it doesn't get lost --- qi-doc/scribblings/field-guide.scrbl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qi-doc/scribblings/field-guide.scrbl b/qi-doc/scribblings/field-guide.scrbl index 659e4cd99..ea3c21c75 100644 --- a/qi-doc/scribblings/field-guide.scrbl +++ b/qi-doc/scribblings/field-guide.scrbl @@ -324,7 +324,7 @@ Another way to do it is to simply promote the expression out of the nest: (~> (3) (get-f 1)) ] -@;{Update this to reflect new partial application behavior} +@;{TODO: Update this to reflect new partial application behavior} Now, you might, once again, expect this to be treated as a partial application template, so that this would be equivalent to @racket[(get-f 3 1)] and would raise an error. But in fact, since the expression @racket[(get-f 1)] happens to be fully qualified with all the arguments it needs, the currying employed under the hood to implement partial application in this case @seclink["Using_Racket_to_Define_Flows"]{evaluates to a function result right away}. This then receives the value @racket[3], and consequently, this expression produces the correct result. So in sum, it's perhaps best to rely on @racket[esc] in such cases to be as explicit as possible about what you mean, rather than rely on quirks of the implementation that are revealed at this boundary between two languages. From a4517fa54dc457f2ba5c8e99afcc1a1c99b8b4f2 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 14 Mar 2023 23:51:49 -0700 Subject: [PATCH 14/29] standardize nonlocal benchmark names for use via CLI --- qi-sdk/profile/competitive/qi/main.rkt | 28 ++++++++--------- qi-sdk/profile/competitive/racket/main.rkt | 30 +++++++++---------- qi-sdk/profile/competitive/spec.rkt | 35 ++++++++-------------- 3 files changed, 41 insertions(+), 52 deletions(-) diff --git a/qi-sdk/profile/competitive/qi/main.rkt b/qi-sdk/profile/competitive/qi/main.rkt index d15594bb6..c4dc012c5 100644 --- a/qi-sdk/profile/competitive/qi/main.rkt +++ b/qi-sdk/profile/competitive/qi/main.rkt @@ -1,13 +1,13 @@ #lang racket/base -(provide cond-fn - compose-fn +(provide conditionals + composition root-mean-square - fact - ping - eratos + factorial + pingala + eratosthenes collatz - filter-map-fn + filter-map filter-map-values double-list double-values) @@ -16,28 +16,28 @@ (only-in racket/list range) qi) -(define-switch cond-fn +(define-switch conditionals [(< 5) sqr] [(> 5) add1] [else _]) -(define-flow compose-fn +(define-flow composition (~> add1 sqr sub1)) (define-flow root-mean-square (~> (-< (~>> △ (>< sqr) +) length) / sqrt)) -(define-switch fact +(define-switch factorial [(< 2) 1] - [else (~> (-< _ (~> sub1 fact)) *)]) + [else (~> (-< _ (~> sub1 factorial)) *)]) -(define-switch ping +(define-switch pingala [(< 2) _] [else (~> (-< sub1 - (- 2)) (>< ping) +)]) + (- 2)) (>< pingala) +)]) -(define-flow (eratos n) +(define-flow (eratosthenes n) (~> (-< (gen null) (~>> add1 (range 2) △)) (feedback (while (~> (block 1) live?)) (then (~> 1> reverse)) @@ -54,7 +54,7 @@ cons)])) -(define-flow filter-map-fn +(define-flow filter-map (~> △ (>< (if odd? sqr ⏚)) ▽)) (define-flow filter-map-values diff --git a/qi-sdk/profile/competitive/racket/main.rkt b/qi-sdk/profile/competitive/racket/main.rkt index 30351831d..4e80ae24f 100644 --- a/qi-sdk/profile/competitive/racket/main.rkt +++ b/qi-sdk/profile/competitive/racket/main.rkt @@ -1,13 +1,13 @@ #lang racket/base -(provide cond-fn - compose-fn +(provide conditionals + composition root-mean-square - fact - ping - eratos + factorial + pingala + eratosthenes collatz - filter-map-fn + filter-map filter-map-values double-list double-values) @@ -16,30 +16,30 @@ racket/list racket/match) -(define (cond-fn x) +(define (conditionals x) (cond [(< x 5) (sqr x)] [(> x 5) (add1 x)] [else x])) -(define (compose-fn v) +(define (composition v) (sub1 (sqr (add1 v)))) (define (root-mean-square vs) (sqrt (/ (apply + (map sqr vs)) (length vs)))) -(define (fact n) +(define (factorial n) (if (< n 2) 1 - (* (fact (sub1 n)) n))) + (* (factorial (sub1 n)) n))) -(define (ping n) +(define (pingala n) (if (< n 2) n - (+ (ping (sub1 n)) - (ping (- n 2))))) + (+ (pingala (sub1 n)) + (pingala (- n 2))))) -(define (eratos n) +(define (eratosthenes n) (let ([lst (range 2 (add1 n))]) (let loop ([rem lst] [result null]) @@ -55,7 +55,7 @@ [(odd? n) (cons n (collatz (+ (* 3 n) 1)))] [(even? n) (cons n (collatz (quotient n 2)))])) -(define (filter-map-fn lst) +(define (filter-map lst) (map sqr (filter odd? lst))) (define (filter-map-values . vs) diff --git a/qi-sdk/profile/competitive/spec.rkt b/qi-sdk/profile/competitive/spec.rkt index 4e589fd37..addaa412d 100644 --- a/qi-sdk/profile/competitive/spec.rkt +++ b/qi-sdk/profile/competitive/spec.rkt @@ -5,52 +5,41 @@ (require "../util.rkt") -(struct bm (name exerciser target times) +(struct bm (name exerciser times) #:transparent) (define specs - (list (bm "Conditionals" + (list (bm "conditionals" check-value - "cond-fn" 300000) - (bm "Composition" + (bm "composition" check-value - "compose-fn" 300000) - (bm "Root Mean Square" + (bm "root-mean-square" check-list - "root-mean-square" 500000) - (bm "Filter-map" + (bm "filter-map" check-list - "filter-map-fn" 500000) - (bm "Filter-map values" + (bm "filter-map-values" check-values - "filter-map-values" 500000) - (bm "Double list" + (bm "double-list" check-list - "double-list" 500000) - (bm "Double values" + (bm "double-values" check-values - "double-values" 500000) - (bm "Factorial" + (bm "factorial" check-value - "fact" 100000) - (bm "Pingala" + (bm "pingala" check-value - "ping" 10000) - (bm "Eratosthenes" + (bm "eratosthenes" check-value-primes - "eratos" 100) ;; See https://en.wikipedia.org/wiki/Collatz_conjecture - (bm "Collatz" + (bm "collatz" check-value - "collatz" 10000))) From c213a94c2ce8f289dae520a7604a46ac382854e6 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 14 Mar 2023 23:54:47 -0700 Subject: [PATCH 15/29] support selecting specific nonlocal benchmarks to run via CLI --- qi-sdk/profile/competitive/intrinsic.rkt | 24 ++++++++++++++---------- qi-sdk/profile/competitive/report.rkt | 8 +++++++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/qi-sdk/profile/competitive/intrinsic.rkt b/qi-sdk/profile/competitive/intrinsic.rkt index 5dd6b791e..1b75772ed 100755 --- a/qi-sdk/profile/competitive/intrinsic.rkt +++ b/qi-sdk/profile/competitive/intrinsic.rkt @@ -6,14 +6,18 @@ (require "../util.rkt" "spec.rkt") -(define (benchmark language) - (define namespace (make-base-namespace)) - (cond [(eq? 'qi language) (eval '(require "qi/main.rkt") namespace)] - [(eq? 'racket language) (eval '(require "racket/main.rkt") namespace)]) +(define (benchmark language benchmarks-to-run) + (let ([namespace (make-base-namespace)] + [benchmarks-to-run (if (null? benchmarks-to-run) + (map bm-name specs) + benchmarks-to-run)]) + (cond [(eq? 'qi language) (eval '(require "qi/main.rkt") namespace)] + [(eq? 'racket language) (eval '(require "racket/main.rkt") namespace)]) - (for/list ([spec specs]) - (let ([name (bm-name spec)] - [exerciser (bm-exerciser spec)] - [f (eval (read (open-input-string (bm-target spec))) namespace)] - [n-times (bm-times spec)]) - (run-nonlocal-benchmark name exerciser f n-times)))) + (for/list ([spec specs] + #:when (member (bm-name spec) benchmarks-to-run)) + (let ([name (bm-name spec)] + [exerciser (bm-exerciser spec)] + [f (eval (read (open-input-string (bm-name spec))) namespace)] + [n-times (bm-times spec)]) + (run-nonlocal-benchmark name exerciser f n-times))))) diff --git a/qi-sdk/profile/competitive/report.rkt b/qi-sdk/profile/competitive/report.rkt index 575691144..f59413a0a 100755 --- a/qi-sdk/profile/competitive/report.rkt +++ b/qi-sdk/profile/competitive/report.rkt @@ -13,6 +13,12 @@ "../intrinsic/regression.rkt" "intrinsic.rkt") +(flag (selected #:param [selected null] name) + ("-s" "--select" "Select benchmark by name") + (selected (cons name (selected)))) + +(constraint (multi selected)) + (help (usage (~a "Run competitive benchmarks between Qi and Racket, " @@ -31,7 +37,7 @@ (program (main) (displayln "\nRunning competitive benchmarks..." (current-error-port)) - (let ([output (benchmark 'qi)]) + (let ([output (benchmark 'qi (selected))]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) From 9411eaf6a96fb1ca69baea1774f03b8028edbf66 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 14 Mar 2023 23:55:28 -0700 Subject: [PATCH 16/29] standardize flag conventions --- qi-sdk/profile/competitive/report.rkt | 8 ++++---- qi-sdk/profile/intrinsic/forms/report.rkt | 14 +++++++------- qi-sdk/profile/intrinsic/loading/report.rkt | 2 +- qi-sdk/profile/intrinsic/report.rkt | 14 +++++++------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/qi-sdk/profile/competitive/report.rkt b/qi-sdk/profile/competitive/report.rkt index f59413a0a..029e17a68 100755 --- a/qi-sdk/profile/competitive/report.rkt +++ b/qi-sdk/profile/competitive/report.rkt @@ -25,7 +25,7 @@ "reporting the results in a configurable output format."))) (flag (output-format #:param [output-format ""] fmt) - ("-o" + ("-f" "--format" "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) @@ -44,7 +44,7 @@ (compute-regression before after)) (format-output output (output-format))))) -;; ;; To run benchmarks for a form interactively, use e.g.: -;; ;; (run main #("-f" "fanout")) +;; To run benchmarks for a form interactively, use e.g.: +;; (run main #("-s" "composition")) -(run main) +(run main #("-s" "composition")) diff --git a/qi-sdk/profile/intrinsic/forms/report.rkt b/qi-sdk/profile/intrinsic/forms/report.rkt index a21b33af7..4e0fef1ba 100755 --- a/qi-sdk/profile/intrinsic/forms/report.rkt +++ b/qi-sdk/profile/intrinsic/forms/report.rkt @@ -13,11 +13,11 @@ "../regression.rkt" (submod "benchmarks.rkt" main)) -(flag (forms #:param [forms null] name) - ("-f" "--form" "Forms to benchmark") - (forms (cons name (forms)))) +(flag (selected #:param [selected null] name) + ("-s" "--select" "Select form to benchmark") + (selected (cons name (selected)))) -(constraint (multi forms)) +(constraint (multi selected)) (help (usage @@ -26,7 +26,7 @@ "in a configurable output format."))) (flag (output-format #:param [output-format ""] fmt) - ("-o" + ("-f" "--format" "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) @@ -36,7 +36,7 @@ (regression-file reg-file)) (program (main) - (let ([output (benchmark (forms))]) + (let ([output (benchmark (selected))]) (if (regression-file) ;; TODO: regression ignores any flags and is a parallel path ;; it should be properly incorporated into the CLI @@ -46,6 +46,6 @@ (format-output output (output-format))))) ;; To run benchmarks for a form interactively, use e.g.: -;; (run main #("-f" "fanout")) +;; (run main #("-s" "fanout")) (run main) diff --git a/qi-sdk/profile/intrinsic/loading/report.rkt b/qi-sdk/profile/intrinsic/loading/report.rkt index 47cdf2411..90f415a35 100755 --- a/qi-sdk/profile/intrinsic/loading/report.rkt +++ b/qi-sdk/profile/intrinsic/loading/report.rkt @@ -18,7 +18,7 @@ (~a "Measure module load time, i.e. the time taken by (require qi)."))) (flag (output-format #:param [output-format ""] fmt) - ("-o" + ("-f" "--format" "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) diff --git a/qi-sdk/profile/intrinsic/report.rkt b/qi-sdk/profile/intrinsic/report.rkt index a8deeab89..02996fab8 100755 --- a/qi-sdk/profile/intrinsic/report.rkt +++ b/qi-sdk/profile/intrinsic/report.rkt @@ -15,11 +15,11 @@ "regression.rkt" (submod "forms/benchmarks.rkt" main)) -(flag (forms #:param [forms null] name) - ("-f" "--form" "Forms to benchmark") - (forms (cons name (forms)))) +(flag (selected #:param [selected null] name) + ("-s" "--select" "Select form to benchmark") + (selected (cons name (selected)))) -(constraint (multi forms)) +(constraint (multi selected)) (help (usage @@ -28,7 +28,7 @@ "in a configurable output format."))) (flag (output-format #:param [output-format ""] fmt) - ("-o" + ("-f" "--format" "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) @@ -49,7 +49,7 @@ ;; https://github.com/countvajhula/cli/issues/3 (program (main) (let* ([forms-data (if (member? (report-type) (list "all" "forms")) - (benchmark (forms)) + (benchmark (selected)) null)] [require-data (if (member? (report-type) (list "all" "loading")) (list (profile-load "qi")) @@ -62,6 +62,6 @@ (format-output output (output-format))))) ;; To run benchmarks for a form interactively, use e.g.: -;; (run main #("-f" "fanout")) +;; (run main #("-s" "fanout")) (run main) From 09b24c4ac57843bfd1136ed486f45f4bc3fe1ba1 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 00:05:38 -0700 Subject: [PATCH 17/29] reorganize benchmarks as local and nonlocal --- Makefile | 6 +++--- qi-sdk/profile/{intrinsic => local}/forms/base.rkt | 0 qi-sdk/profile/{intrinsic => local}/forms/benchmarks.rkt | 0 qi-sdk/profile/{intrinsic => local}/forms/report.rkt | 2 +- qi-sdk/profile/{intrinsic => local}/loading/loadlib.rkt | 0 qi-sdk/profile/{intrinsic => local}/loading/report.rkt | 2 +- qi-sdk/profile/{intrinsic => local}/report.rkt | 2 +- qi-sdk/profile/{competitive => nonlocal}/intrinsic.rkt | 0 qi-sdk/profile/{competitive => nonlocal}/qi/main.rkt | 0 qi-sdk/profile/{competitive => nonlocal}/racket/main.rkt | 0 qi-sdk/profile/{competitive => nonlocal}/report.rkt | 4 ++-- qi-sdk/profile/{competitive => nonlocal}/spec.rkt | 0 qi-sdk/profile/{intrinsic => }/regression.rkt | 0 13 files changed, 8 insertions(+), 8 deletions(-) rename qi-sdk/profile/{intrinsic => local}/forms/base.rkt (100%) rename qi-sdk/profile/{intrinsic => local}/forms/benchmarks.rkt (100%) rename qi-sdk/profile/{intrinsic => local}/forms/report.rkt (97%) rename qi-sdk/profile/{intrinsic => local}/loading/loadlib.rkt (100%) rename qi-sdk/profile/{intrinsic => local}/loading/report.rkt (97%) rename qi-sdk/profile/{intrinsic => local}/report.rkt (98%) rename qi-sdk/profile/{competitive => nonlocal}/intrinsic.rkt (100%) rename qi-sdk/profile/{competitive => nonlocal}/qi/main.rkt (100%) rename qi-sdk/profile/{competitive => nonlocal}/racket/main.rkt (100%) rename qi-sdk/profile/{competitive => nonlocal}/report.rkt (95%) rename qi-sdk/profile/{competitive => nonlocal}/spec.rkt (100%) rename qi-sdk/profile/{intrinsic => }/regression.rkt (100%) diff --git a/Makefile b/Makefile index 0f0de57e6..f7efcae88 100644 --- a/Makefile +++ b/Makefile @@ -182,14 +182,14 @@ profile-selected-forms: profile-competitive: echo "Running competitive benchmarks..." - cd $(PACKAGE-NAME)-sdk/profile/competitive; racket report.rkt + cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report.rkt profile: profile-competitive profile-forms performance-report: - @racket $(PACKAGE-NAME)-sdk/profile/intrinsic/report.rkt -o json + @racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt -f json performance-regression-report: - @racket $(PACKAGE-NAME)-sdk/profile/intrinsic/report.rkt -r $(REF) + @racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt -r $(REF) .PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-loading profile-selected-forms profile-competitive profile performance-report performance-regression-report diff --git a/qi-sdk/profile/intrinsic/forms/base.rkt b/qi-sdk/profile/local/forms/base.rkt similarity index 100% rename from qi-sdk/profile/intrinsic/forms/base.rkt rename to qi-sdk/profile/local/forms/base.rkt diff --git a/qi-sdk/profile/intrinsic/forms/benchmarks.rkt b/qi-sdk/profile/local/forms/benchmarks.rkt similarity index 100% rename from qi-sdk/profile/intrinsic/forms/benchmarks.rkt rename to qi-sdk/profile/local/forms/benchmarks.rkt diff --git a/qi-sdk/profile/intrinsic/forms/report.rkt b/qi-sdk/profile/local/forms/report.rkt similarity index 97% rename from qi-sdk/profile/intrinsic/forms/report.rkt rename to qi-sdk/profile/local/forms/report.rkt index 4e0fef1ba..9c1198bf9 100755 --- a/qi-sdk/profile/intrinsic/forms/report.rkt +++ b/qi-sdk/profile/local/forms/report.rkt @@ -10,7 +10,7 @@ for/call write-csv format-output) - "../regression.rkt" + "../../regression.rkt" (submod "benchmarks.rkt" main)) (flag (selected #:param [selected null] name) diff --git a/qi-sdk/profile/intrinsic/loading/loadlib.rkt b/qi-sdk/profile/local/loading/loadlib.rkt similarity index 100% rename from qi-sdk/profile/intrinsic/loading/loadlib.rkt rename to qi-sdk/profile/local/loading/loadlib.rkt diff --git a/qi-sdk/profile/intrinsic/loading/report.rkt b/qi-sdk/profile/local/loading/report.rkt similarity index 97% rename from qi-sdk/profile/intrinsic/loading/report.rkt rename to qi-sdk/profile/local/loading/report.rkt index 90f415a35..1e15eb84a 100755 --- a/qi-sdk/profile/intrinsic/loading/report.rkt +++ b/qi-sdk/profile/local/loading/report.rkt @@ -10,7 +10,7 @@ for/call write-csv format-output) - "../regression.rkt" + "../../regression.rkt" "loadlib.rkt") (help diff --git a/qi-sdk/profile/intrinsic/report.rkt b/qi-sdk/profile/local/report.rkt similarity index 98% rename from qi-sdk/profile/intrinsic/report.rkt rename to qi-sdk/profile/local/report.rkt index 02996fab8..f7e229d8a 100755 --- a/qi-sdk/profile/intrinsic/report.rkt +++ b/qi-sdk/profile/local/report.rkt @@ -12,7 +12,7 @@ format-output) "loading/loadlib.rkt" - "regression.rkt" + "../regression.rkt" (submod "forms/benchmarks.rkt" main)) (flag (selected #:param [selected null] name) diff --git a/qi-sdk/profile/competitive/intrinsic.rkt b/qi-sdk/profile/nonlocal/intrinsic.rkt similarity index 100% rename from qi-sdk/profile/competitive/intrinsic.rkt rename to qi-sdk/profile/nonlocal/intrinsic.rkt diff --git a/qi-sdk/profile/competitive/qi/main.rkt b/qi-sdk/profile/nonlocal/qi/main.rkt similarity index 100% rename from qi-sdk/profile/competitive/qi/main.rkt rename to qi-sdk/profile/nonlocal/qi/main.rkt diff --git a/qi-sdk/profile/competitive/racket/main.rkt b/qi-sdk/profile/nonlocal/racket/main.rkt similarity index 100% rename from qi-sdk/profile/competitive/racket/main.rkt rename to qi-sdk/profile/nonlocal/racket/main.rkt diff --git a/qi-sdk/profile/competitive/report.rkt b/qi-sdk/profile/nonlocal/report.rkt similarity index 95% rename from qi-sdk/profile/competitive/report.rkt rename to qi-sdk/profile/nonlocal/report.rkt index 029e17a68..61e549054 100755 --- a/qi-sdk/profile/competitive/report.rkt +++ b/qi-sdk/profile/nonlocal/report.rkt @@ -10,7 +10,7 @@ for/call write-csv format-output) - "../intrinsic/regression.rkt" + "../regression.rkt" "intrinsic.rkt") (flag (selected #:param [selected null] name) @@ -47,4 +47,4 @@ ;; To run benchmarks for a form interactively, use e.g.: ;; (run main #("-s" "composition")) -(run main #("-s" "composition")) +(run main) diff --git a/qi-sdk/profile/competitive/spec.rkt b/qi-sdk/profile/nonlocal/spec.rkt similarity index 100% rename from qi-sdk/profile/competitive/spec.rkt rename to qi-sdk/profile/nonlocal/spec.rkt diff --git a/qi-sdk/profile/intrinsic/regression.rkt b/qi-sdk/profile/regression.rkt similarity index 100% rename from qi-sdk/profile/intrinsic/regression.rkt rename to qi-sdk/profile/regression.rkt From 551b9a14fa2262c7a94cc77d4481632fda02a197 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 00:30:53 -0700 Subject: [PATCH 18/29] continue reorganizing benchmarks.. --- Makefile | 10 +++---- .../profile/{local => }/loading/loadlib.rkt | 0 qi-sdk/profile/{local => }/loading/report.rkt | 4 +-- qi-sdk/profile/local/{forms => }/base.rkt | 4 +-- .../profile/local/{forms => }/benchmarks.rkt | 2 +- qi-sdk/profile/local/report.rkt | 24 +++-------------- qi-sdk/profile/{local/forms => }/report.rkt | 27 ++++++++++++++----- 7 files changed, 35 insertions(+), 36 deletions(-) rename qi-sdk/profile/{local => }/loading/loadlib.rkt (100%) rename qi-sdk/profile/{local => }/loading/report.rkt (93%) rename qi-sdk/profile/local/{forms => }/base.rkt (67%) rename qi-sdk/profile/local/{forms => }/benchmarks.rkt (99%) rename qi-sdk/profile/{local/forms => }/report.rkt (59%) diff --git a/Makefile b/Makefile index f7efcae88..69777a9f3 100644 --- a/Makefile +++ b/Makefile @@ -171,14 +171,14 @@ cover-coveralls: profile-forms: echo "Profiling forms..." - racket $(PACKAGE-NAME)-sdk/profile/intrinsic/forms/report.rkt + racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt profile-loading: echo "Profiling module loading..." - racket $(PACKAGE-NAME)-sdk/profile/intrinsic/loading/report.rkt + racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt profile-selected-forms: - @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/intrinsic/forms/report.rkt' directly, with -f form-name for each form." + @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt' directly, with -f form-name for each form." profile-competitive: echo "Running competitive benchmarks..." @@ -187,9 +187,9 @@ profile-competitive: profile: profile-competitive profile-forms performance-report: - @racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt -f json + @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -f json performance-regression-report: - @racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt -r $(REF) + @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF) .PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-loading profile-selected-forms profile-competitive profile performance-report performance-regression-report diff --git a/qi-sdk/profile/local/loading/loadlib.rkt b/qi-sdk/profile/loading/loadlib.rkt similarity index 100% rename from qi-sdk/profile/local/loading/loadlib.rkt rename to qi-sdk/profile/loading/loadlib.rkt diff --git a/qi-sdk/profile/local/loading/report.rkt b/qi-sdk/profile/loading/report.rkt similarity index 93% rename from qi-sdk/profile/local/loading/report.rkt rename to qi-sdk/profile/loading/report.rkt index 1e15eb84a..8c56eddad 100755 --- a/qi-sdk/profile/local/loading/report.rkt +++ b/qi-sdk/profile/loading/report.rkt @@ -5,12 +5,12 @@ racket/format relation qi - (only-in "../../util.rkt" + (only-in "../util.rkt" only-if for/call write-csv format-output) - "../../regression.rkt" + "../regression.rkt" "loadlib.rkt") (help diff --git a/qi-sdk/profile/local/forms/base.rkt b/qi-sdk/profile/local/base.rkt similarity index 67% rename from qi-sdk/profile/local/forms/base.rkt rename to qi-sdk/profile/local/base.rkt index a3ccd9fc7..7431b112f 100644 --- a/qi-sdk/profile/local/forms/base.rkt +++ b/qi-sdk/profile/local/base.rkt @@ -2,9 +2,9 @@ (provide (all-from-out racket/base) (all-from-out qi) - (all-from-out "../../util.rkt") + (all-from-out "../util.rkt") sqr) (require qi - "../../util.rkt" + "../util.rkt" (only-in math sqr)) diff --git a/qi-sdk/profile/local/forms/benchmarks.rkt b/qi-sdk/profile/local/benchmarks.rkt similarity index 99% rename from qi-sdk/profile/local/forms/benchmarks.rkt rename to qi-sdk/profile/local/benchmarks.rkt index 0decd7036..84e8d0713 100755 --- a/qi-sdk/profile/local/forms/benchmarks.rkt +++ b/qi-sdk/profile/local/benchmarks.rkt @@ -900,7 +900,7 @@ for the forms are run. qi json csv-writing - (only-in "../../util.rkt" + (only-in "../util.rkt" only-if for/call)) (require diff --git a/qi-sdk/profile/local/report.rkt b/qi-sdk/profile/local/report.rkt index f7e229d8a..d2d47e8e0 100755 --- a/qi-sdk/profile/local/report.rkt +++ b/qi-sdk/profile/local/report.rkt @@ -10,10 +10,8 @@ for/call write-csv format-output) - - "loading/loadlib.rkt" "../regression.rkt" - (submod "forms/benchmarks.rkt" main)) + (submod "benchmarks.rkt" main)) (flag (selected #:param [selected null] name) ("-s" "--select" "Select form to benchmark") @@ -33,29 +31,15 @@ "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) -(flag (type #:param [report-type "all"] typ) - ("-t" - "--type" - "Type of report, either `forms`, `loading` or `all` (default `all`)") - (report-type typ)) - (flag (regression-file #:param [regression-file #f] reg-file) ("-r" "--regression" "'Before' data to compute regression against") (regression-file reg-file)) -;; Note: much of this file is duplicated across forms/report.rkt -;; and loading/report.rkt. It could be avoided if we had -;; "composition of commands", see: -;; https://github.com/countvajhula/cli/issues/3 (program (main) - (let* ([forms-data (if (member? (report-type) (list "all" "forms")) - (benchmark (selected)) - null)] - [require-data (if (member? (report-type) (list "all" "loading")) - (list (profile-load "qi")) - null)] - [output (append forms-data require-data)]) + (let ([output (benchmark (selected))]) (if (regression-file) + ;; TODO: regression ignores any flags and is a parallel path + ;; it should be properly incorporated into the CLI (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) (compute-regression before after)) diff --git a/qi-sdk/profile/local/forms/report.rkt b/qi-sdk/profile/report.rkt similarity index 59% rename from qi-sdk/profile/local/forms/report.rkt rename to qi-sdk/profile/report.rkt index 9c1198bf9..3b9e2c264 100755 --- a/qi-sdk/profile/local/forms/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -5,13 +5,14 @@ racket/format relation qi - (only-in "../../util.rkt" + (only-in "util.rkt" only-if for/call write-csv format-output) - "../../regression.rkt" - (submod "benchmarks.rkt" main)) + "loading/loadlib.rkt" + "regression.rkt" + (submod "local/benchmarks.rkt" main)) (flag (selected #:param [selected null] name) ("-s" "--select" "Select form to benchmark") @@ -31,15 +32,29 @@ "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") (output-format fmt)) +(flag (type #:param [report-type "all"] typ) + ("-t" + "--type" + "Type of report, either `forms`, `loading` or `all` (default `all`)") + (report-type typ)) + (flag (regression-file #:param [regression-file #f] reg-file) ("-r" "--regression" "'Before' data to compute regression against") (regression-file reg-file)) +;; Note: much of this file is duplicated across forms/report.rkt +;; and loading/report.rkt. It could be avoided if we had +;; "composition of commands", see: +;; https://github.com/countvajhula/cli/issues/3 (program (main) - (let ([output (benchmark (selected))]) + (let* ([forms-data (if (member? (report-type) (list "all" "forms")) + (benchmark (selected)) + null)] + [require-data (if (member? (report-type) (list "all" "loading")) + (list (profile-load "qi")) + null)] + [output (append forms-data require-data)]) (if (regression-file) - ;; TODO: regression ignores any flags and is a parallel path - ;; it should be properly incorporated into the CLI (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) (compute-regression before after)) From 7c34773a4f822b60719f1f05d77e284c83f25d00 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 00:53:23 -0700 Subject: [PATCH 19/29] run nonlocal benchmarks for racket or qi via CLI --- qi-sdk/profile/nonlocal/intrinsic.rkt | 4 ++-- qi-sdk/profile/nonlocal/report.rkt | 8 +++++++- qi-sdk/profile/util.rkt | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/qi-sdk/profile/nonlocal/intrinsic.rkt b/qi-sdk/profile/nonlocal/intrinsic.rkt index 1b75772ed..731e379a4 100755 --- a/qi-sdk/profile/nonlocal/intrinsic.rkt +++ b/qi-sdk/profile/nonlocal/intrinsic.rkt @@ -11,8 +11,8 @@ [benchmarks-to-run (if (null? benchmarks-to-run) (map bm-name specs) benchmarks-to-run)]) - (cond [(eq? 'qi language) (eval '(require "qi/main.rkt") namespace)] - [(eq? 'racket language) (eval '(require "racket/main.rkt") namespace)]) + (cond [(equal? "qi" language) (eval '(require "qi/main.rkt") namespace)] + [(equal? "racket" language) (eval '(require "racket/main.rkt") namespace)]) (for/list ([spec specs] #:when (member (bm-name spec) benchmarks-to-run)) diff --git a/qi-sdk/profile/nonlocal/report.rkt b/qi-sdk/profile/nonlocal/report.rkt index 61e549054..2b0b99b5d 100755 --- a/qi-sdk/profile/nonlocal/report.rkt +++ b/qi-sdk/profile/nonlocal/report.rkt @@ -34,10 +34,16 @@ ("-r" "--regression" "'Before' data to compute regression against") (regression-file reg-file)) +(flag (language #:param [language "qi"] lang) + ("-l" + "--language" + "Language to benchmark, either 'qi' or 'racket'. If none is specified, assumes 'qi'.") + (language lang)) + (program (main) (displayln "\nRunning competitive benchmarks..." (current-error-port)) - (let ([output (benchmark 'qi (selected))]) + (let ([output (benchmark (language) (selected))]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) diff --git a/qi-sdk/profile/util.rkt b/qi-sdk/profile/util.rkt index 0ba2951b5..38a560cfe 100644 --- a/qi-sdk/profile/util.rkt +++ b/qi-sdk/profile/util.rkt @@ -127,11 +127,11 @@ ;; Run different implementations of the same benchmark (e.g. a Racket vs a Qi ;; implementation) a specified number of times, and report the time taken ;; by each implementation. -(define (run-nonlocal-benchmark bm-name runner f n-times) - (displayln (~a bm-name ":") (current-error-port)) +(define (run-nonlocal-benchmark name runner f n-times) + (displayln (~a name ":") (current-error-port)) (let ([ms (measure runner f n-times)]) (displayln (~a ms " ms") (current-error-port)) - (hash 'name bm-name 'unit "ms" 'value ms))) + (hash 'name name 'unit "ms" 'value ms))) (define (write-csv data) (~> (data) From 71c638e7e0b8cc7d689d3413b60aea093d6bf169 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 01:26:34 -0700 Subject: [PATCH 20/29] use regression logic to implement competitive benchmarks --- Makefile | 6 ++- .../profile/nonlocal/report-competitive.rkt | 46 +++++++++++++++++++ .../{report.rkt => report-intrinsic.rkt} | 2 +- qi-sdk/profile/regression.rkt | 9 +++- 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100755 qi-sdk/profile/nonlocal/report-competitive.rkt rename qi-sdk/profile/nonlocal/{report.rkt => report-intrinsic.rkt} (96%) diff --git a/Makefile b/Makefile index 69777a9f3..683723e73 100644 --- a/Makefile +++ b/Makefile @@ -182,7 +182,11 @@ profile-selected-forms: profile-competitive: echo "Running competitive benchmarks..." - cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report.rkt + cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-competitive.rkt + +profile-nonlocal: + echo "Running nonlocal benchmarks..." + cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-intrinsic.rkt -l qi profile: profile-competitive profile-forms diff --git a/qi-sdk/profile/nonlocal/report-competitive.rkt b/qi-sdk/profile/nonlocal/report-competitive.rkt new file mode 100755 index 000000000..ed3a42dfd --- /dev/null +++ b/qi-sdk/profile/nonlocal/report-competitive.rkt @@ -0,0 +1,46 @@ +#!/usr/bin/env racket +#lang cli + +(require racket/match + racket/format + relation + qi + (only-in "../util.rkt" + only-if + for/call + write-csv + format-output) + "../regression.rkt" + "intrinsic.rkt") + +(flag (selected #:param [selected null] name) + ("-s" "--select" "Select benchmark by name") + (selected (cons name (selected)))) + +(constraint (multi selected)) + +(help + (usage + (~a "Run competitive benchmarks between Qi and Racket, " + "reporting the results in a configurable output format."))) + +(flag (output-format #:param [output-format ""] fmt) + ("-f" + "--format" + "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.") + (output-format fmt)) + +(program (main) + (displayln "\nRunning competitive benchmarks..." (current-error-port)) + + (let* ([racket-output (benchmark "racket" (selected))] + [qi-output (benchmark "qi" (selected))] + [before (parse-benchmarks racket-output)] + [after (parse-benchmarks qi-output)]) + (format-output (compute-regression before after) + (output-format)))) + +;; To run benchmarks for a form interactively, use e.g.: +;; (run main #("-s" "composition")) + +(run main) diff --git a/qi-sdk/profile/nonlocal/report.rkt b/qi-sdk/profile/nonlocal/report-intrinsic.rkt similarity index 96% rename from qi-sdk/profile/nonlocal/report.rkt rename to qi-sdk/profile/nonlocal/report-intrinsic.rkt index 2b0b99b5d..9849f9aa4 100755 --- a/qi-sdk/profile/nonlocal/report.rkt +++ b/qi-sdk/profile/nonlocal/report-intrinsic.rkt @@ -21,7 +21,7 @@ (help (usage - (~a "Run competitive benchmarks between Qi and Racket, " + (~a "Run nonlocal benchmarks on either Qi or Racket, " "reporting the results in a configurable output format."))) (flag (output-format #:param [output-format ""] fmt) diff --git a/qi-sdk/profile/regression.rkt b/qi-sdk/profile/regression.rkt index d27ddedd9..c3eba8146 100644 --- a/qi-sdk/profile/regression.rkt +++ b/qi-sdk/profile/regression.rkt @@ -42,6 +42,12 @@ 1 (~r #:precision 2)))) + (define-flow reformat + (~> △ + (>< (~> (-< car cadr) + (hash 'name _ 'value _ 'unit "x"))) + ▽)) + (define results (~>> (before) hash-keys @@ -52,6 +58,7 @@ calculate-ratio) ▽)) ▽ - (sort > #:key (☯ (~> cadr ->inexact))))) + (sort > #:key (☯ (~> cadr ->inexact))) + reformat)) results) From a3f230eb94d400b50ea8a75eab3ca6efc05e37c8 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 09:47:38 -0700 Subject: [PATCH 21/29] respect CLI flags in performance regression reporting --- qi-sdk/profile/loading/report.rkt | 3 ++- qi-sdk/profile/local/report.rkt | 5 ++--- qi-sdk/profile/nonlocal/report-intrinsic.rkt | 5 +++-- qi-sdk/profile/regression.rkt | 2 ++ qi-sdk/profile/report.rkt | 3 ++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/qi-sdk/profile/loading/report.rkt b/qi-sdk/profile/loading/report.rkt index 8c56eddad..1b1436447 100755 --- a/qi-sdk/profile/loading/report.rkt +++ b/qi-sdk/profile/loading/report.rkt @@ -32,7 +32,8 @@ (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) - (compute-regression before after)) + (format-output (compute-regression before after) + (output-format))) (format-output output (output-format))))) (run main) diff --git a/qi-sdk/profile/local/report.rkt b/qi-sdk/profile/local/report.rkt index d2d47e8e0..0a80a6cc0 100755 --- a/qi-sdk/profile/local/report.rkt +++ b/qi-sdk/profile/local/report.rkt @@ -38,11 +38,10 @@ (program (main) (let ([output (benchmark (selected))]) (if (regression-file) - ;; TODO: regression ignores any flags and is a parallel path - ;; it should be properly incorporated into the CLI (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) - (compute-regression before after)) + (format-output (compute-regression before after) + (output-format))) (format-output output (output-format))))) ;; To run benchmarks for a form interactively, use e.g.: diff --git a/qi-sdk/profile/nonlocal/report-intrinsic.rkt b/qi-sdk/profile/nonlocal/report-intrinsic.rkt index 9849f9aa4..5ee75633d 100755 --- a/qi-sdk/profile/nonlocal/report-intrinsic.rkt +++ b/qi-sdk/profile/nonlocal/report-intrinsic.rkt @@ -41,13 +41,14 @@ (language lang)) (program (main) - (displayln "\nRunning competitive benchmarks..." (current-error-port)) + (displayln "\nRunning nonlocal benchmarks..." (current-error-port)) (let ([output (benchmark (language) (selected))]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) - (compute-regression before after)) + (format-output (compute-regression before after) + (output-format))) (format-output output (output-format))))) ;; To run benchmarks for a form interactively, use e.g.: diff --git a/qi-sdk/profile/regression.rkt b/qi-sdk/profile/regression.rkt index c3eba8146..ad139e73c 100644 --- a/qi-sdk/profile/regression.rkt +++ b/qi-sdk/profile/regression.rkt @@ -19,6 +19,8 @@ (read-json port)))) (define (parse-benchmarks benchmarks) + ;; renames some forms so they're consistently named + ;; but otherwise leaves the original data unmodified (make-hash (map (☯ (~> (-< (~> (hash-ref 'name) (switch diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index 3b9e2c264..1a51770cc 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -57,7 +57,8 @@ (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) - (compute-regression before after)) + (format-output (compute-regression before after) + (output-format))) (format-output output (output-format))))) ;; To run benchmarks for a form interactively, use e.g.: From a796dac4204c9adeacb341b8b3b71294f820c4ef Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 09:49:22 -0700 Subject: [PATCH 22/29] check regression wrt the "after" data to respect narrowed selection --- qi-sdk/profile/regression.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qi-sdk/profile/regression.rkt b/qi-sdk/profile/regression.rkt index ad139e73c..93c14ed45 100644 --- a/qi-sdk/profile/regression.rkt +++ b/qi-sdk/profile/regression.rkt @@ -51,7 +51,7 @@ ▽)) (define results - (~>> (before) + (~>> (after) hash-keys △ (>< From e0556097b80285029effe3ce49839672f7f50b6a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 10:13:56 -0700 Subject: [PATCH 23/29] update makefile targets and name things consistently --- Makefile | 15 ++++++--------- qi-sdk/profile/loading/report.rkt | 2 ++ qi-sdk/profile/local/report.rkt | 2 ++ qi-sdk/profile/report.rkt | 5 ++++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 683723e73..0d0b17a61 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,8 @@ help: @echo "docs - view docs in a browser" @echo "profile - Run comprehensive performance benchmarks" @echo "profile-competitive - Run competitive benchmarks" - @echo "profile-forms - Run benchmarks for individual Qi forms" + @echo "profile-local - Run benchmarks for individual Qi forms" + @echo "profile-nonlocal - Run nonlocal benchmarks exercising many components at once" @echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)" @echo "performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression" @echo " For use in regression: make performance-report > /path/to/before.json" @@ -169,26 +170,22 @@ cover: coverage-check coverage-report cover-coveralls: raco cover -b -f coveralls -p $(PACKAGE-NAME)-{lib,test} -profile-forms: - echo "Profiling forms..." +profile-local: racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt profile-loading: - echo "Profiling module loading..." racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt profile-selected-forms: - @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt' directly, with -f form-name for each form." + @echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt' directly, with -s form-name for each form." profile-competitive: - echo "Running competitive benchmarks..." cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-competitive.rkt profile-nonlocal: - echo "Running nonlocal benchmarks..." cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-intrinsic.rkt -l qi -profile: profile-competitive profile-forms +profile: profile-local profile-nonlocal profile-loading performance-report: @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -f json @@ -196,4 +193,4 @@ performance-report: performance-regression-report: @racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF) -.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-loading profile-selected-forms profile-competitive profile performance-report performance-regression-report +.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-local profile-loading profile-selected-forms profile-competitive profile-nonlocal profile performance-report performance-regression-report diff --git a/qi-sdk/profile/loading/report.rkt b/qi-sdk/profile/loading/report.rkt index 1b1436447..e91d64de6 100755 --- a/qi-sdk/profile/loading/report.rkt +++ b/qi-sdk/profile/loading/report.rkt @@ -28,6 +28,8 @@ (regression-file reg-file)) (program (main) + (displayln "\nMeasuring module load time..." (current-error-port)) + (let ([output (profile-load "qi")]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] diff --git a/qi-sdk/profile/local/report.rkt b/qi-sdk/profile/local/report.rkt index 0a80a6cc0..85201dbd1 100755 --- a/qi-sdk/profile/local/report.rkt +++ b/qi-sdk/profile/local/report.rkt @@ -36,6 +36,8 @@ (regression-file reg-file)) (program (main) + (displayln "\nRunning local (forms) benchmarks..." (current-error-port)) + (let ([output (benchmark (selected))]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index 1a51770cc..fb3fa1cd7 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -42,11 +42,14 @@ ("-r" "--regression" "'Before' data to compute regression against") (regression-file reg-file)) -;; Note: much of this file is duplicated across forms/report.rkt +;; Note: much of this file is duplicated across local/report.rkt ;; and loading/report.rkt. It could be avoided if we had ;; "composition of commands", see: ;; https://github.com/countvajhula/cli/issues/3 (program (main) + (displayln "\nRunning local (forms) benchmarks and measuring module load time..." + (current-error-port)) + (let* ([forms-data (if (member? (report-type) (list "all" "forms")) (benchmark (selected)) null)] From ab28a32c0914635e8b5389e34270beebfa5b8438 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 10:30:50 -0700 Subject: [PATCH 24/29] improve live output in competitive report --- qi-sdk/profile/nonlocal/report-competitive.rkt | 8 ++++++-- qi-sdk/profile/regression.rkt | 8 +++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/qi-sdk/profile/nonlocal/report-competitive.rkt b/qi-sdk/profile/nonlocal/report-competitive.rkt index ed3a42dfd..458f99f35 100755 --- a/qi-sdk/profile/nonlocal/report-competitive.rkt +++ b/qi-sdk/profile/nonlocal/report-competitive.rkt @@ -33,8 +33,12 @@ (program (main) (displayln "\nRunning competitive benchmarks..." (current-error-port)) - (let* ([racket-output (benchmark "racket" (selected))] - [qi-output (benchmark "qi" (selected))] + (let* ([racket-output + (begin (displayln "\nRunning Racket benchmarks..." (current-error-port)) + (benchmark "racket" (selected)))] + [qi-output + (begin (displayln "\nRunning Qi benchmarks..." (current-error-port)) + (benchmark "qi" (selected)))] [before (parse-benchmarks racket-output)] [after (parse-benchmarks qi-output)]) (format-output (compute-regression before after) diff --git a/qi-sdk/profile/regression.rkt b/qi-sdk/profile/regression.rkt index 93c14ed45..0e1e072b3 100644 --- a/qi-sdk/profile/regression.rkt +++ b/qi-sdk/profile/regression.rkt @@ -8,7 +8,8 @@ (require qi relation json - racket/format) + racket/format + racket/pretty) (define LOWER-THRESHOLD 0.75) (define HIGHER-THRESHOLD 1.5) @@ -50,6 +51,10 @@ (hash 'name _ 'value _ 'unit "x"))) ▽)) + (define (show-results results) + (displayln "\nPerformance relative to baseline:" (current-error-port)) + (pretty-display results (current-error-port))) + (define results (~>> (after) hash-keys @@ -61,6 +66,7 @@ ▽)) ▽ (sort > #:key (☯ (~> cadr ->inexact))) + (ε show-results) reformat)) results) From 00337422da9c135ddbbb8833c85d8cfcf8c6646a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 10:43:16 -0700 Subject: [PATCH 25/29] cleanup, remove unused imports --- qi-sdk/profile/local/benchmarks.rkt | 2 -- qi-sdk/profile/local/report.rkt | 8 +------- qi-sdk/profile/nonlocal/intrinsic.rkt | 2 +- qi-sdk/profile/nonlocal/report-competitive.rkt | 8 +------- qi-sdk/profile/nonlocal/report-intrinsic.rkt | 8 +------- qi-sdk/profile/report.rkt | 8 +------- qi-sdk/profile/util.rkt | 6 ++---- 7 files changed, 7 insertions(+), 35 deletions(-) diff --git a/qi-sdk/profile/local/benchmarks.rkt b/qi-sdk/profile/local/benchmarks.rkt index 84e8d0713..75c3d73be 100755 --- a/qi-sdk/profile/local/benchmarks.rkt +++ b/qi-sdk/profile/local/benchmarks.rkt @@ -898,8 +898,6 @@ for the forms are run. racket/format relation qi - json - csv-writing (only-in "../util.rkt" only-if for/call)) diff --git a/qi-sdk/profile/local/report.rkt b/qi-sdk/profile/local/report.rkt index 85201dbd1..2ff1e96ea 100755 --- a/qi-sdk/profile/local/report.rkt +++ b/qi-sdk/profile/local/report.rkt @@ -1,14 +1,8 @@ #!/usr/bin/env racket #lang cli -(require racket/match - racket/format - relation - qi +(require racket/format (only-in "../util.rkt" - only-if - for/call - write-csv format-output) "../regression.rkt" (submod "benchmarks.rkt" main)) diff --git a/qi-sdk/profile/nonlocal/intrinsic.rkt b/qi-sdk/profile/nonlocal/intrinsic.rkt index 731e379a4..e632d7361 100755 --- a/qi-sdk/profile/nonlocal/intrinsic.rkt +++ b/qi-sdk/profile/nonlocal/intrinsic.rkt @@ -1,5 +1,5 @@ #!/usr/bin/env racket -#lang cli +#lang racket/base (provide benchmark) diff --git a/qi-sdk/profile/nonlocal/report-competitive.rkt b/qi-sdk/profile/nonlocal/report-competitive.rkt index 458f99f35..7e03033ff 100755 --- a/qi-sdk/profile/nonlocal/report-competitive.rkt +++ b/qi-sdk/profile/nonlocal/report-competitive.rkt @@ -1,14 +1,8 @@ #!/usr/bin/env racket #lang cli -(require racket/match - racket/format - relation - qi +(require racket/format (only-in "../util.rkt" - only-if - for/call - write-csv format-output) "../regression.rkt" "intrinsic.rkt") diff --git a/qi-sdk/profile/nonlocal/report-intrinsic.rkt b/qi-sdk/profile/nonlocal/report-intrinsic.rkt index 5ee75633d..c451cd71d 100755 --- a/qi-sdk/profile/nonlocal/report-intrinsic.rkt +++ b/qi-sdk/profile/nonlocal/report-intrinsic.rkt @@ -1,14 +1,8 @@ #!/usr/bin/env racket #lang cli -(require racket/match - racket/format - relation - qi +(require racket/format (only-in "../util.rkt" - only-if - for/call - write-csv format-output) "../regression.rkt" "intrinsic.rkt") diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index fb3fa1cd7..d9de9b1d0 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -1,14 +1,8 @@ #!/usr/bin/env racket #lang cli -(require racket/match - racket/format - relation - qi +(require racket/format (only-in "util.rkt" - only-if - for/call - write-csv format-output) "loading/loadlib.rkt" "regression.rkt" diff --git a/qi-sdk/profile/util.rkt b/qi-sdk/profile/util.rkt index 38a560cfe..a751e2123 100644 --- a/qi-sdk/profile/util.rkt +++ b/qi-sdk/profile/util.rkt @@ -22,10 +22,6 @@ curryr) (only-in adjutor values->list) - (only-in data/collection - cycle - take - in) csv-writing json racket/format @@ -94,6 +90,8 @@ ;; Run a single benchmarking function a specified number of times ;; and report the time taken. +;; TODO: this is very similar to run-nonlocal-benchmark and these +;; should be unified. (define-syntax-parse-rule (run-benchmark f-name runner n-times) #:with name (datum->syntax #'f-name ;; this is because of the name collision between From 73b5b6b80493ccc26c8bc0dc945a3e053dad4c9c Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 12:28:11 -0700 Subject: [PATCH 26/29] use "local" instead of "forms" --- qi-sdk/profile/report.rkt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index d9de9b1d0..d1482e5f0 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -29,7 +29,7 @@ (flag (type #:param [report-type "all"] typ) ("-t" "--type" - "Type of report, either `forms`, `loading` or `all` (default `all`)") + "Type of report, either `local`, `loading` or `all` (default `all`)") (report-type typ)) (flag (regression-file #:param [regression-file #f] reg-file) @@ -44,13 +44,13 @@ (displayln "\nRunning local (forms) benchmarks and measuring module load time..." (current-error-port)) - (let* ([forms-data (if (member? (report-type) (list "all" "forms")) + (let* ([local-data (if (member? (report-type) (list "all" "local")) (benchmark (selected)) null)] [require-data (if (member? (report-type) (list "all" "loading")) (list (profile-load "qi")) null)] - [output (append forms-data require-data)]) + [output (append local-data require-data)]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) From 916b7b9206f5e7b9c65b3155cfa01205d2f803b1 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 15 Mar 2023 12:48:57 -0700 Subject: [PATCH 27/29] add back needed import --- qi-sdk/profile/report.rkt | 1 + 1 file changed, 1 insertion(+) diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index d1482e5f0..554b9cd27 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -2,6 +2,7 @@ #lang cli (require racket/format + relation (only-in "util.rkt" format-output) "loading/loadlib.rkt" From f8f3e8a383aed8fdb34e09125469cec5c9b34691 Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Tue, 21 Mar 2023 20:15:44 -0700 Subject: [PATCH 28/29] add nonlocal benchmarks to the performance report --- qi-sdk/profile/nonlocal/intrinsic.rkt | 35 ++++++++++++++++++++++++--- qi-sdk/profile/report.rkt | 10 +++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/qi-sdk/profile/nonlocal/intrinsic.rkt b/qi-sdk/profile/nonlocal/intrinsic.rkt index e632d7361..f0afbd1ae 100755 --- a/qi-sdk/profile/nonlocal/intrinsic.rkt +++ b/qi-sdk/profile/nonlocal/intrinsic.rkt @@ -3,16 +3,45 @@ (provide benchmark) -(require "../util.rkt" +(require racket/runtime-path + "../util.rkt" "spec.rkt") +;; We use `eval` in this module to `require` the appropriate objective +;; functions (either Racket or Qi) for benchmarking in a dynamically +;; constructed namespace (following +;; https://docs.racket-lang.org/guide/eval.html). This allows us to +;; define those functions symmetrically in the Racket and Qi modules, and +;; invoke them in a common way here. But as this eval namespace is +;; dynamically constructed, the require paths are interpreted as being +;; relative to the path from which this module is executed (e.g. either +;; locally from this folder or from the qi root via the Makefile) and may +;; therefore fail to find the modules if executed from "the wrong" +;; location. To avoid this, we set the "load relative" directory to the +;; module's path, so that requiring modules is always relative to the +;; present module path, allowing it to behave the same no matter where it +;; is executed from. Another possibility is to simply assume that the +;; qi-sdk package is installed so that the modules are available via +;; collection paths, but currently, having the SDK "officially" installed +;; slows down building of other packages for reasons as yet unknown. See: +;; https://github.com/drym-org/qi/wiki/Installing-the-SDK#install-the-sdk +;; So for now, we use this fix so that we can have the SDK remain +;; uninstalled. + +(define-runtime-path lexical-module-path ".") +(current-load-relative-directory lexical-module-path) + (define (benchmark language benchmarks-to-run) (let ([namespace (make-base-namespace)] [benchmarks-to-run (if (null? benchmarks-to-run) (map bm-name specs) benchmarks-to-run)]) - (cond [(equal? "qi" language) (eval '(require "qi/main.rkt") namespace)] - [(equal? "racket" language) (eval '(require "racket/main.rkt") namespace)]) + (cond [(equal? "qi" language) + (eval '(require "qi/main.rkt") + namespace)] + [(equal? "racket" language) + (eval '(require "racket/main.rkt") + namespace)]) (for/list ([spec specs] #:when (member (bm-name spec) benchmarks-to-run)) diff --git a/qi-sdk/profile/report.rkt b/qi-sdk/profile/report.rkt index 554b9cd27..1208491ac 100755 --- a/qi-sdk/profile/report.rkt +++ b/qi-sdk/profile/report.rkt @@ -7,7 +7,8 @@ format-output) "loading/loadlib.rkt" "regression.rkt" - (submod "local/benchmarks.rkt" main)) + (submod "local/benchmarks.rkt" main) + (prefix-in n: "nonlocal/intrinsic.rkt")) (flag (selected #:param [selected null] name) ("-s" "--select" "Select form to benchmark") @@ -30,7 +31,7 @@ (flag (type #:param [report-type "all"] typ) ("-t" "--type" - "Type of report, either `local`, `loading` or `all` (default `all`)") + "Type of report, either `local`, `nonlocal`, `loading` or `all` (default `all`)") (report-type typ)) (flag (regression-file #:param [regression-file #f] reg-file) @@ -48,10 +49,13 @@ (let* ([local-data (if (member? (report-type) (list "all" "local")) (benchmark (selected)) null)] + [nonlocal-data (if (member? (report-type) (list "all" "nonlocal")) + (n:benchmark "qi" (selected)) + null)] [require-data (if (member? (report-type) (list "all" "loading")) (list (profile-load "qi")) null)] - [output (append local-data require-data)]) + [output (~ local-data nonlocal-data require-data)]) (if (regression-file) (let ([before (parse-benchmarks (parse-json-file (regression-file)))] [after (parse-benchmarks output)]) From e2bcf0641f89f09cd1a2ce7183d294ecf711f39a Mon Sep 17 00:00:00 2001 From: Siddhartha Date: Wed, 22 Mar 2023 17:29:34 -0700 Subject: [PATCH 29/29] contain load path parameter to eval where it's needed --- qi-sdk/profile/nonlocal/intrinsic.rkt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/qi-sdk/profile/nonlocal/intrinsic.rkt b/qi-sdk/profile/nonlocal/intrinsic.rkt index f0afbd1ae..6607f1844 100755 --- a/qi-sdk/profile/nonlocal/intrinsic.rkt +++ b/qi-sdk/profile/nonlocal/intrinsic.rkt @@ -29,7 +29,6 @@ ;; uninstalled. (define-runtime-path lexical-module-path ".") -(current-load-relative-directory lexical-module-path) (define (benchmark language benchmarks-to-run) (let ([namespace (make-base-namespace)] @@ -37,11 +36,13 @@ (map bm-name specs) benchmarks-to-run)]) (cond [(equal? "qi" language) - (eval '(require "qi/main.rkt") - namespace)] + (parameterize ([current-load-relative-directory lexical-module-path]) + (eval '(require "qi/main.rkt") + namespace))] [(equal? "racket" language) - (eval '(require "racket/main.rkt") - namespace)]) + (parameterize ([current-load-relative-directory lexical-module-path]) + (eval '(require "racket/main.rkt") + namespace))]) (for/list ([spec specs] #:when (member (bm-name spec) benchmarks-to-run))