Merge pull request #95 from countvajhula/nonlocal-benchmarks-starter

Organize performance benchmarks

.github/workflows/benchmarks.yml

@@ -25,7 +25,7 @@ jobs:
         run: make install-sdk
       - name: Run benchmark
         shell: 'bash --noprofile --norc -eo pipefail {0}'
-        run: make form-performance-report | tee benchmarks.txt
+        run: make performance-report | tee benchmarks.txt
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:

Makefile

@@ -7,12 +7,15 @@ DEPS-FLAGS=--check-pkg-deps --unused-pkg-deps
 
 help:
 	@echo "install - install package along with dependencies"
+	@echo "install-sdk - install the SDK which includes developer tools"
 	@echo "remove - remove package"
+	@echo "remove-sdk - remove SDK; this will not remove SDK dependencies"
 	@echo "build - Compile libraries"
 	@echo "build-docs - Build docs"
 	@echo "build-standalone-docs - Build self-contained docs that could be hosted somewhere"
 	@echo "build-all - Compile libraries, build docs, and check dependencies"
 	@echo "clean - remove all build artifacts"
+	@echo "clean-sdk - remove all build artifacts in SDK paths"
 	@echo "check-deps - check dependencies"
 	@echo "test - run tests"
 	@echo "test-with-errortrace - run tests with error tracing"
@@ -37,9 +40,14 @@ help:
 	@echo "docs - view docs in a browser"
 	@echo "profile - Run comprehensive performance benchmarks"
 	@echo "profile-competitive - Run competitive benchmarks"
-	@echo "profile-forms - Run benchmarks for individual Qi forms"
+	@echo "profile-local - Run benchmarks for individual Qi forms"
+	@echo "profile-nonlocal - Run nonlocal benchmarks exercising many components at once"
 	@echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)"
-	@echo "form-performance-report - Run benchmarks for Qi forms and produce results for use in CI"
+	@echo "performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression"
+	@echo "  For use in regression: make performance-report > /path/to/before.json"
+	@echo "performance-regression-report - Run benchmarks for Qi forms against a reference report."
+	@echo "  make performance-regression-report REF=/path/to/before.json"
+
 
 # Primarily for use by CI.
 # Installs dependencies as well as linking this as a package.
@@ -162,20 +170,27 @@ cover: coverage-check coverage-report
 cover-coveralls:
 	raco cover -b -f coveralls -p $(PACKAGE-NAME)-{lib,test}
 
-profile-forms:
-	echo "Profiling forms..."
-	racket $(PACKAGE-NAME)-sdk/profile/forms.rkt
+profile-local:
+	racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt
+
+profile-loading:
+	racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt
 
 profile-selected-forms:
-	@echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/forms.rkt' directly, with -f form-name for each form."
+	@echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt' directly, with -s form-name for each form."
 
 profile-competitive:
-	echo "Running competitive benchmarks..."
-	racket $(PACKAGE-NAME)-sdk/profile/competitive.rkt
+	cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-competitive.rkt
+
+profile-nonlocal:
+	cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-intrinsic.rkt -l qi
+
+profile: profile-local profile-nonlocal profile-loading
 
-profile: profile-competitive profile-forms
+performance-report:
+	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt -f json
 
-form-performance-report:
-	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt
+performance-regression-report:
+	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF)
 
-.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile form-performance-report
+.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-local profile-loading profile-selected-forms profile-competitive profile-nonlocal profile performance-report performance-regression-report

qi-doc/scribblings/field-guide.scrbl

@@ -324,7 +324,7 @@ Another way to do it is to simply promote the expression out of the nest:
     (~> (3) (get-f 1))
   ]
 
-@;{Update this to reflect new partial application behavior}
+@;{TODO: Update this to reflect new partial application behavior}
 Now, you might, once again, expect this to be treated as a partial application template, so that this would be equivalent to @racket[(get-f 3 1)] and would raise an error. But in fact, since the expression @racket[(get-f 1)] happens to be fully qualified with all the arguments it needs, the currying employed under the hood to implement partial application in this case @seclink["Using_Racket_to_Define_Flows"]{evaluates to a function result right away}. This then receives the value @racket[3], and consequently, this expression produces the correct result.
 
 So in sum, it's perhaps best to rely on @racket[esc] in such cases to be as explicit as possible about what you mean, rather than rely on quirks of the implementation that are revealed at this boundary between two languages.

qi-sdk/info.rkt

@@ -10,6 +10,7 @@
                "collections-lib"
                "relation-lib"
                "csv-writing"
+               "require-latency"
                "cover"
                "cover-coveralls"))
 (define build-deps '())

qi-sdk/profile/loading/loadlib.rkt

@@ -0,0 +1,16 @@
+#!/usr/bin/env racket
+#lang racket/base
+
+(provide profile-load)
+
+(require pkg/require-latency
+         racket/format)
+
+(define (profile-load module-name)
+  (let ([name (~a "(require " module-name ")")]
+        [ms (cdr (time-module-ms module-name))])
+    (displayln (~a name ": " ms " ms")
+               (current-error-port))
+    (hash 'name name
+          'unit "ms"
+          'value ms)))

qi-sdk/profile/loading/report.rkt

@@ -0,0 +1,41 @@
+#!/usr/bin/env racket
+#lang cli
+
+(require racket/match
+         racket/format
+         relation
+         qi
+         (only-in "../util.rkt"
+                  only-if
+                  for/call
+                  write-csv
+                  format-output)
+         "../regression.rkt"
+         "loadlib.rkt")
+
+(help
+ (usage
+  (~a "Measure module load time, i.e. the time taken by (require qi).")))
+
+(flag (output-format #:param [output-format ""] fmt)
+  ("-f"
+   "--format"
+   "Output format to use, either 'json' or 'csv'. If none is specified, no output is generated.")
+  (output-format fmt))
+
+(flag (regression-file #:param [regression-file #f] reg-file)
+  ("-r" "--regression" "'Before' data to compute regression against")
+  (regression-file reg-file))
+
+(program (main)
+  (displayln "\nMeasuring module load time..." (current-error-port))
+
+  (let ([output (profile-load "qi")])
+    (if (regression-file)
+        (let ([before (parse-benchmarks (parse-json-file (regression-file)))]
+              [after (parse-benchmarks output)])
+          (format-output (compute-regression before after)
+                         (output-format)))
+        (format-output output (output-format)))))
+
+(run main)

qi-sdk/profile/forms-base.rkt → qi-sdk/profile/local/base.rkt

@@ -2,11 +2,9 @@
 
 (provide (all-from-out racket/base)
          (all-from-out qi)
-         (all-from-out "util.rkt")
+         (all-from-out "../util.rkt")
          sqr)
 
 (require qi
-         "util.rkt"
+         "../util.rkt"
          (only-in math sqr))
-
-