From db5d58e84252042236b8de80fe0e1b722a83e61a Mon Sep 17 00:00:00 2001
From: isaacs <i@izs.me>
Date: Wed, 6 Mar 2013 12:32:59 -0800
Subject: [PATCH] bench: Use environ to run compares more than once

This will run the benchmarks the number of times specified by NODE_BENCH_RUNS,
to attempt to reduce variability.

If the number of runs is high enough, it'll also throw out the top and bottom
quartiles, since that's where the outliers will be.

It's not very fancy statistics-fu, but it's better than nothing.

Also, linted this file.  It had tabs in it.  TABS!
---
 benchmark/compare.js | 101 +++++++++++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 38 deletions(-)

diff --git a/benchmark/compare.js b/benchmark/compare.js
index b3576ba4acc637..d2221eeb78d808 100644
--- a/benchmark/compare.js
+++ b/benchmark/compare.js
@@ -1,4 +1,6 @@
-var usage = 'node benchmark/compare.js <node-binary1> <node-binary2> [--html] [--red|-r] [--green|-g]';
+var usage = 'node benchmark/compare.js ' +
+            '<node-binary1> <node-binary2> ' +
+            '[--html] [--red|-r] [--green|-g]';
 
 var show = 'both';
 var nodes = [];
@@ -42,73 +44,85 @@ if (!html) {
 var runBench = process.env.NODE_BENCH || 'bench';
 
 if (nodes.length !== 2)
-	return console.error('usage:\n  %s', usage);
+  return console.error('usage:\n  %s', usage);
 
 var spawn = require('child_process').spawn;
 var results = {};
-var n = 2;
+var n = 1;
 
 run();
 
+var RUNS = +process.env.NODE_BENCH_RUNS || 1;
+var r = RUNS;
 function run() {
-  if (n === 0)
-		return compare();
+  // Flip back and forth between the two binaries.
+  if (n === 1) {
+    n--;
+  } else {
+    r--;
+    if (r === 0)
+      return compare();
+    else
+      n++;
+  }
 
-  n--;
+  if (n === -1)
+    return compare();
 
-	var node = nodes[n];
+  var node = nodes[n];
   console.error('running %s', node);
-	var env = {};
+  var env = {};
   for (var i in process.env)
     env[i] = process.env[i];
   env.NODE = node;
-	var child = spawn('make', [ runBench ], { env: env });
+  var child = spawn('make', [runBench], { env: env });
 
-	var out = '';
-	child.stdout.setEncoding('utf8');
-	child.stdout.on('data', function(c) {
-		out += c;
-	});
+  var out = '';
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', function(c) {
+    out += c;
+  });
 
   child.stderr.pipe(process.stderr);
 
-	child.on('close', function(code) {
-		if (code) {
-			console.error('%s exited with code=%d', node, code);
-			process.exit(code);
-		} else {
-			out.trim().split(/\r?\n/).forEach(function(line) {
+  child.on('close', function(code) {
+    if (code) {
+      console.error('%s exited with code=%d', node, code);
+      process.exit(code);
+    } else {
+      out.trim().split(/\r?\n/).forEach(function(line) {
         line = line.trim();
         if (!line)
           return;
 
-				var s = line.split(':');
-				var num = +s.pop();
-				if (!num && num !== 0)
-          return
+        var s = line.split(':');
+        var num = +s.pop();
+        if (!num && num !== 0)
+          return;
 
-				line = s.join(':');
-				var res = results[line] = results[line] || {};
-				res[node] = num;
-			});
+        line = s.join(':');
+        var res = results[line] = results[line] || {};
+        res[node] = res[node] || [];
+        res[node].push(num);
+      });
 
-			run();
-		}
-	});
+      run();
+    }
+  });
 }
 
 function compare() {
-	// each result is an object with {"foo.js arg=bar":12345,...}
-	// compare each thing, and show which node did the best.
+  // each result is an object with {"foo.js arg=bar":12345,...}
+  // compare each thing, and show which node did the best.
   // node[0] is shown in green, node[1] shown in red.
   var maxLen = -Infinity;
   var util = require('util');
   console.log(start);
 
-	Object.keys(results).map(function(bench) {
+  Object.keys(results).map(function(bench) {
     var res = results[bench];
-    var n0 = res[nodes[0]];
-    var n1 = res[nodes[1]];
+    var n0 = avg(res[nodes[0]]);
+    var n1 = avg(res[nodes[1]]);
 
     var pct = ((n0 - n1) / n1 * 100).toFixed(2);
 
@@ -119,8 +133,8 @@ function compare() {
     if (show === 'green' && !g || show === 'red' && !r)
       return;
 
-    var r0 = util.format('%s%s: %d%s', g, nodes[0], n0, reset);
-    var r1 = util.format('%s%s: %d%s', r, nodes[1], n1, reset);
+    var r0 = util.format('%s%s: %d%s', g, nodes[0], n0, g ? reset : '');
+    var r1 = util.format('%s%s: %d%s', r, nodes[1], n1, r ? reset : '');
     var pct = c + pct + '%' + reset;
     var l = util.format('%s: %s %s', bench, r0, r1);
     maxLen = Math.max(l.length + pct.length, maxLen);
@@ -136,3 +150,14 @@ function compare() {
   });
   console.log(end);
 }
+
+function avg(list) {
+  if (list.length >= 3) {
+    list = list.sort();
+    var q = Math.floor(list.length / 4) || 1;
+    list = list.slice(q, -q);
+  }
+  return (list.reduce(function(a, b) {
+    return a + b;
+  }, 0) / list.length).toPrecision(5);
+}