[benchmark] Driver Improvements

benchmark/scripts/compare_perf_tests.py

@@ -229,7 +229,8 @@ class PerformanceTestResult(object):
     `--quantile`parameter. In both cases, the last column, MAX_RSS is optional.
     """
 
-    def __init__(self, csv_row, quantiles=False, memory=False, delta=False):
+    def __init__(self, csv_row, quantiles=False, memory=False, delta=False,
+                 meta=False):
         """Initialize from a row of multiple columns with benchmark summary.
 
         The row is an iterable, such as a row provided by the CSV parser.
@@ -239,7 +240,8 @@ def __init__(self, csv_row, quantiles=False, memory=False, delta=False):
         self.num_samples = int(csv_row[2])  # Number of measurements taken
 
         if quantiles:  # Variable number of columns representing quantiles
-            runtimes = csv_row[3:-1] if memory else csv_row[3:]
+            mem_index = (-1 if memory else 0) + (-3 if meta else 0)
+            runtimes = csv_row[3:mem_index] if memory or meta else csv_row[3:]
             if delta:
                 runtimes = [int(x) if x else 0 for x in runtimes]
                 runtimes = reduce(lambda l, x: l.append(l[-1] + x) or  # runnin
@@ -261,7 +263,7 @@ def __init__(self, csv_row, quantiles=False, memory=False, delta=False):
             self.min, self.max, self.median, self.mean, self.sd = \
                 sams.min, sams.max, sams.median, sams.mean, sams.sd
             self.max_rss = (                # Maximum Resident Set Size (B)
-                int(csv_row[-1]) if memory else None)
+                int(csv_row[mem_index]) if memory else None)
         else:  # Legacy format with statistics for normal distribution.
             self.min = int(csv_row[3])      # Minimum runtime (μs)
             self.max = int(csv_row[4])      # Maximum runtime (μs)
@@ -271,6 +273,11 @@ def __init__(self, csv_row, quantiles=False, memory=False, delta=False):
             self.max_rss = (                # Maximum Resident Set Size (B)
                 int(csv_row[8]) if len(csv_row) > 8 else None)
             self.samples = None
+
+        # Optional measurement metadata. The number of:
+        # memory pages used, involuntary context switches and voluntary yields
+        self.mem_pages, self.involuntary_cs, self.yield_count = \
+            [int(x) for x in csv_row[-3:]] if meta else (None, None, None)
         self.yields = None
         self.setup = None
 
@@ -352,6 +359,7 @@ def __init__(self):
         """Create instance of `LogParser`."""
         self.results = []
         self.quantiles, self.delta, self.memory = False, False, False
+        self.meta = False
         self._reset()
 
     def _reset(self):
@@ -371,12 +379,12 @@ def _append_result(self, result):
         columns = result.split(',') if ',' in result else result.split()
         r = PerformanceTestResult(
             columns, quantiles=self.quantiles, memory=self.memory,
-            delta=self.delta)
+            delta=self.delta, meta=self.meta)
         r.setup = self.setup
         r.max_rss = r.max_rss or self.max_rss
-        r.mem_pages = self.mem_pages
+        r.mem_pages = r.mem_pages or self.mem_pages
         r.voluntary_cs = self.voluntary_cs
-        r.involuntary_cs = self.involuntary_cs
+        r.involuntary_cs = r.involuntary_cs or self.involuntary_cs
         if self.samples:
             r.samples = PerformanceTestSamples(r.name, self.samples)
             r.samples.exclude_outliers()
@@ -391,6 +399,7 @@ def _store_memory_stats(self, max_rss, mem_pages):
     def _configure_format(self, header):
         self.quantiles = 'MEAN' not in header
         self.memory = 'MAX_RSS' in header
+        self.meta = 'PAGES' in header
         self.delta = '𝚫' in header
 
     # Regular expression and action to take when it matches the parsed line

benchmark/scripts/test_compare_perf_tests.py

@@ -308,6 +308,46 @@ def validatePTR(deq):  # construct from delta encoded quantiles string
 1,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1"""
         map(validatePTR, delta_encoded_quantiles.split('\n')[1:])
 
+    def test_init_meta(self):
+        # #,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),…
+        # …PAGES,ICS,YIELD
+        log = '1,Ackermann,200,715,1281,726,47,715,7,29,15'
+        r = PerformanceTestResult(log.split(','), meta=True)
+        self.assertEqual((r.test_num, r.name), ('1', 'Ackermann'))
+        self.assertEqual(
+            (r.num_samples, r.min, r.max, r.mean, r.sd, r.median),
+            (200, 715, 1281, 726, 47, 715))
+        self.assertEqual((r.mem_pages, r.involuntary_cs, r.yield_count),
+                         (7, 29, 15))
+        # #,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),MAX_RSS(B),…
+        # …PAGES,ICS,YIELD
+        log = '1,Ackermann,200,715,1951,734,97,715,36864,9,50,15'
+        r = PerformanceTestResult(log.split(','), memory=True, meta=True)
+        self.assertEqual(
+            (r.num_samples, r.min, r.max, r.mean, r.sd, r.median),
+            (200, 715, 1951, 734, 97, 715))
+        self.assertEqual(
+            (r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss),
+            (9, 50, 15, 36864))
+        # #,TEST,SAMPLES,MIN(μs),MAX(μs),PAGES,ICS,YIELD
+        log = '1,Ackermann,200,715,3548,8,31,15'
+        r = PerformanceTestResult(log.split(','), quantiles=True, meta=True)
+        self.assertEqual((r.num_samples, r.min, r.max), (200, 715, 3548))
+        self.assertEqual((r.samples.count, r.samples.min, r.samples.max),
+                         (2, 715, 3548))
+        self.assertEqual((r.mem_pages, r.involuntary_cs, r.yield_count),
+                         (8, 31, 15))
+        # #,TEST,SAMPLES,MIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD
+        log = '1,Ackermann,200,715,1259,32768,8,28,15'
+        r = PerformanceTestResult(
+            log.split(','), quantiles=True, memory=True, meta=True)
+        self.assertEqual((r.num_samples, r.min, r.max), (200, 715, 1259))
+        self.assertEqual((r.samples.count, r.samples.min, r.samples.max),
+                         (2, 715, 1259))
+        self.assertEquals(r.max_rss, 32768)
+        self.assertEqual((r.mem_pages, r.involuntary_cs, r.yield_count),
+                         (8, 28, 15))
+
     def test_repr(self):
         log_line = '1,AngryPhonebook,20,10664,12933,11035,576,10884'
         r = PerformanceTestResult(log_line.split(','))
@@ -517,6 +557,33 @@ def test_parse_delta_quantiles(self):
             # last 3 ventiles were outliers and were excluded from the sample
             (200, 214, 215, 18))
 
+    def test_parse_meta(self):
+        r = LogParser.results_from_string(
+            '#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),' +
+            'PAGES,ICS,YIELD\n' +
+            '0,B,1,2,2,2,0,2,7,29,15')['B']
+        self.assertEqual(
+            (r.min, r.mem_pages, r.involuntary_cs, r.yield_count),
+            (2, 7, 29, 15))
+        r = LogParser.results_from_string(
+            '#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),' +
+            'MAX_RSS(B),PAGES,ICS,YIELD\n' +
+            '0,B,1,3,3,3,0,3,36864,9,50,15')['B']
+        self.assertEqual(
+            (r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss),
+            (3, 9, 50, 15, 36864))
+        r = LogParser.results_from_string(
+            '#,TEST,SAMPLES,MIN(μs),MAX(μs),PAGES,ICS,YIELD\n' +
+            '0,B,1,4,4,8,31,15')['B']
+        self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count),
+                         (4, 8, 31, 15))
+        r = LogParser.results_from_string(
+            '#,TEST,SAMPLES,MIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD\n' +
+            '0,B,1,5,5,32768,8,28,15')['B']
+        self.assertEqual(
+            (r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss),
+            (5, 8, 28, 15, 32768))
+
     def test_parse_results_verbose(self):
         """Parse multiple performance test results with 2 sample formats:
         single line for N = 1; two lines for N > 1.

benchmark/utils/ArgParse.swift

@@ -112,7 +112,8 @@ class ArgumentParser<U> {
           .split(separator: "\n")
           .joined(separator: "\n" + padded(""))
        }
-      let positional = f("TEST", "name or number of the benchmark to measure")
+      let positional = f("TEST", "name or number of the benchmark to measure;\n"
+                         + "use +/- prefix to filter by substring match")
       let optional =  arguments.filter { $0.name != nil }
                                 .map { f($0.name!, $0.help ?? "") }
                                 .joined(separator: "\n")
@@ -152,7 +153,6 @@ class ArgumentParser<U> {
     /// We assume that optional switch args are of the form:
     ///
     ///     --opt-name[=opt-value]
-    ///     -opt-name[=opt-value]
     ///
     /// with `opt-name` and `opt-value` not containing any '=' signs. Any
     /// other option passed in is assumed to be a positional argument.
@@ -165,7 +165,7 @@ class ArgumentParser<U> {
       for arg in CommandLine.arguments[1..<CommandLine.arguments.count] {
         // If the argument doesn't match the optional argument pattern. Add
         // it to the positional argument list and continue...
-        if !arg.starts(with: "-") {
+        if !arg.starts(with: "--") {
           positionalArgs.append(arg)
           continue
         }