From dd473dfa39af8fc9e908fa0906590bbfcc32c932 Mon Sep 17 00:00:00 2001 From: Tyson Smith Date: Fri, 5 Jan 2024 12:04:42 -0800 Subject: [PATCH] [reduce] Update analysis Don't skip running single test case with harness relaunching each iteration. This is more robust and less complex. --- grizzly/reduce/core.py | 72 ++++++-------- grizzly/reduce/test_reduce.py | 173 ++++++++++++++++++---------------- 2 files changed, 120 insertions(+), 125 deletions(-) diff --git a/grizzly/reduce/core.py b/grizzly/reduce/core.py index 98f4c171..99206fd6 100644 --- a/grizzly/reduce/core.py +++ b/grizzly/reduce/core.py @@ -268,34 +268,26 @@ def run_reliability_analysis(self): # N.B. We only use `last_test_only` if `len(self.testcases) > 1` .. # if `self.testcases` only has 1 entry to begin with, we don't need # `last_test_only` to trim it - for use_harness, last_test_only in ( - (True, True), - (True, False), - # only one of the two use_harness=False cases will run: - # input is len(self.testcases)>1 and we will only try the last testcase - (False, True), - # input is len(self.testcases)==1 already and there's no need to trim - (False, False), + for use_harness, last_test_only, relaunch in ( + # don't relaunch between iterations (be quick) + (True, True, self.ANALYSIS_ITERATIONS), + # relaunch between iterations regardless of testcase count (be thorough) + (True, False, 1), + # without the harness we only try the last testcase + # relaunch between iterations is implied + (False, True, 1), ): - if use_harness and (not self._original_use_harness or harness_crashes): - # Don't test with harness again if we already found crashes with the - # harness (last_test_only) or if it was disabled by command-line. + harness_best = max(harness_crashes, harness_last_crashes) + if use_harness and ( + not self._original_use_harness + or harness_last_crashes >= self.ANALYSIS_ITERATIONS / 2 + ): + # Don't test with harness again if we already found enough crashes with + # the harness (last_test_only) or if it was disabled by command-line. continue - if not use_harness and harness_crashes >= self.ANALYSIS_ITERATIONS / 2: + if not use_harness and harness_best >= self.ANALYSIS_ITERATIONS / 2: # Don't test without harness if harness found > 50% crashes continue - if last_test_only and len(self.testcases) == 1: - # Only set `last_test_only` if we initially have more than one testcase - continue - if not use_harness and (not last_test_only and len(self.testcases) > 1): - # Can't run without harness if we have more than one testcase - # (`last_test_only` will run) - continue - - if use_harness and (last_test_only or len(self.testcases) == 1): - relaunch = self.ANALYSIS_ITERATIONS - else: - relaunch = 1 with ReplayManager( self.ignore, @@ -312,7 +304,7 @@ def run_reliability_analysis(self): "using" if use_harness else "without", ) testcases = self.testcases - if last_test_only: + if last_test_only and len(testcases) > 1: if use_harness: LOG.warning("Checking reliability with only the last testcase.") else: @@ -346,12 +338,10 @@ def run_reliability_analysis(self): testcases, ) if use_harness: - # set harness_crashes in both cases (last_test True/False) - # we only want to iterate through all testcases if the last - # testcase alone never reproduced (crashes == 0). - harness_crashes = crashes if last_test_only: harness_last_crashes = crashes + else: + harness_crashes = crashes else: non_harness_crashes = crashes finally: @@ -379,24 +369,20 @@ def run_reliability_analysis(self): # ensure same signature is always used self._signature = replay.signature - if not (harness_crashes or non_harness_crashes): + if not (harness_last_crashes or harness_crashes or non_harness_crashes): raise NotReproducible("Did not reproduce during analysis") - # if harness is selected, we'll only use the last testcase - if harness_last_crashes: - harness_crashes = harness_last_crashes - # should we use the harness? go with harness unless no-harness crashed 50% more self._use_harness = not ( - non_harness_crashes > harness_crashes + non_harness_crashes > harness_best and ( - harness_crashes == 0 - or (non_harness_crashes - harness_crashes) / harness_crashes >= 0.5 + harness_best == 0 + or (non_harness_crashes - harness_best) / harness_best >= 0.5 ) ) - if (self._use_harness and harness_last_crashes) or ( - not self._use_harness and len(self.testcases) > 1 + if len(self.testcases) > 1 and ( + not self._use_harness or harness_last_crashes >= harness_crashes ): LOG.warning( "Last testcase %s harness was selected, other %d " @@ -408,7 +394,7 @@ def run_reliability_analysis(self): self.testcases.pop(0).cleanup() crashes_percent = ( - harness_crashes if self._use_harness else non_harness_crashes + harness_best if self._use_harness else non_harness_crashes ) / self.ANALYSIS_ITERATIONS # adjust repeat/min-crashes depending on how reliable the testcase was @@ -428,19 +414,19 @@ def run_reliability_analysis(self): ) LOG.info("Analysis results:") - if harness_crashes >= self.ANALYSIS_ITERATIONS / 2: + if harness_best >= self.ANALYSIS_ITERATIONS / 2: LOG.info( "* testcase was better than 50% reliable with the harness " "(--no-harness not assessed)" ) - elif harness_crashes == non_harness_crashes: + elif harness_best == non_harness_crashes: LOG.info("* testcase was equally reliable with/without the harness") elif not self._original_use_harness: LOG.info("* --no-harness was already set") else: LOG.info( "* testcase was %s reliable with the harness", - "more" if harness_crashes > non_harness_crashes else "less", + "more" if harness_best > non_harness_crashes else "less", ) return (repeat, min_crashes) diff --git a/grizzly/reduce/test_reduce.py b/grizzly/reduce/test_reduce.py index def4f672..8d953e1b 100644 --- a/grizzly/reduce/test_reduce.py +++ b/grizzly/reduce/test_reduce.py @@ -58,9 +58,9 @@ def _fake_save_logs_bar(result_logs): "expected_min_crashes,use_harness,result_harness", [ param( - None, 11, None, + None, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, True, @@ -69,32 +69,42 @@ def _fake_save_logs_bar(result_logs): ), param( None, - 11, None, + 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, False, False, id="perfect, use_harness=False", ), - param(None, 10, None, 2, 1, True, True, id="10/11 with harness"), - param(None, 9, None, 2, 1, True, True, id="9/11 with harness"), - param(None, 8, None, 3, 1, True, True, id="8/11 with harness"), - param(None, 7, None, 3, 1, True, True, id="7/11 with harness"), - param(None, 6, None, 4, 1, True, True, id="6/11 with harness"), - param(None, 5, 0, 5, 1, True, True, id="5/11 with harness"), - param(None, 5, 1, 5, 1, True, True, id="5/11 with harness, 1/11 without"), - param(None, 5, 2, 5, 1, True, True, id="5/11 with harness, 2/11 without"), - param(None, 5, 3, 5, 1, True, True, id="5/11 with harness, 3/11 without"), - param(None, 5, 4, 5, 1, True, True, id="5/11 with harness, 4/11 without"), - param(None, 5, 5, 5, 1, True, True, id="5/11 with harness, 5/11 without"), - param(None, 5, 6, 5, 1, True, True, id="5/11 with harness, 6/11 without"), - param(None, 5, 7, 5, 1, True, True, id="5/11 with harness, 7/11 without"), - param(None, 5, 8, 3, 1, True, False, id="5/11 with harness, 8/11 without"), - param(None, 5, 9, 2, 1, True, False, id="5/11 with harness, 9/11 without"), - param(None, 5, 10, 2, 1, True, False, id="5/11 with harness, 10/11 without"), param( + 1, + 11, None, + ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, + ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, + True, + True, + id="1/11 with harness last test, perfect all tests", + ), + param(0, 10, None, 2, 1, True, True, id="10/11 with harness"), + param(0, 9, None, 2, 1, True, True, id="9/11 with harness"), + param(0, 8, None, 3, 1, True, True, id="8/11 with harness"), + param(0, 7, None, 3, 1, True, True, id="7/11 with harness"), + param(0, 6, None, 4, 1, True, True, id="6/11 with harness"), + param(0, 5, 0, 5, 1, True, True, id="5/11 with harness"), + param(0, 5, 1, 5, 1, True, True, id="5/11 with harness, 1/11 without"), + param(0, 5, 2, 5, 1, True, True, id="5/11 with harness, 2/11 without"), + param(0, 5, 3, 5, 1, True, True, id="5/11 with harness, 3/11 without"), + param(0, 5, 4, 5, 1, True, True, id="5/11 with harness, 4/11 without"), + param(0, 5, 5, 5, 1, True, True, id="5/11 with harness, 5/11 without"), + param(0, 5, 6, 5, 1, True, True, id="5/11 with harness, 6/11 without"), + param(0, 5, 7, 5, 1, True, True, id="5/11 with harness, 7/11 without"), + param(0, 5, 8, 3, 1, True, False, id="5/11 with harness, 8/11 without"), + param(0, 5, 9, 2, 1, True, False, id="5/11 with harness, 9/11 without"), + param(0, 5, 10, 2, 1, True, False, id="5/11 with harness, 10/11 without"), + param( + 0, 5, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -103,19 +113,19 @@ def _fake_save_logs_bar(result_logs): False, id="5/11 with harness, perfect without", ), - param(None, 4, 0, 7, 1, True, True, id="4/11 with harness"), - param(None, 4, 1, 7, 1, True, True, id="4/11 with harness, 1/11 without"), - param(None, 4, 2, 7, 1, True, True, id="4/11 with harness, 2/11 without"), - param(None, 4, 3, 7, 1, True, True, id="4/11 with harness, 3/11 without"), - param(None, 4, 4, 7, 1, True, True, id="4/11 with harness, 4/11 without"), - param(None, 4, 5, 7, 1, True, True, id="4/11 with harness, 5/11 without"), - param(None, 4, 6, 4, 1, True, False, id="4/11 with harness, 6/11 without"), - param(None, 4, 7, 3, 1, True, False, id="4/11 with harness, 7/11 without"), - param(None, 4, 8, 3, 1, True, False, id="4/11 with harness, 8/11 without"), - param(None, 4, 9, 2, 1, True, False, id="4/11 with harness, 9/11 without"), - param(None, 4, 10, 2, 1, True, False, id="4/11 with harness, 10/11 without"), + param(0, 4, 0, 7, 1, True, True, id="4/11 with harness"), + param(0, 4, 1, 7, 1, True, True, id="4/11 with harness, 1/11 without"), + param(0, 4, 2, 7, 1, True, True, id="4/11 with harness, 2/11 without"), + param(0, 4, 3, 7, 1, True, True, id="4/11 with harness, 3/11 without"), + param(0, 4, 4, 7, 1, True, True, id="4/11 with harness, 4/11 without"), + param(0, 4, 5, 7, 1, True, True, id="4/11 with harness, 5/11 without"), + param(0, 4, 6, 4, 1, True, False, id="4/11 with harness, 6/11 without"), + param(0, 4, 7, 3, 1, True, False, id="4/11 with harness, 7/11 without"), + param(0, 4, 8, 3, 1, True, False, id="4/11 with harness, 8/11 without"), + param(0, 4, 9, 2, 1, True, False, id="4/11 with harness, 9/11 without"), + param(0, 4, 10, 2, 1, True, False, id="4/11 with harness, 10/11 without"), param( - None, + 0, 4, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -124,19 +134,19 @@ def _fake_save_logs_bar(result_logs): False, id="4/11 with harness, perfect without", ), - param(None, 3, 0, 10, 1, True, True, id="3/11 with harness"), - param(None, 3, 1, 10, 1, True, True, id="3/11 with harness, 1/11 without"), - param(None, 3, 2, 10, 1, True, True, id="3/11 with harness, 2/11 without"), - param(None, 3, 3, 10, 1, True, True, id="3/11 with harness, 3/11 without"), - param(None, 3, 4, 10, 1, True, True, id="3/11 with harness, 4/11 without"), - param(None, 3, 5, 5, 1, True, False, id="3/11 with harness, 5/11 without"), - param(None, 3, 6, 4, 1, True, False, id="3/11 with harness, 6/11 without"), - param(None, 3, 7, 3, 1, True, False, id="3/11 with harness, 7/11 without"), - param(None, 3, 8, 3, 1, True, False, id="3/11 with harness, 8/11 without"), - param(None, 3, 9, 2, 1, True, False, id="3/11 with harness, 9/11 without"), - param(None, 3, 10, 2, 1, True, False, id="3/11 with harness, 10/11 without"), + param(0, 3, 0, 10, 1, True, True, id="3/11 with harness"), + param(0, 3, 1, 10, 1, True, True, id="3/11 with harness, 1/11 without"), + param(0, 3, 2, 10, 1, True, True, id="3/11 with harness, 2/11 without"), + param(0, 3, 3, 10, 1, True, True, id="3/11 with harness, 3/11 without"), + param(0, 3, 4, 10, 1, True, True, id="3/11 with harness, 4/11 without"), + param(0, 3, 5, 5, 1, True, False, id="3/11 with harness, 5/11 without"), + param(0, 3, 6, 4, 1, True, False, id="3/11 with harness, 6/11 without"), + param(0, 3, 7, 3, 1, True, False, id="3/11 with harness, 7/11 without"), + param(0, 3, 8, 3, 1, True, False, id="3/11 with harness, 8/11 without"), + param(0, 3, 9, 2, 1, True, False, id="3/11 with harness, 9/11 without"), + param(0, 3, 10, 2, 1, True, False, id="3/11 with harness, 10/11 without"), param( - None, + 0, 3, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -145,19 +155,19 @@ def _fake_save_logs_bar(result_logs): False, id="3/11 with harness, perfect without", ), - param(None, 2, 0, 15, 1, True, True, id="2/11 with harness"), - param(None, 2, 1, 15, 1, True, True, id="2/11 with harness, 1/11 without"), - param(None, 2, 2, 15, 1, True, True, id="2/11 with harness, 2/11 without"), - param(None, 2, 3, 10, 1, True, False, id="2/11 with harness, 3/11 without"), - param(None, 2, 4, 7, 1, True, False, id="2/11 with harness, 4/11 without"), - param(None, 2, 5, 5, 1, True, False, id="2/11 with harness, 5/11 without"), - param(None, 2, 6, 4, 1, True, False, id="2/11 with harness, 6/11 without"), - param(None, 2, 7, 3, 1, True, False, id="2/11 with harness, 7/11 without"), - param(None, 2, 8, 3, 1, True, False, id="2/11 with harness, 8/11 without"), - param(None, 2, 9, 2, 1, True, False, id="2/11 with harness, 9/11 without"), - param(None, 2, 10, 2, 1, True, False, id="2/11 with harness, 10/11 without"), + param(0, 2, 0, 15, 1, True, True, id="2/11 with harness"), + param(0, 2, 1, 15, 1, True, True, id="2/11 with harness, 1/11 without"), + param(0, 2, 2, 15, 1, True, True, id="2/11 with harness, 2/11 without"), + param(0, 2, 3, 10, 1, True, False, id="2/11 with harness, 3/11 without"), + param(0, 2, 4, 7, 1, True, False, id="2/11 with harness, 4/11 without"), + param(0, 2, 5, 5, 1, True, False, id="2/11 with harness, 5/11 without"), + param(0, 2, 6, 4, 1, True, False, id="2/11 with harness, 6/11 without"), + param(0, 2, 7, 3, 1, True, False, id="2/11 with harness, 7/11 without"), + param(0, 2, 8, 3, 1, True, False, id="2/11 with harness, 8/11 without"), + param(0, 2, 9, 2, 1, True, False, id="2/11 with harness, 9/11 without"), + param(0, 2, 10, 2, 1, True, False, id="2/11 with harness, 10/11 without"), param( - None, + 0, 2, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -166,19 +176,19 @@ def _fake_save_logs_bar(result_logs): False, id="2/11 with harness, perfect without", ), - param(None, 1, 0, 32, 1, True, True, id="1/11 with harness"), - param(None, 1, 1, 32, 1, True, True, id="1/11 with harness, 1/11 without"), - param(None, 1, 2, 15, 1, True, False, id="1/11 with harness, 2/11 without"), - param(None, 1, 3, 10, 1, True, False, id="1/11 with harness, 3/11 without"), - param(None, 1, 4, 7, 1, True, False, id="1/11 with harness, 4/11 without"), - param(None, 1, 5, 5, 1, True, False, id="1/11 with harness, 5/11 without"), - param(None, 1, 6, 4, 1, True, False, id="1/11 with harness, 6/11 without"), - param(None, 1, 7, 3, 1, True, False, id="1/11 with harness, 7/11 without"), - param(None, 1, 8, 3, 1, True, False, id="1/11 with harness, 8/11 without"), - param(None, 1, 9, 2, 1, True, False, id="1/11 with harness, 9/11 without"), - param(None, 1, 10, 2, 1, True, False, id="1/11 with harness, 10/11 without"), + param(0, 1, 0, 32, 1, True, True, id="1/11 with harness"), + param(0, 1, 1, 32, 1, True, True, id="1/11 with harness, 1/11 without"), + param(0, 1, 2, 15, 1, True, False, id="1/11 with harness, 2/11 without"), + param(0, 1, 3, 10, 1, True, False, id="1/11 with harness, 3/11 without"), + param(0, 1, 4, 7, 1, True, False, id="1/11 with harness, 4/11 without"), + param(0, 1, 5, 5, 1, True, False, id="1/11 with harness, 5/11 without"), + param(0, 1, 6, 4, 1, True, False, id="1/11 with harness, 6/11 without"), + param(0, 1, 7, 3, 1, True, False, id="1/11 with harness, 7/11 without"), + param(0, 1, 8, 3, 1, True, False, id="1/11 with harness, 8/11 without"), + param(0, 1, 9, 2, 1, True, False, id="1/11 with harness, 9/11 without"), + param(0, 1, 10, 2, 1, True, False, id="1/11 with harness, 10/11 without"), param( - None, + 0, 1, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -188,7 +198,7 @@ def _fake_save_logs_bar(result_logs): id="1/11 with harness, perfect without", ), param( - None, + 0, 0, 11, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES, @@ -197,26 +207,25 @@ def _fake_save_logs_bar(result_logs): False, id="perfect without harness", ), - param(None, 0, 10, 2, 1, True, False, id="10/11 without harness"), - param(None, 0, 9, 2, 1, True, False, id="9/11 without harness"), - param(None, 0, 8, 3, 1, True, False, id="8/11 without harness"), - param(None, 0, 7, 3, 1, True, False, id="7/11 without harness"), - param(None, 0, 6, 4, 1, True, False, id="6/11 without harness"), - param(None, 0, 5, 5, 1, True, False, id="5/11 without harness"), - param(None, 0, 4, 7, 1, True, False, id="4/11 without harness"), - param(None, 0, 3, 10, 1, True, False, id="3/11 without harness"), - param(None, 0, 2, 15, 1, True, False, id="2/11 without harness"), - param(None, 0, 1, 32, 1, True, False, id="1/11 without harness"), + param(0, 0, 10, 2, 1, True, False, id="10/11 without harness"), + param(0, 0, 9, 2, 1, True, False, id="9/11 without harness"), + param(0, 0, 8, 3, 1, True, False, id="8/11 without harness"), + param(0, 0, 7, 3, 1, True, False, id="7/11 without harness"), + param(0, 0, 6, 4, 1, True, False, id="6/11 without harness"), + param(0, 0, 5, 5, 1, True, False, id="5/11 without harness"), + param(0, 0, 4, 7, 1, True, False, id="4/11 without harness"), + param(0, 0, 3, 10, 1, True, False, id="3/11 without harness"), + param(0, 0, 2, 15, 1, True, False, id="2/11 without harness"), + param(0, 0, 1, 32, 1, True, False, id="1/11 without harness"), param( 1, - None, + 0, 0, 32, 1, True, True, - id="last test repros (1/11), all tests are not tried, " - "without_harness is tried", + id="last test repros (1/11), all tests are tried, without_harness is tried", ), param( 11, @@ -229,7 +238,7 @@ def _fake_save_logs_bar(result_logs): id="last test repros (11/11), no other rounds are tried", ), param( - None, + 0, 1, 0, 32, @@ -240,7 +249,7 @@ def _fake_save_logs_bar(result_logs): "without harness is tried", ), param( - None, + 0, 11, None, ReduceManager.ANALYSIS_PERFECT_MIN_CRASHES,