From bc4a5b94325f6b085e959b8256cabc8478c8b00b Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Fri, 7 Nov 2014 14:15:24 +0100 Subject: [PATCH 1/6] test: runner support for flaky tests Adding --flaky-tests option, to allow regarding flaky tests failures as non-fatal. Currently only observed by the TapProgressIndicator, which will add a # TODO directive to tests classified as flaky. According to the TAP specification, the test harness is supposed to treat failures that have a # TODO directive as non-fatal. Ported from https://github.com/joyent/node/commit/df3a2b2cf21274fe7afc19d14ec0259b964e13f7 --- tools/test.py | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tools/test.py b/tools/test.py index 7b5f287064d3fd..eeb4aa621f5bdb 100755 --- a/tools/test.py +++ b/tools/test.py @@ -61,8 +61,9 @@ class ProgressIndicator(object): - def __init__(self, cases): + def __init__(self, cases, flaky_tests_mode): self.cases = cases + self.flaky_tests_mode = flaky_tests_mode self.parallel_queue = Queue(len(cases)) self.sequential_queue = Queue(len(cases)) for case in cases: @@ -251,7 +252,10 @@ def HasRun(self, output): self._done += 1 command = basename(output.command[-1]) if output.UnexpectedOutput(): - logger.info('not ok %i - %s' % (self._done, command)) + status_line = 'not ok %i - %s' % (self._done, command) + if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE: + status_line = status_line + ' # TODO : Fix flaky test' + logger.info(status_line) for l in output.output.stderr.splitlines(): logger.info('#' + l) for l in output.output.stdout.splitlines(): @@ -262,7 +266,10 @@ def HasRun(self, output): logger.info( 'ok %i - %s # skip %s' % (self._done, command, skip.group(1))) else: - logger.info('ok %i - %s' % (self._done, command)) + status_line = 'ok %i - %s' % (self._done, command) + if FLAKY in output.test.outcomes: + status_line = status_line + ' # TODO : Fix flaky test' + logger.info(status_line) duration = output.test.duration @@ -280,8 +287,8 @@ def Done(self): class CompactProgressIndicator(ProgressIndicator): - def __init__(self, cases, templates): - super(CompactProgressIndicator, self).__init__(cases) + def __init__(self, cases, flaky_tests_mode, templates): + super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode) self.templates = templates self.last_status_length = 0 self.start_time = time.time() @@ -336,13 +343,13 @@ def PrintProgress(self, name): class ColorProgressIndicator(CompactProgressIndicator): - def __init__(self, cases): + def __init__(self, cases, flaky_tests_mode): templates = { 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s", 'stdout': "\033[1m%s\033[0m", 'stderr': "\033[31m%s\033[0m", } - super(ColorProgressIndicator, self).__init__(cases, templates) + super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates) def ClearLine(self, last_line_length): print "\033[1K\r", @@ -350,7 +357,7 @@ def ClearLine(self, last_line_length): class MonochromeProgressIndicator(CompactProgressIndicator): - def __init__(self, cases): + def __init__(self, cases, flaky_tests_mode): templates = { 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", 'stdout': '%s', @@ -358,7 +365,7 @@ def __init__(self, cases): 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), 'max_length': 78 } - super(MonochromeProgressIndicator, self).__init__(cases, templates) + super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates) def ClearLine(self, last_line_length): print ("\r" + (" " * last_line_length) + "\r"), @@ -780,8 +787,8 @@ def GetVmFlags(self, testcase, mode): def GetTimeout(self, mode): return self.timeout * TIMEOUT_SCALEFACTOR[ARCH_GUESS or 'ia32'][mode] -def RunTestCases(cases_to_run, progress, tasks): - progress = PROGRESS_INDICATORS[progress](cases_to_run) +def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode): + progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode) return progress.Run(tasks) @@ -805,7 +812,8 @@ def BuildRequirements(context, requirements, mode, scons_flags): TIMEOUT = 'timeout' CRASH = 'crash' SLOW = 'slow' - +FLAKY = 'flaky' +DONTCARE = 'dontcare' class Expression(object): pass @@ -1253,6 +1261,9 @@ def BuildOptions(): default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") + result.add_option("--flaky-tests", + help="Regard tests marked as flaky (run|skip|dontcare)", + default="run") result.add_option("--warn-unused", help="Report unused rules", default=False, action="store_true") result.add_option("-j", help="The number of parallel tasks to run", @@ -1303,6 +1314,9 @@ def ProcessOptions(options): return False if options.J: options.j = multiprocessing.cpu_count() + if options.flaky_tests not in ["run", "skip", "dontcare"]: + print "Unknown flaky-tests mode %s" % options.flaky_tests + return False return True @@ -1505,7 +1519,9 @@ def Main(): result = None def DoSkip(case): - return SKIP in case.outcomes or SLOW in case.outcomes + if SKIP in case.outcomes or SLOW in case.outcomes: + return True + return FLAKY in case.outcomes and options.flaky_tests == SKIP cases_to_run = [ c for c in all_cases if not DoSkip(c) ] if options.run is not None: # Must ensure the list of tests is sorted before selecting, to avoid @@ -1522,7 +1538,7 @@ def DoSkip(case): else: try: start = time.time() - if RunTestCases(cases_to_run, options.progress, options.j): + if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests): result = 0 else: result = 1 From 50319ed78c16123b75592966a31fa8ddab4b1473 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Tue, 7 Jul 2015 22:48:26 +0200 Subject: [PATCH 2/6] test: runner should return 0 on flaky tests Make the test runner return a 0 exit code when only flaky tests fail and --flaky-tests=dontcare is specified. Ported from https://github.com/joyent/node/commit/a9b642cf5b539e1b86698465d3d0b941d84c5352 --- tools/test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/test.py b/tools/test.py index eeb4aa621f5bdb..e237722b268d2c 100755 --- a/tools/test.py +++ b/tools/test.py @@ -75,7 +75,9 @@ def __init__(self, cases, flaky_tests_mode): self.remaining = len(cases) self.total = len(cases) self.failed = [ ] + self.flaky_failed = [ ] self.crashed = 0 + self.flaky_crashed = 0 self.lock = threading.Lock() self.shutdown_event = threading.Event() @@ -143,9 +145,14 @@ def RunSingle(self, parallel, thread_id): return self.lock.acquire() if output.UnexpectedOutput(): - self.failed.append(output) - if output.HasCrashed(): - self.crashed += 1 + if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE: + self.flaky_failed.append(output) + if output.HasCrashed(): + self.flaky_crashed += 1 + else: + self.failed.append(output) + if output.HasCrashed(): + self.crashed += 1 else: self.succeeded += 1 self.remaining -= 1 From b448b8351ca686e88ae08764b2465263992b4af6 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Tue, 18 Aug 2015 14:04:14 +0200 Subject: [PATCH 3/6] test: add test configuration templates Template configuration files for marking tests as flaky. --- test/message/message.status | 18 ++++++++++++++++++ test/parallel/parallel.status | 18 ++++++++++++++++++ test/sequential/sequential.status | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 test/message/message.status create mode 100644 test/parallel/parallel.status create mode 100644 test/sequential/sequential.status diff --git a/test/message/message.status b/test/message/message.status new file mode 100644 index 00000000000000..1a4a0e3adc2727 --- /dev/null +++ b/test/message/message.status @@ -0,0 +1,18 @@ +prefix message + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$system==solaris] # Also applies to SmartOS + +[$system==freebsd] + diff --git a/test/parallel/parallel.status b/test/parallel/parallel.status new file mode 100644 index 00000000000000..e19754834ef3a6 --- /dev/null +++ b/test/parallel/parallel.status @@ -0,0 +1,18 @@ +prefix parallel + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$system==solaris] # Also applies to SmartOS + +[$system==freebsd] + diff --git a/test/sequential/sequential.status b/test/sequential/sequential.status new file mode 100644 index 00000000000000..3ff77992f15543 --- /dev/null +++ b/test/sequential/sequential.status @@ -0,0 +1,18 @@ +prefix sequential + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$system==solaris] # Also applies to SmartOS + +[$system==freebsd] + From fd05d466d2700da133d9a4d8adca7cc8ba698fe7 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Wed, 26 Aug 2015 12:38:48 +0200 Subject: [PATCH 4/6] test: support flaky tests in test-ci Adding support for specifying flaky test mode to the test runner: - via an environment variable FLAKY_TESTS for Makefile - via an argument ignore-flaky for vcbuild.bat Ported from https://github.com/joyent/node/commit/2d2494cf140c38327218a087593ff2177a9d0ec9 --- Makefile | 3 ++- vcbuild.bat | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bccdc9afa054ab..c82a7991317754 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PYTHON ?= python DESTDIR ?= SIGN ?= PREFIX ?= /usr/local +FLAKY_TESTS ?= run STAGINGSERVER ?= node-www OSTYPE := $(shell uname -s | tr '[A-Z]' '[a-z]') @@ -140,7 +141,7 @@ test-all-valgrind: test-build $(PYTHON) tools/test.py --mode=debug,release --valgrind test-ci: | build-addons - $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release \ + $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --flaky-tests=$(FLAKY_TESTS) \ addons message parallel sequential test-release: test-build diff --git a/vcbuild.bat b/vcbuild.bat index 1073a3e5886eea..4ebb0ede00b469 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -70,6 +70,7 @@ if /i "%1"=="small-icu" set i18n_arg=%1&goto arg-ok if /i "%1"=="full-icu" set i18n_arg=%1&goto arg-ok if /i "%1"=="intl-none" set i18n_arg=%1&goto arg-ok if /i "%1"=="download-all" set download_arg="--download=all"&goto arg-ok +if /i "%1"=="ignore-flaky" set test_args=%test_args% --flaky-tests=dontcare&goto arg-ok echo Warning: ignoring invalid command line option `%1`. From 8567b2ae685089bba6f1da7fc2caea130dfac567 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Wed, 26 Aug 2015 12:41:16 +0200 Subject: [PATCH 5/6] test: pass args to test-ci via env variable --- Makefile | 3 ++- vcbuild.bat | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c82a7991317754..3f8f5fa30de32e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ DESTDIR ?= SIGN ?= PREFIX ?= /usr/local FLAKY_TESTS ?= run +TEST_CI_ARGS ?= STAGINGSERVER ?= node-www OSTYPE := $(shell uname -s | tr '[A-Z]' '[a-z]') @@ -142,7 +143,7 @@ test-all-valgrind: test-build test-ci: | build-addons $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --flaky-tests=$(FLAKY_TESTS) \ - addons message parallel sequential + $(TEST_CI_ARGS) addons message parallel sequential test-release: test-build $(PYTHON) tools/test.py --mode=release diff --git a/vcbuild.bat b/vcbuild.bat index 4ebb0ede00b469..056a1c7f04c8e6 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -54,7 +54,7 @@ if /i "%1"=="noetw" set noetw=1&goto arg-ok if /i "%1"=="noperfctr" set noperfctr=1&goto arg-ok if /i "%1"=="licensertf" set licensertf=1&goto arg-ok if /i "%1"=="test" set test_args=%test_args% sequential parallel message -J&set jslint=1&goto arg-ok -if /i "%1"=="test-ci" set test_args=%test_args% -p tap --logfile test.tap message sequential parallel&goto arg-ok +if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap message sequential parallel&goto arg-ok if /i "%1"=="test-simple" set test_args=%test_args% sequential parallel -J&goto arg-ok if /i "%1"=="test-message" set test_args=%test_args% message&goto arg-ok if /i "%1"=="test-gc" set test_args=%test_args% gc&set buildnodeweak=1&goto arg-ok From 8e60b287ccf269ecca0221275e9e0f534a4c4078 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Wed, 26 Aug 2015 12:42:45 +0200 Subject: [PATCH 6/6] test: initial list of flaky tests This poplulates the lists of flaky tests with tests that failed recently in Jenkins. --- test/parallel/parallel.status | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/parallel/parallel.status b/test/parallel/parallel.status index e19754834ef3a6..86c271956d9389 100644 --- a/test/parallel/parallel.status +++ b/test/parallel/parallel.status @@ -7,12 +7,21 @@ prefix parallel [true] # This section applies to all platforms [$system==win32] +test-tls-ticket-cluster : PASS,FLAKY [$system==linux] +test-cluster-worker-forced-exit : PASS,FLAKY +test-http-client-timeout-event : PASS,FLAKY +test-process-argv-0 : PASS,FLAKY +test-tick-processor : PASS,FLAKY +test-tls-no-sslv3 : PASS,FLAKY +test-child-process-buffering : PASS,FLAKY +test-child-process-exit-code : PASS,FLAKY [$system==macos] [$system==solaris] # Also applies to SmartOS +test-debug-signal-cluster : PASS,FLAKY [$system==freebsd] - +test-net-socket-local-address : PASS,FLAKY