From deaddd212c499c7ff88d20034753b5f3f00d5153 Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Wed, 10 Oct 2018 13:22:55 -0400 Subject: [PATCH] tools,test: add list of slow tests they will be skipped if run with `--flaky-tests=skip` PR-URL: https://github.com/nodejs/node/pull/23251 Reviewed-By: Rich Trott --- test/root.status | 190 +++++++++++++++++++++++++++++++++++++++++++++++ tools/test.py | 45 ++++++----- 2 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 test/root.status diff --git a/test/root.status b/test/root.status new file mode 100644 index 00000000000000..9ed9004c2169d0 --- /dev/null +++ b/test/root.status @@ -0,0 +1,190 @@ +[$mode==debug] +async-hooks/test-callback-error: SLOW +async-hooks/test-callback-error: SLOW +async-hooks/test-emit-init: SLOW +async-hooks/test-emit-init: SLOW +async-hooks/test-querywrap: SLOW +async-hooks/test-querywrap: SLOW +async-hooks/test-tlswrap: SLOW +async-hooks/test-tlswrap: SLOW +message/eval_messages: SLOW +message/stdin_messages: SLOW +parallel/test-benchmark-assert: SLOW +parallel/test-benchmark-cluster: SLOW +parallel/test-benchmark-crypto: SLOW +parallel/test-benchmark-dns: SLOW +parallel/test-benchmark-domain: SLOW +parallel/test-benchmark-es: SLOW +parallel/test-benchmark-events: SLOW +parallel/test-benchmark-fs: SLOW +parallel/test-benchmark-misc: SLOW +parallel/test-benchmark-module: SLOW +parallel/test-benchmark-os: SLOW +parallel/test-benchmark-process: SLOW +parallel/test-benchmark-querystring: SLOW +parallel/test-benchmark-streams: SLOW +parallel/test-benchmark-string_decoder: SLOW +parallel/test-benchmark-timers: SLOW +parallel/test-benchmark-url: SLOW +parallel/test-benchmark-util: SLOW +parallel/test-benchmark-v8: SLOW +parallel/test-benchmark-vm: SLOW +parallel/test-benchmark-zlib: SLOW +parallel/test-buffer-constructor-node-modules-paths: SLOW +parallel/test-buffer-indexof: SLOW +parallel/test-child-process-spawnsync-input: SLOW +parallel/test-child-process-windows-hide: SLOW +parallel/test-cli-eval: SLOW +parallel/test-cli-eval-event: SLOW +parallel/test-cli-node-options: SLOW +parallel/test-cli-node-options-disallowed: SLOW +parallel/test-cli-node-print-help: SLOW +parallel/test-cli-syntax: SLOW +parallel/test-cluster-basic: SLOW +parallel/test-cluster-bind-privileged-port: SLOW +parallel/test-cluster-bind-twice: SLOW +parallel/test-cluster-disconnect: SLOW +parallel/test-cluster-disconnect-idle-worker: SLOW +parallel/test-crypto-fips: SLOW +parallel/test-domain-abort-on-uncaught: SLOW +parallel/test-domain-uncaught-exception: SLOW +parallel/test-domain-with-abort-on-uncaught-exception: SLOW +parallel/test-env-var-no-warnings: SLOW +parallel/test-error-reporting: SLOW +parallel/test-eslint-alphabetize-errors: SLOW +parallel/test-eslint-buffer-constructor: SLOW +parallel/test-eslint-crypto-check: SLOW +parallel/test-eslint-documented-errors: SLOW +parallel/test-eslint-duplicate-requires: SLOW +parallel/test-eslint-eslint-check: SLOW +parallel/test-eslint-inspector-check: SLOW +parallel/test-eslint-lowercase-name-for-primitive: SLOW +parallel/test-eslint-no-let-in-for-declaration: SLOW +parallel/test-eslint-no-unescaped-regexp-dot: SLOW +parallel/test-eslint-number-isnan: SLOW +parallel/test-eslint-prefer-assert-iferror: SLOW +parallel/test-eslint-prefer-assert-methods: SLOW +parallel/test-eslint-prefer-common-expectserror: SLOW +parallel/test-eslint-prefer-common-mustnotcall: SLOW +parallel/test-eslint-prefer-util-format-errors: SLOW +parallel/test-eslint-require-buffer: SLOW +parallel/test-eslint-required-modules: SLOW +parallel/test-fs-read-stream-concurrent-reads: SLOW +parallel/test-gc-tls-external-memory: SLOW +parallel/test-heapdump-dns: SLOW +parallel/test-heapdump-fs-promise: SLOW +parallel/test-heapdump-http2: SLOW +parallel/test-heapdump-inspector: SLOW +parallel/test-heapdump-tls: SLOW +parallel/test-heapdump-worker: SLOW +parallel/test-heapdump-zlib: SLOW +parallel/test-http-client-timeout-option-with-agent: SLOW +parallel/test-http-pipeline-flood: SLOW +parallel/test-http-pipeline-requests-connection-leak: SLOW +parallel/test-http2-forget-closed-streams: SLOW +parallel/test-http2-multiplex: SLOW +parallel/test-inspector-tracing-domain: SLOW +parallel/test-listen-fd-cluster: SLOW +parallel/test-module-loading-globalpaths: SLOW +parallel/test-module-main-fail: SLOW +parallel/test-module-main-preserve-symlinks-fail: SLOW +parallel/test-net-pingpong: SLOW +parallel/test-next-tick-fixed-queue-regression: SLOW +parallel/test-npm-install: SLOW +parallel/test-preload: SLOW +parallel/test-repl: SLOW +parallel/test-repl-tab-complete: SLOW +parallel/test-repl-top-level-await: SLOW +parallel/test-stdio-pipe-access: SLOW +parallel/test-stream-pipeline: SLOW +parallel/test-stream2-read-sync-stack: SLOW +parallel/test-stringbytes-external: SLOW +parallel/test-sync-io-option: SLOW +parallel/test-tick-processor-arguments: SLOW +parallel/test-tls-env-bad-extra-ca: SLOW +parallel/test-tls-env-extra-ca: SLOW +parallel/test-tls-handshake-exception: SLOW +parallel/test-tls-securepair-leak: SLOW +parallel/test-tls-server-verify: SLOW +parallel/test-tls-session-cache: SLOW +parallel/test-tls-ticket-cluster: SLOW +parallel/test-tls-timeout-server: SLOW +parallel/test-tls-timeout-server-2: SLOW +parallel/test-tls-tlswrap-segfault: SLOW +parallel/test-trace-events-all: SLOW +parallel/test-trace-events-api: SLOW +parallel/test-trace-events-async-hooks: SLOW +parallel/test-trace-events-binding: SLOW +parallel/test-trace-events-bootstrap: SLOW +parallel/test-trace-events-category-used: SLOW +parallel/test-trace-events-file-pattern: SLOW +parallel/test-trace-events-fs-sync: SLOW +parallel/test-trace-events-metadata: SLOW +parallel/test-trace-events-none: SLOW +parallel/test-trace-events-perf: SLOW +parallel/test-trace-events-process-exit: SLOW +parallel/test-trace-events-promises: SLOW +parallel/test-trace-events-v8: SLOW +parallel/test-trace-events-vm: SLOW +parallel/test-trace-events-worker-metadata: SLOW +parallel/test-tracing-no-crash: SLOW +parallel/test-url-relative: SLOW +parallel/test-util-callbackify: SLOW +parallel/test-util-inspect: SLOW +parallel/test-util-inspect-long-running: SLOW +parallel/test-util-types: SLOW +parallel/test-v8-coverage: SLOW +parallel/test-vm-api-handles-getter-errors: SLOW +parallel/test-vm-basic: SLOW +parallel/test-vm-cached-data: SLOW +parallel/test-vm-sigint: SLOW +parallel/test-vm-sigint-existing-handler: SLOW +parallel/test-vm-symbols: SLOW +parallel/test-vm-syntax-error-message: SLOW +parallel/test-vm-syntax-error-stderr: SLOW +parallel/test-worker: SLOW +parallel/test-worker-cleanup-handles: SLOW +parallel/test-worker-debug: SLOW +parallel/test-worker-esmodule: SLOW +parallel/test-worker-exit-code: SLOW +parallel/test-worker-memory: SLOW +parallel/test-worker-message-channel: SLOW +parallel/test-worker-message-channel-sharedarraybuffer: SLOW +parallel/test-worker-nexttick-terminate: SLOW +parallel/test-worker-onmessage: SLOW +parallel/test-worker-onmessage-not-a-function: SLOW +parallel/test-worker-parent-port-ref: SLOW +parallel/test-worker-relative-path: SLOW +parallel/test-worker-relative-path-double-dot: SLOW +parallel/test-worker-stdio: SLOW +parallel/test-worker-syntax-error: SLOW +parallel/test-worker-syntax-error-file: SLOW +parallel/test-worker-uncaught-exception: SLOW +parallel/test-worker-uncaught-exception-async: SLOW +parallel/test-worker-unsupported-things: SLOW +parallel/test-worker-workerdata-sharedarraybuffer: SLOW +parallel/test-zlib-bytes-read: SLOW +parallel/test-zlib-convenience-methods: SLOW +sequential/test-benchmark-buffer: SLOW +sequential/test-benchmark-child-process: SLOW +sequential/test-benchmark-dgram: SLOW +sequential/test-benchmark-http: SLOW +sequential/test-benchmark-net: SLOW +sequential/test-benchmark-path: SLOW +sequential/test-benchmark-tls: SLOW +sequential/test-child-process-execsync: SLOW +sequential/test-child-process-exit: SLOW +sequential/test-child-process-pass-fd: SLOW +sequential/test-fs-readfile-tostring-fail: SLOW +sequential/test-fs-watch-system-limit: SLOW +sequential/test-gc-http-client: SLOW +sequential/test-gc-http-client-connaborted: SLOW +sequential/test-gc-http-client-onerror: SLOW +sequential/test-gc-http-client-timeout: SLOW +sequential/test-gc-net-timeout: SLOW +sequential/test-http2-ping-flood: SLOW +sequential/test-http2-settings-flood: SLOW +sequential/test-inspector-port-cluster: SLOW +sequential/test-net-bytes-per-incoming-chunk-overhead: SLOW +sequential/test-pipe: SLOW +sequential/test-util-debug: SLOW diff --git a/tools/test.py b/tools/test.py index 3d62eedd1631e2..0571f3394bcc38 100755 --- a/tools/test.py +++ b/tools/test.py @@ -850,15 +850,15 @@ def GetTestStatus(self, context, sections, defs): class LiteralTestSuite(TestSuite): - - def __init__(self, tests): + def __init__(self, tests_repos, test_root): super(LiteralTestSuite, self).__init__('root') - self.tests = tests + self.tests_repos = tests_repos + self.test_root = test_root def GetBuildRequirements(self, path, context): (name, rest) = CarCdr(path) result = [ ] - for test in self.tests: + for test in self.tests_repos: if not name or name.match(test.GetName()): result += test.GetBuildRequirements(rest, context) return result @@ -866,7 +866,7 @@ def GetBuildRequirements(self, path, context): def ListTests(self, current_path, path, context, arch, mode): (name, rest) = CarCdr(path) result = [ ] - for test in self.tests: + for test in self.tests_repos: test_name = test.GetName() if not name or name.match(test_name): full_path = current_path + [test_name] @@ -875,8 +875,11 @@ def ListTests(self, current_path, path, context, arch, mode): return result def GetTestStatus(self, context, sections, defs): - for test in self.tests: - test.GetTestStatus(context, sections, defs) + # Just read the test configuration from root_path/root.status. + root = TestConfiguration(context, self.test_root, 'root') + root.GetTestStatus(sections, defs) + for tests_repos in self.tests_repos: + tests_repos.GetTestStatus(context, sections, defs) TIMEOUT_SCALEFACTOR = { @@ -1198,23 +1201,18 @@ def __init__(self, sections, defs): self.defs = defs def ClassifyTests(self, cases, env): - sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)] + sections = [ s for s in self.sections if s.condition.Evaluate(env, self.defs) ] all_rules = reduce(list.__add__, [s.rules for s in sections], []) unused_rules = set(all_rules) - result = [ ] - all_outcomes = set([]) + result = [] for case in cases: matches = [ r for r in all_rules if r.Contains(case.path) ] - outcomes = set([]) - for rule in matches: - outcomes = outcomes.union(rule.GetOutcomes(env, self.defs)) - unused_rules.discard(rule) - if not outcomes: - outcomes = [PASS] - case.outcomes = outcomes - all_outcomes = all_outcomes.union(outcomes) - result.append(ClassifiedTest(case, outcomes)) - return (result, list(unused_rules), all_outcomes) + outcomes_list = [ r.GetOutcomes(env, self.defs) for r in matches ] + outcomes = reduce(set.union, outcomes_list, set()) + unused_rules.difference_update(matches) + case.outcomes = set(outcomes) or set([PASS]) + result.append(case) + return result, unused_rules class Section(object): @@ -1552,7 +1550,7 @@ def Main(): repositories = [TestRepository(join(workspace, 'test', name)) for name in suites] repositories += [TestRepository(a) for a in options.suite] - root = LiteralTestSuite(repositories) + root = LiteralTestSuite(repositories, test_root) paths = ArgsToTestPaths(test_root, args, suites) # Check for --valgrind option. If enabled, we overwrite the special @@ -1623,8 +1621,7 @@ def Main(): } test_list = root.ListTests([], path, context, arch, mode) unclassified_tests += test_list - (cases, unused_rules, _) = ( - config.ClassifyTests(test_list, env)) + cases, unused_rules = config.ClassifyTests(test_list, env) if globally_unused_rules is None: globally_unused_rules = set(unused_rules) else: @@ -1671,7 +1668,7 @@ def should_keep(case): return False elif SKIP in case.outcomes: return False - elif (options.flaky_tests == SKIP) and (set([FLAKY]) & case.outcomes): + elif (options.flaky_tests == SKIP) and (set([SLOW, FLAKY]) & case.outcomes): return False else: return True