From d2080014342b345fde399f27bf3d2270d8dd5804 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 25 Feb 2015 16:39:32 +0000 Subject: [PATCH 1/3] Refactor logging and test environment into their own files. --- test/test.py | 5 +- wptrunner/environment.py | 215 ++++++++++++++++++++++++ wptrunner/wptlogging.py | 125 ++++++++++++++ wptrunner/wptrunner.py | 351 +++------------------------------------ 4 files changed, 368 insertions(+), 328 deletions(-) create mode 100644 wptrunner/environment.py create mode 100644 wptrunner/wptlogging.py diff --git a/test/test.py b/test/test.py index b6412caf5e7e0d..bca2c106132a5c 100644 --- a/test/test.py +++ b/test/test.py @@ -18,7 +18,7 @@ def setup_wptrunner_logging(logger): structuredlog.set_default_logger(logger) wptrunner.logger = logger - wptrunner.setup_stdlib_logger() + wptrunner.wptlogging.setup_stdlib_logger() class ResultHandler(BaseHandler): def __init__(self, verbose=False, logger=None): @@ -155,7 +155,8 @@ def main(): import pdb, traceback print traceback.format_exc() pdb.post_mortem() - + else: + raise if __name__ == "__main__": main() diff --git a/wptrunner/environment.py b/wptrunner/environment.py new file mode 100644 index 00000000000000..2f48d177b87bb4 --- /dev/null +++ b/wptrunner/environment.py @@ -0,0 +1,215 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import shutil +import socket +import sys +import time + +from mozlog.structured import get_default_logger, handlers + +from wptlogging import LogLevelRewriter + +here = os.path.split(__file__)[0] + +serve = None +sslutils = None + +def do_delayed_imports(logger, test_paths): + global serve, sslutils + + serve_root = serve_path(test_paths) + + sys.path.insert(0, serve_root) + + failed = [] + + try: + from tools.serve import serve + except ImportError: + failed.append("serve") + + try: + import sslutils + except ImportError: + raise + failed.append("sslutils") + + if failed: + logger.critical( + "Failed to import %s. Ensure that tests path %s contains web-platform-tests" % + (", ".join(failed), serve_root)) + sys.exit(1) + + +def serve_path(test_paths): + return test_paths["/"]["tests_path"] + + +def get_ssl_kwargs(**kwargs): + if kwargs["ssl_type"] == "openssl": + args = {"openssl_binary": kwargs["openssl_binary"]} + elif kwargs["ssl_type"] == "pregenerated": + args = {"host_key_path": kwargs["host_key_path"], + "host_cert_path": kwargs["host_cert_path"], + "ca_cert_path": kwargs["ca_cert_path"]} + else: + args = {} + return args + + +def ssl_env(logger, **kwargs): + ssl_env_cls = sslutils.environments[kwargs["ssl_type"]] + return ssl_env_cls(logger, **get_ssl_kwargs(**kwargs)) + + +class TestEnvironmentError(Exception): + pass + + +class TestEnvironment(object): + def __init__(self, test_paths, ssl_env, options): + """Context manager that owns the test environment i.e. the http and + websockets servers""" + self.test_paths = test_paths + self.ssl_env = ssl_env + self.server = None + self.config = None + self.external_config = None + self.test_server_port = options.pop("test_server_port", True) + self.options = options if options is not None else {} + self.required_files = options.pop("required_files", []) + self.files_to_restore = [] + + def __enter__(self): + self.ssl_env.__enter__() + self.copy_required_files() + self.setup_server_logging() + self.setup_routes() + self.config = self.load_config() + serve.set_computed_defaults(self.config) + self.external_config, self.servers = serve.start(self.config, self.ssl_env) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.ssl_env.__exit__(exc_type, exc_val, exc_tb) + + self.restore_files() + for scheme, servers in self.servers.iteritems(): + for port, server in servers: + server.kill() + + def load_config(self): + default_config_path = os.path.join(serve_path(self.test_paths), "config.default.json") + local_config_path = os.path.join(here, "config.json") + + with open(default_config_path) as f: + default_config = json.load(f) + + with open(local_config_path) as f: + data = f.read() + local_config = json.loads(data % self.options) + + #TODO: allow non-default configuration for ssl + + local_config["external_host"] = self.options.get("external_host", None) + local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False) + + config = serve.merge_json(default_config, local_config) + config["doc_root"] = serve_path(self.test_paths) + + if not self.ssl_env.ssl_enabled: + config["ports"]["https"] = [None] + + host = self.options.get("certificate_domain", config["host"]) + hosts = [host] + hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values()) + key_file, certificate = self.ssl_env.host_cert_path(hosts) + + config["key_file"] = key_file + config["certificate"] = certificate + + return config + + def setup_server_logging(self): + server_logger = get_default_logger(component="wptserve") + assert server_logger is not None + log_filter = handlers.LogLevelFilter(lambda x:x, "info") + # Downgrade errors to warnings for the server + log_filter = LogLevelRewriter(log_filter, ["error"], "warning") + server_logger.component_filter = log_filter + + try: + #Set as the default logger for wptserve + serve.set_logger(server_logger) + serve.logger = server_logger + except Exception: + # This happens if logging has already been set up for wptserve + pass + + def setup_routes(self): + for url, paths in self.test_paths.iteritems(): + if url == "/": + continue + + path = paths["tests_path"] + url = "/%s/" % url.strip("/") + + for (method, + suffix, + handler_cls) in [(serve.any_method, + b"*.py", + serve.handlers.PythonScriptHandler), + (b"GET", + "*.asis", + serve.handlers.AsIsHandler), + (b"GET", + "*", + serve.handlers.FileHandler)]: + route = (method, b"%s%s" % (str(url), str(suffix)), handler_cls(path, url_base=url)) + serve.routes.insert(-3, route) + + if "/" not in self.test_paths: + serve.routes = serve.routes[:-3] + + def copy_required_files(self): + logger.info("Placing required files in server environment.") + for source, destination, copy_if_exists in self.required_files: + source_path = os.path.join(here, source) + dest_path = os.path.join(serve_path(self.test_paths), destination, os.path.split(source)[1]) + dest_exists = os.path.exists(dest_path) + if not dest_exists or copy_if_exists: + if dest_exists: + backup_path = dest_path + ".orig" + logger.info("Backing up %s to %s" % (dest_path, backup_path)) + self.files_to_restore.append(dest_path) + shutil.copy2(dest_path, backup_path) + logger.info("Copying %s to %s" % (source_path, dest_path)) + shutil.copy2(source_path, dest_path) + + def ensure_started(self): + # Pause for a while to ensure that the server has a chance to start + time.sleep(2) + for scheme, servers in self.servers.iteritems(): + for port, server in servers: + if self.test_server_port: + s = socket.socket() + try: + s.connect((self.config["host"], port)) + except socket.error: + raise EnvironmentError( + "%s server on port %d failed to start" % (scheme, port)) + finally: + s.close() + + if not server.is_alive(): + raise EnvironmentError("%s server on port %d failed to start" % (scheme, port)) + + def restore_files(self): + for path in self.files_to_restore: + os.unlink(path) + if os.path.exists(path + ".orig"): + os.rename(path + ".orig", path) diff --git a/wptrunner/wptlogging.py b/wptrunner/wptlogging.py new file mode 100644 index 00000000000000..d9bfed55352615 --- /dev/null +++ b/wptrunner/wptlogging.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import sys +import threading +from StringIO import StringIO +from multiprocessing import Queue + +from mozlog.structured import commandline, stdadapter + +def setup(args, defaults): + logger = commandline.setup_logging("web-platform-tests", args, defaults) + setup_stdlib_logger() + + for name in args.keys(): + if name.startswith("log_"): + args.pop(name) + + return logger + + +def setup_stdlib_logger(): + logging.root.handlers = [] + logging.root = stdadapter.std_logging_adapter(logging.root) + + +class LogLevelRewriter(object): + """Filter that replaces log messages at specified levels with messages + at a different level. + + This can be used to e.g. downgrade log messages from ERROR to WARNING + in some component where ERRORs are not critical. + + :param inner: Handler to use for messages that pass this filter + :param from_levels: List of levels which should be affected + :param to_level: Log level to set for the affected messages + """ + def __init__(self, inner, from_levels, to_level): + self.inner = inner + self.from_levels = [item.upper() for item in from_levels] + self.to_level = to_level.upper() + + def __call__(self, data): + if data["action"] == "log" and data["level"].upper() in self.from_levels: + data = data.copy() + data["level"] = self.to_level + return self.inner(data) + + + +class LogThread(threading.Thread): + def __init__(self, queue, logger, level): + self.queue = queue + self.log_func = getattr(logger, level) + threading.Thread.__init__(self, name="Thread-Log") + self.daemon = True + + def run(self): + while True: + try: + msg = self.queue.get() + except (EOFError, IOError): + break + if msg is None: + break + else: + self.log_func(msg) + + +class LoggingWrapper(StringIO): + """Wrapper for file like objects to redirect output to logger + instead""" + + def __init__(self, queue, prefix=None): + self.queue = queue + self.prefix = prefix + + def write(self, data): + if isinstance(data, str): + data = data.decode("utf8") + + if data.endswith("\n"): + data = data[:-1] + if data.endswith("\r"): + data = data[:-1] + if not data: + return + if self.prefix is not None: + data = "%s: %s" % (self.prefix, data) + self.queue.put(data) + + def flush(self): + pass + +class CaptureIO(object): + def __init__(self, logger, do_capture): + self.logger = logger + self.do_capture = do_capture + self.logging_queue = None + self.logging_thread = None + self.original_stdio = None + + def __enter__(self): + if self.do_capture: + self.original_stdio = (sys.stdout, sys.stderr) + self.logging_queue = Queue() + self.logging_thread = LogThread(self.logging_queue, self.logger, "info") + sys.stdout = LoggingWrapper(self.logging_queue, prefix="STDOUT") + sys.stderr = LoggingWrapper(self.logging_queue, prefix="STDERR") + self.logging_thread.start() + + def __exit__(self, *args, **kwargs): + if self.do_capture: + sys.stdout, sys.stderr = self.original_stdio + if self.logging_queue is not None: + self.logger.info("Closing logging queue") + self.logging_queue.put(None) + if self.logging_thread is not None: + self.logging_thread.join(10) + self.logging_queue.close() + self.logger.info("queue closed") + + diff --git a/wptrunner/wptrunner.py b/wptrunner/wptrunner.py index 1d70a714a9cba3..af7043f9c403d7 100644 --- a/wptrunner/wptrunner.py +++ b/wptrunner/wptrunner.py @@ -5,30 +5,20 @@ from __future__ import unicode_literals import json -import logging import os -import shutil -import socket import sys -import threading -import time -import urlparse -from Queue import Empty -from StringIO import StringIO - -from multiprocessing import Queue - -from mozlog.structured import (commandline, stdadapter, get_default_logger, - structuredlog, handlers, formatters) +import environment as env import products import testloader import wptcommandline +import wptlogging import wpttest from testrunner import ManagerGroup here = os.path.split(__file__)[0] +logger = None """Runner for web-platform-tests @@ -46,266 +36,9 @@ metadata files are used to store the expected test results. """ -logger = None - - -def setup_logging(args, defaults): +def setup_logging(*args, **kwargs): global logger - logger = commandline.setup_logging("web-platform-tests", args, defaults) - setup_stdlib_logger() - - for name in args.keys(): - if name.startswith("log_"): - args.pop(name) - - return logger - - -def setup_stdlib_logger(): - logging.root.handlers = [] - logging.root = stdadapter.std_logging_adapter(logging.root) - - -def do_delayed_imports(serve_root): - global serve, sslutils - - sys.path.insert(0, serve_root) - - failed = [] - - try: - from tools.serve import serve - except ImportError: - failed.append("serve") - - try: - import sslutils - except ImportError: - raise - failed.append("sslutils") - - if failed: - logger.critical( - "Failed to import %s. Ensure that tests path %s contains web-platform-tests" % - (", ".join(failed), serve_root)) - sys.exit(1) - - -class TestEnvironmentError(Exception): - pass - - -class LogLevelRewriter(object): - """Filter that replaces log messages at specified levels with messages - at a different level. - - This can be used to e.g. downgrade log messages from ERROR to WARNING - in some component where ERRORs are not critical. - - :param inner: Handler to use for messages that pass this filter - :param from_levels: List of levels which should be affected - :param to_level: Log level to set for the affected messages - """ - def __init__(self, inner, from_levels, to_level): - self.inner = inner - self.from_levels = [item.upper() for item in from_levels] - self.to_level = to_level.upper() - - def __call__(self, data): - if data["action"] == "log" and data["level"].upper() in self.from_levels: - data = data.copy() - data["level"] = self.to_level - return self.inner(data) - - -class TestEnvironment(object): - def __init__(self, test_paths, ssl_env, options): - """Context manager that owns the test environment i.e. the http and - websockets servers""" - self.test_paths = test_paths - self.ssl_env = ssl_env - self.server = None - self.config = None - self.external_config = None - self.test_server_port = options.pop("test_server_port", True) - self.options = options if options is not None else {} - self.required_files = options.pop("required_files", []) - self.files_to_restore = [] - - def __enter__(self): - self.ssl_env.__enter__() - self.copy_required_files() - self.setup_server_logging() - self.setup_routes() - self.config = self.load_config() - serve.set_computed_defaults(self.config) - self.external_config, self.servers = serve.start(self.config, self.ssl_env) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.ssl_env.__exit__(exc_type, exc_val, exc_tb) - - self.restore_files() - for scheme, servers in self.servers.iteritems(): - for port, server in servers: - server.kill() - - def load_config(self): - default_config_path = os.path.join(serve_path(self.test_paths), "config.default.json") - local_config_path = os.path.join(here, "config.json") - - with open(default_config_path) as f: - default_config = json.load(f) - - with open(local_config_path) as f: - data = f.read() - local_config = json.loads(data % self.options) - - #TODO: allow non-default configuration for ssl - - local_config["external_host"] = self.options.get("external_host", None) - local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False) - - config = serve.merge_json(default_config, local_config) - config["doc_root"] = serve_path(self.test_paths) - - if not self.ssl_env.ssl_enabled: - config["ports"]["https"] = [None] - - host = self.options.get("certificate_domain", config["host"]) - hosts = [host] - hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values()) - key_file, certificate = self.ssl_env.host_cert_path(hosts) - - config["key_file"] = key_file - config["certificate"] = certificate - - return config - - def setup_server_logging(self): - server_logger = get_default_logger(component="wptserve") - assert server_logger is not None - log_filter = handlers.LogLevelFilter(lambda x:x, "info") - # Downgrade errors to warnings for the server - log_filter = LogLevelRewriter(log_filter, ["error"], "warning") - server_logger.component_filter = log_filter - - try: - #Set as the default logger for wptserve - serve.set_logger(server_logger) - serve.logger = server_logger - except Exception: - # This happens if logging has already been set up for wptserve - pass - - def setup_routes(self): - for url, paths in self.test_paths.iteritems(): - if url == "/": - continue - - path = paths["tests_path"] - url = "/%s/" % url.strip("/") - - for (method, - suffix, - handler_cls) in [(serve.any_method, - b"*.py", - serve.handlers.PythonScriptHandler), - (b"GET", - "*.asis", - serve.handlers.AsIsHandler), - (b"GET", - "*", - serve.handlers.FileHandler)]: - route = (method, b"%s%s" % (str(url), str(suffix)), handler_cls(path, url_base=url)) - serve.routes.insert(-3, route) - - if "/" not in self.test_paths: - serve.routes = serve.routes[:-3] - - def copy_required_files(self): - logger.info("Placing required files in server environment.") - for source, destination, copy_if_exists in self.required_files: - source_path = os.path.join(here, source) - dest_path = os.path.join(serve_path(self.test_paths), destination, os.path.split(source)[1]) - dest_exists = os.path.exists(dest_path) - if not dest_exists or copy_if_exists: - if dest_exists: - backup_path = dest_path + ".orig" - logger.info("Backing up %s to %s" % (dest_path, backup_path)) - self.files_to_restore.append(dest_path) - shutil.copy2(dest_path, backup_path) - logger.info("Copying %s to %s" % (source_path, dest_path)) - shutil.copy2(source_path, dest_path) - - def ensure_started(self): - # Pause for a while to ensure that the server has a chance to start - time.sleep(2) - for scheme, servers in self.servers.iteritems(): - for port, server in servers: - if self.test_server_port: - s = socket.socket() - try: - s.connect((self.config["host"], port)) - except socket.error: - raise EnvironmentError( - "%s server on port %d failed to start" % (scheme, port)) - finally: - s.close() - - if not server.is_alive(): - raise EnvironmentError("%s server on port %d failed to start" % (scheme, port)) - - def restore_files(self): - for path in self.files_to_restore: - os.unlink(path) - if os.path.exists(path + ".orig"): - os.rename(path + ".orig", path) - - -class LogThread(threading.Thread): - def __init__(self, queue, logger, level): - self.queue = queue - self.log_func = getattr(logger, level) - threading.Thread.__init__(self, name="Thread-Log") - self.daemon = True - - def run(self): - while True: - try: - msg = self.queue.get() - except (EOFError, IOError): - break - if msg is None: - break - else: - self.log_func(msg) - - -class LoggingWrapper(StringIO): - """Wrapper for file like objects to redirect output to logger - instead""" - - def __init__(self, queue, prefix=None): - self.queue = queue - self.prefix = prefix - - def write(self, data): - if isinstance(data, str): - data = data.decode("utf8") - - if data.endswith("\n"): - data = data[:-1] - if data.endswith("\r"): - data = data[:-1] - if not data: - return - if self.prefix is not None: - data = "%s: %s" % (self.prefix, data) - self.queue.put(data) - - def flush(self): - pass + logger = wptlogging.setup(*args, **kwargs) def get_loader(test_paths, product, debug=False, **kwargs): run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug) @@ -327,8 +60,7 @@ def get_loader(test_paths, product, debug=False, **kwargs): return run_info, test_loader def list_test_groups(test_paths, product, **kwargs): - - do_delayed_imports(serve_path(test_paths)) + env.do_delayed_imports(logger, test_paths) run_info, test_loader = get_loader(test_paths, product, **kwargs) @@ -338,7 +70,8 @@ def list_test_groups(test_paths, product, **kwargs): def list_disabled(test_paths, product, **kwargs): - do_delayed_imports(serve_path(test_paths)) + env.do_delayed_imports(logger, test_paths) + rv = [] run_info, test_loader = get_loader(test_paths, product, @@ -350,47 +83,27 @@ def list_disabled(test_paths, product, **kwargs): print json.dumps(rv, indent=2) -def get_ssl_kwargs(**kwargs): - if kwargs["ssl_type"] == "openssl": - args = {"openssl_binary": kwargs["openssl_binary"]} - elif kwargs["ssl_type"] == "pregenerated": - args = {"host_key_path": kwargs["host_key_path"], - "host_cert_path": kwargs["host_cert_path"], - "ca_cert_path": kwargs["ca_cert_path"]} - else: - args = {} - return args +def get_pause_after_test(test_loader, **kwargs): + total_tests = sum(len(item) for item in test_loader.tests.itervalues()) + if kwargs["pause_after_test"] is None: + if kwargs["repeat"] == 1 and total_tests == 1: + return True + return False + return kwargs["pause_after_test"] -def serve_path(test_paths): - return test_paths["/"]["tests_path"] def run_tests(config, test_paths, product, **kwargs): - logging_queue = None - logging_thread = None - original_stdio = (sys.stdout, sys.stderr) - test_queues = None - - try: - if not kwargs["no_capture_stdio"]: - logging_queue = Queue() - logging_thread = LogThread(logging_queue, logger, "info") - sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT") - sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR") - logging_thread.start() - - do_delayed_imports(serve_path(test_paths)) + with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): + env.do_delayed_imports(logger, test_paths) (check_args, browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options) = products.load_product(config, product) - check_args(**kwargs) - - ssl_env_cls = sslutils.environments[kwargs["ssl_type"]] - ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs)) + ssl_env = env.ssl_env(logger, **kwargs) - unexpected_total = 0 + check_args(**kwargs) if "test_loader" in kwargs: run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) @@ -409,12 +122,14 @@ def run_tests(config, test_paths, product, **kwargs): logger.info("Using %i client processes" % kwargs["processes"]) - with TestEnvironment(test_paths, - ssl_env, - env_options) as test_environment: + unexpected_total = 0 + + with env.TestEnvironment(test_paths, + ssl_env, + env_options) as test_environment: try: test_environment.ensure_started() - except TestEnvironmentError as e: + except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise @@ -469,22 +184,6 @@ def run_tests(config, test_paths, product, **kwargs): unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) logger.suite_end() - except KeyboardInterrupt: - if test_queues is not None: - for queue in test_queues.itervalues(): - queue.cancel_join_thread() - finally: - if test_queues is not None: - for queue in test_queues.itervalues(): - queue.close() - sys.stdout, sys.stderr = original_stdio - if not kwargs["no_capture_stdio"] and logging_queue is not None: - logger.info("Closing logging queue") - logging_queue.put(None) - if logging_thread is not None: - logging_thread.join(10) - logging_queue.close() - logger.info("queue closed") return unexpected_total == 0 From 7dc6d8799d4b70ea2d660594957c76da8a4a0ff6 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 25 Feb 2015 15:46:24 +0000 Subject: [PATCH 2/3] Add ability to pause after running tests. By default the browser will pause if it is given a single test to run. This behaviour can be changed through the --[no-]pause-after-test command line options. When set up to pause after tests, the testharness.js output will also be displayed. --- wptrunner/browsers/b2g.py | 5 +-- wptrunner/browsers/chrome.py | 6 +-- wptrunner/browsers/firefox.py | 3 +- wptrunner/environment.py | 51 +++++++++++------------ wptrunner/executors/executormarionette.py | 4 -- wptrunner/executors/executorselenium.py | 3 -- wptrunner/testharnessreport.js | 4 +- wptrunner/testrunner.py | 16 ++++--- wptrunner/wptcommandline.py | 6 +++ wptrunner/wptrunner.py | 4 ++ 10 files changed, 53 insertions(+), 49 deletions(-) diff --git a/wptrunner/browsers/b2g.py b/wptrunner/browsers/b2g.py index 89bb42fccc109f..6473f888ea7331 100644 --- a/wptrunner/browsers/b2g.py +++ b/wptrunner/browsers/b2g.py @@ -18,7 +18,7 @@ from mozprofile import FirefoxProfile, Preferences from .base import get_free_port, BrowserError, Browser, ExecutorBrowser -from ..executors.executormarionette import MarionetteTestharnessExecutor, required_files +from ..executors.executormarionette import MarionetteTestharnessExecutor from ..hosts import HostsFile, HostsLine here = os.path.split(__file__)[0] @@ -55,8 +55,7 @@ def executor_kwargs(http_server_url, **kwargs): def env_options(): return {"host": "web-platform.test", "bind_hostname": "false", - "test_server_port": False, - "required_files": required_files} + "test_server_port": False} class B2GBrowser(Browser): diff --git a/wptrunner/browsers/chrome.py b/wptrunner/browsers/chrome.py index 5bf199713e5e86..92c58e3cb919ef 100644 --- a/wptrunner/browsers/chrome.py +++ b/wptrunner/browsers/chrome.py @@ -6,8 +6,7 @@ from .webdriver import ChromedriverLocalServer from ..executors import executor_kwargs as base_executor_kwargs from ..executors.executorselenium import (SeleniumTestharnessExecutor, - SeleniumRefTestExecutor, - required_files) + SeleniumRefTestExecutor) __wptrunner__ = {"product": "chrome", @@ -43,8 +42,7 @@ def executor_kwargs(test_type, http_server_url, **kwargs): def env_options(): return {"host": "web-platform.test", - "bind_hostname": "true", - "required_files": required_files} + "bind_hostname": "true"} class ChromeBrowser(Browser): diff --git a/wptrunner/browsers/firefox.py b/wptrunner/browsers/firefox.py index f0ce10dcdd544e..a11df67508a7c4 100644 --- a/wptrunner/browsers/firefox.py +++ b/wptrunner/browsers/firefox.py @@ -14,7 +14,7 @@ from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg from ..executors import executor_kwargs as base_executor_kwargs -from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor, required_files +from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor here = os.path.join(os.path.split(__file__)[0]) @@ -55,7 +55,6 @@ def env_options(): return {"host": "127.0.0.1", "external_host": "web-platform.test", "bind_hostname": "false", - "required_files": required_files, "certificate_domain": "web-platform.test", "encrypt_after_connect": True} diff --git a/wptrunner/environment.py b/wptrunner/environment.py index 2f48d177b87bb4..de199984cc977a 100644 --- a/wptrunner/environment.py +++ b/wptrunner/environment.py @@ -4,7 +4,6 @@ import json import os -import shutil import socket import sys import time @@ -70,8 +69,23 @@ class TestEnvironmentError(Exception): pass +def static_handler(path, format_args, content_type, **headers): + with open(path) as f: + data = f.read() % format_args + + resp_headers = [("Content-Type", content_type)] + for k, v in headers.iteritems(): + resp_headers.append((k.replace("_", "-"), v)) + + @serve.handlers.handler + def func(request, response): + return resp_headers, data + + return func + + class TestEnvironment(object): - def __init__(self, test_paths, ssl_env, options): + def __init__(self, test_paths, ssl_env, pause_after_test, options): """Context manager that owns the test environment i.e. the http and websockets servers""" self.test_paths = test_paths @@ -79,14 +93,12 @@ def __init__(self, test_paths, ssl_env, options): self.server = None self.config = None self.external_config = None + self.pause_after_test = pause_after_test self.test_server_port = options.pop("test_server_port", True) self.options = options if options is not None else {} - self.required_files = options.pop("required_files", []) - self.files_to_restore = [] def __enter__(self): self.ssl_env.__enter__() - self.copy_required_files() self.setup_server_logging() self.setup_routes() self.config = self.load_config() @@ -97,7 +109,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.ssl_env.__exit__(exc_type, exc_val, exc_tb) - self.restore_files() for scheme, servers in self.servers.iteritems(): for port, server in servers: server.kill() @@ -151,6 +162,13 @@ def setup_server_logging(self): pass def setup_routes(self): + for path, format_args, content_type, route in [ + ("testharness_runner.html", {}, "text/html", b"/testharness_runner.html"), + ("testharnessreport.js", {"output": self.pause_after_test}, + "text/javascript", b"/resources/testharnessreport.js")]: + handler = static_handler(os.path.join(here, path), format_args, content_type) + serve.routes.insert(0, (b"GET", route, handler)) + for url, paths in self.test_paths.iteritems(): if url == "/": continue @@ -175,21 +193,6 @@ def setup_routes(self): if "/" not in self.test_paths: serve.routes = serve.routes[:-3] - def copy_required_files(self): - logger.info("Placing required files in server environment.") - for source, destination, copy_if_exists in self.required_files: - source_path = os.path.join(here, source) - dest_path = os.path.join(serve_path(self.test_paths), destination, os.path.split(source)[1]) - dest_exists = os.path.exists(dest_path) - if not dest_exists or copy_if_exists: - if dest_exists: - backup_path = dest_path + ".orig" - logger.info("Backing up %s to %s" % (dest_path, backup_path)) - self.files_to_restore.append(dest_path) - shutil.copy2(dest_path, backup_path) - logger.info("Copying %s to %s" % (source_path, dest_path)) - shutil.copy2(source_path, dest_path) - def ensure_started(self): # Pause for a while to ensure that the server has a chance to start time.sleep(2) @@ -207,9 +210,3 @@ def ensure_started(self): if not server.is_alive(): raise EnvironmentError("%s server on port %d failed to start" % (scheme, port)) - - def restore_files(self): - for path in self.files_to_restore: - os.unlink(path) - if os.path.exists(path + ".orig"): - os.rename(path + ".orig", path) diff --git a/wptrunner/executors/executormarionette.py b/wptrunner/executors/executormarionette.py index 5d21129c6953ca..29fbea908c4cdf 100644 --- a/wptrunner/executors/executormarionette.py +++ b/wptrunner/executors/executormarionette.py @@ -31,10 +31,6 @@ # should force a timeout extra_timeout = 5 # seconds -required_files = [("testharness_runner.html", "", False), - ("testharnessreport.js", "resources/", True)] - - def do_delayed_imports(): global marionette global errors diff --git a/wptrunner/executors/executorselenium.py b/wptrunner/executors/executorselenium.py index 72fcc5797d96df..7cf0b74c8a4301 100644 --- a/wptrunner/executors/executorselenium.py +++ b/wptrunner/executors/executorselenium.py @@ -27,9 +27,6 @@ webdriver = None exceptions = None -required_files = [("testharness_runner.html", "", False), - ("testharnessreport.js", "resources/", True)] - extra_timeout = 5 def do_delayed_imports(): diff --git a/wptrunner/testharnessreport.js b/wptrunner/testharnessreport.js index 046e8a5fe55e00..f27f25a58ba01a 100644 --- a/wptrunner/testharnessreport.js +++ b/wptrunner/testharnessreport.js @@ -2,11 +2,13 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -var props = {output:false, +var props = {output:%(output)d, explicit_timeout: true}; + if (window.opener && "timeout_multiplier" in window.opener) { props["timeout_multiplier"] = window.opener.timeout_multiplier; } + if (window.opener && window.opener.explicit_timeout) { props["explicit_timeout"] = window.opener.explicit_timeout; } diff --git a/wptrunner/testrunner.py b/wptrunner/testrunner.py index 9c8980e0f01724..d86539196b384b 100644 --- a/wptrunner/testrunner.py +++ b/wptrunner/testrunner.py @@ -162,8 +162,8 @@ class TestRunnerManager(threading.Thread): init_lock = threading.Lock() def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs, - executor_cls, executor_kwargs, stop_flag, pause_on_unexpected=False, - debug_args=None): + executor_cls, executor_kwargs, stop_flag, pause_after_test=False, + pause_on_unexpected=False, debug_args=None): """Thread that owns a single TestRunner process and any processes required by the TestRunner (e.g. the Firefox binary). @@ -199,6 +199,7 @@ def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser self.parent_stop_flag = stop_flag self.child_stop_flag = multiprocessing.Event() + self.pause_after_test = pause_after_test self.pause_on_unexpected = pause_on_unexpected self.debug_args = debug_args @@ -513,8 +514,9 @@ def test_ended(self, test, results): self.test = None - if self.pause_on_unexpected and (subtest_unexpected or is_unexpected): - self.logger.info("Got an unexpected result, pausing until the browser exits") + if (self.pause_after_test or + (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))): + self.logger.info("Pausing until the browser exits") self.browser.runner.process_handler.wait() # Handle starting the next test, with a runner restart if required @@ -573,7 +575,9 @@ def __exit__(self, *args, **kwargs): class ManagerGroup(object): def __init__(self, suite_name, size, test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, - executor_cls, executor_kwargs, pause_on_unexpected=False, + executor_cls, executor_kwargs, + pause_after_test=False, + pause_on_unexpected=False, debug_args=None): """Main thread object that owns all the TestManager threads.""" self.suite_name = suite_name @@ -584,6 +588,7 @@ def __init__(self, suite_name, size, test_source_cls, test_source_kwargs, self.browser_kwargs = browser_kwargs self.executor_cls = executor_cls self.executor_kwargs = executor_kwargs + self.pause_after_test = pause_after_test self.pause_on_unexpected = pause_on_unexpected self.debug_args = debug_args @@ -621,6 +626,7 @@ def run(self, test_type, tests): self.executor_cls, self.executor_kwargs, self.stop_flag, + self.pause_after_test, self.pause_on_unexpected, self.debug_args) manager.start() diff --git a/wptrunner/wptcommandline.py b/wptrunner/wptcommandline.py index 3fe076e57ea100..59da40e3a54806 100644 --- a/wptrunner/wptcommandline.py +++ b/wptrunner/wptcommandline.py @@ -108,6 +108,12 @@ def create_parser(product_choices=None): debugging_group.add_argument('--debugger', help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") + + debugging_group.add_argument('--pause-after-test', action="store_true", default=None, + help="Halt the test runner after each test (this happens by default if only a single test is run)") + debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", + help="Don't halt the test runner irrespective of the number of tests run") + debugging_group.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") diff --git a/wptrunner/wptrunner.py b/wptrunner/wptrunner.py index af7043f9c403d7..b94bab2321f53c 100644 --- a/wptrunner/wptrunner.py +++ b/wptrunner/wptrunner.py @@ -124,8 +124,11 @@ def run_tests(config, test_paths, product, **kwargs): unexpected_total = 0 + pause_after_test = get_pause_after_test(test_loader, **kwargs) + with env.TestEnvironment(test_paths, ssl_env, + pause_after_test, env_options) as test_environment: try: test_environment.ensure_started() @@ -171,6 +174,7 @@ def run_tests(config, test_paths, product, **kwargs): browser_kwargs, executor_cls, executor_kwargs, + pause_after_test, kwargs["pause_on_unexpected"], kwargs["debug_args"]) as manager_group: try: From 215a4d30d64a2a88ce24633ed963e8aed374c820 Mon Sep 17 00:00:00 2001 From: James Graham Date: Thu, 26 Feb 2015 18:00:27 +0000 Subject: [PATCH 3/3] fixup! Add ability to pause after running tests. --- wptrunner/wptlogging.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wptrunner/wptlogging.py b/wptrunner/wptlogging.py index d9bfed55352615..3058a58aaebca0 100644 --- a/wptrunner/wptlogging.py +++ b/wptrunner/wptlogging.py @@ -121,5 +121,3 @@ def __exit__(self, *args, **kwargs): self.logging_thread.join(10) self.logging_queue.close() self.logger.info("queue closed") - -