Merge pull request #76 from w3c/jgraham/pause_after_test

Add ability to pause after running tests

test/test.py

@@ -18,7 +18,7 @@
 def setup_wptrunner_logging(logger):
     structuredlog.set_default_logger(logger)
     wptrunner.logger = logger
-    wptrunner.setup_stdlib_logger()
+    wptrunner.wptlogging.setup_stdlib_logger()
 
 class ResultHandler(BaseHandler):
     def __init__(self, verbose=False, logger=None):
@@ -155,7 +155,8 @@ def main():
             import pdb, traceback
             print traceback.format_exc()
             pdb.post_mortem()
-
+        else:
+            raise
 
 if __name__ == "__main__":
     main()

wptrunner/browsers/b2g.py

@@ -18,7 +18,7 @@
 from mozprofile import FirefoxProfile, Preferences
 
 from .base import get_free_port, BrowserError, Browser, ExecutorBrowser
-from ..executors.executormarionette import MarionetteTestharnessExecutor, required_files
+from ..executors.executormarionette import MarionetteTestharnessExecutor
 from ..hosts import HostsFile, HostsLine
 
 here = os.path.split(__file__)[0]
@@ -55,8 +55,7 @@ def executor_kwargs(http_server_url, **kwargs):
 def env_options():
     return {"host": "web-platform.test",
             "bind_hostname": "false",
-            "test_server_port": False,
-            "required_files": required_files}
+            "test_server_port": False}
 
 
 class B2GBrowser(Browser):

wptrunner/browsers/chrome.py

@@ -6,8 +6,7 @@
 from .webdriver import ChromedriverLocalServer
 from ..executors import executor_kwargs as base_executor_kwargs
 from ..executors.executorselenium import (SeleniumTestharnessExecutor,
-                                          SeleniumRefTestExecutor,
-                                          required_files)
+                                          SeleniumRefTestExecutor)
 
 
 __wptrunner__ = {"product": "chrome",
@@ -43,8 +42,7 @@ def executor_kwargs(test_type, http_server_url, **kwargs):
 
 def env_options():
     return {"host": "web-platform.test",
-            "bind_hostname": "true",
-            "required_files": required_files}
+            "bind_hostname": "true"}
 
 
 class ChromeBrowser(Browser):

wptrunner/browsers/firefox.py

@@ -14,7 +14,7 @@
 
 from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg
 from ..executors import executor_kwargs as base_executor_kwargs
-from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor, required_files
+from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor
 
 here = os.path.join(os.path.split(__file__)[0])
 
@@ -55,7 +55,6 @@ def env_options():
     return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
             "bind_hostname": "false",
-            "required_files": required_files,
             "certificate_domain": "web-platform.test",
             "encrypt_after_connect": True}
 

wptrunner/environment.py

@@ -0,0 +1,212 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import socket
+import sys
+import time
+
+from mozlog.structured import get_default_logger, handlers
+
+from wptlogging import LogLevelRewriter
+
+here = os.path.split(__file__)[0]
+
+serve = None
+sslutils = None
+
+def do_delayed_imports(logger, test_paths):
+    global serve, sslutils
+
+    serve_root = serve_path(test_paths)
+
+    sys.path.insert(0, serve_root)
+
+    failed = []
+
+    try:
+        from tools.serve import serve
+    except ImportError:
+        failed.append("serve")
+
+    try:
+        import sslutils
+    except ImportError:
+        raise
+        failed.append("sslutils")
+
+    if failed:
+        logger.critical(
+            "Failed to import %s. Ensure that tests path %s contains web-platform-tests" %
+            (", ".join(failed), serve_root))
+        sys.exit(1)
+
+
+def serve_path(test_paths):
+    return test_paths["/"]["tests_path"]
+
+
+def get_ssl_kwargs(**kwargs):
+    if kwargs["ssl_type"] == "openssl":
+        args = {"openssl_binary": kwargs["openssl_binary"]}
+    elif kwargs["ssl_type"] == "pregenerated":
+        args = {"host_key_path": kwargs["host_key_path"],
+                "host_cert_path": kwargs["host_cert_path"],
+                 "ca_cert_path": kwargs["ca_cert_path"]}
+    else:
+        args = {}
+    return args
+
+
+def ssl_env(logger, **kwargs):
+    ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
+    return ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
+
+
+class TestEnvironmentError(Exception):
+    pass
+
+
+def static_handler(path, format_args, content_type, **headers):
+    with open(path) as f:
+        data = f.read() % format_args
+
+    resp_headers = [("Content-Type", content_type)]
+    for k, v in headers.iteritems():
+        resp_headers.append((k.replace("_", "-"), v))
+
+    @serve.handlers.handler
+    def func(request, response):
+        return resp_headers, data
+
+    return func
+
+
+class TestEnvironment(object):
+    def __init__(self, test_paths, ssl_env, pause_after_test, options):
+        """Context manager that owns the test environment i.e. the http and
+        websockets servers"""
+        self.test_paths = test_paths
+        self.ssl_env = ssl_env
+        self.server = None
+        self.config = None
+        self.external_config = None
+        self.pause_after_test = pause_after_test
+        self.test_server_port = options.pop("test_server_port", True)
+        self.options = options if options is not None else {}
+
+    def __enter__(self):
+        self.ssl_env.__enter__()
+        self.setup_server_logging()
+        self.setup_routes()
+        self.config = self.load_config()
+        serve.set_computed_defaults(self.config)
+        self.external_config, self.servers = serve.start(self.config, self.ssl_env)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
+
+        for scheme, servers in self.servers.iteritems():
+            for port, server in servers:
+                server.kill()
+
+    def load_config(self):
+        default_config_path = os.path.join(serve_path(self.test_paths), "config.default.json")
+        local_config_path = os.path.join(here, "config.json")
+
+        with open(default_config_path) as f:
+            default_config = json.load(f)
+
+        with open(local_config_path) as f:
+            data = f.read()
+            local_config = json.loads(data % self.options)
+
+        #TODO: allow non-default configuration for ssl
+
+        local_config["external_host"] = self.options.get("external_host", None)
+        local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False)
+
+        config = serve.merge_json(default_config, local_config)
+        config["doc_root"] = serve_path(self.test_paths)
+
+        if not self.ssl_env.ssl_enabled:
+            config["ports"]["https"] = [None]
+
+        host = self.options.get("certificate_domain", config["host"])
+        hosts = [host]
+        hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values())
+        key_file, certificate = self.ssl_env.host_cert_path(hosts)
+
+        config["key_file"] = key_file
+        config["certificate"] = certificate
+
+        return config
+
+    def setup_server_logging(self):
+        server_logger = get_default_logger(component="wptserve")
+        assert server_logger is not None
+        log_filter = handlers.LogLevelFilter(lambda x:x, "info")
+        # Downgrade errors to warnings for the server
+        log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
+        server_logger.component_filter = log_filter
+
+        try:
+            #Set as the default logger for wptserve
+            serve.set_logger(server_logger)
+            serve.logger = server_logger
+        except Exception:
+            # This happens if logging has already been set up for wptserve
+            pass
+
+    def setup_routes(self):
+        for path, format_args, content_type, route in [
+                ("testharness_runner.html", {}, "text/html", b"/testharness_runner.html"),
+                ("testharnessreport.js", {"output": self.pause_after_test},
+                 "text/javascript", b"/resources/testharnessreport.js")]:
+            handler = static_handler(os.path.join(here, path), format_args, content_type)
+            serve.routes.insert(0, (b"GET", route, handler))
+
+        for url, paths in self.test_paths.iteritems():
+            if url == "/":
+                continue
+
+            path = paths["tests_path"]
+            url = "/%s/" % url.strip("/")
+
+            for (method,
+                 suffix,
+                 handler_cls) in [(serve.any_method,
+                                   b"*.py",
+                                   serve.handlers.PythonScriptHandler),
+                                  (b"GET",
+                                   "*.asis",
+                                   serve.handlers.AsIsHandler),
+                                  (b"GET",
+                                   "*",
+                                   serve.handlers.FileHandler)]:
+                route = (method, b"%s%s" % (str(url), str(suffix)), handler_cls(path, url_base=url))
+                serve.routes.insert(-3, route)
+
+        if "/" not in self.test_paths:
+            serve.routes = serve.routes[:-3]
+
+    def ensure_started(self):
+        # Pause for a while to ensure that the server has a chance to start
+        time.sleep(2)
+        for scheme, servers in self.servers.iteritems():
+            for port, server in servers:
+                if self.test_server_port:
+                    s = socket.socket()
+                    try:
+                        s.connect((self.config["host"], port))
+                    except socket.error:
+                        raise EnvironmentError(
+                            "%s server on port %d failed to start" % (scheme, port))
+                    finally:
+                        s.close()
+
+                if not server.is_alive():
+                    raise EnvironmentError("%s server on port %d failed to start" % (scheme, port))

wptrunner/executors/executormarionette.py

@@ -31,10 +31,6 @@
 # should force a timeout
 extra_timeout = 5 # seconds
 
-required_files = [("testharness_runner.html", "", False),
-                  ("testharnessreport.js", "resources/", True)]
-
-
 def do_delayed_imports():
     global marionette
     global errors

wptrunner/executors/executorselenium.py

@@ -27,9 +27,6 @@
 webdriver = None
 exceptions = None
 
-required_files = [("testharness_runner.html", "", False),
-                  ("testharnessreport.js", "resources/", True)]
-
 extra_timeout = 5
 
 def do_delayed_imports():

wptrunner/testharnessreport.js

@@ -2,11 +2,13 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-var props = {output:false,
+var props = {output:%(output)d,
              explicit_timeout: true};
+
 if (window.opener && "timeout_multiplier" in window.opener) {
     props["timeout_multiplier"] = window.opener.timeout_multiplier;
 }
+
 if (window.opener && window.opener.explicit_timeout) {
     props["explicit_timeout"] = window.opener.explicit_timeout;
 }

wptrunner/testrunner.py

@@ -162,8 +162,8 @@ class TestRunnerManager(threading.Thread):
     init_lock = threading.Lock()
 
     def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
-                 executor_cls, executor_kwargs, stop_flag, pause_on_unexpected=False,
-                 debug_args=None):
+                 executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
+                 pause_on_unexpected=False, debug_args=None):
         """Thread that owns a single TestRunner process and any processes required
         by the TestRunner (e.g. the Firefox binary).
 
@@ -199,6 +199,7 @@ def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser
         self.parent_stop_flag = stop_flag
         self.child_stop_flag = multiprocessing.Event()
 
+        self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.debug_args = debug_args
 
@@ -513,8 +514,9 @@ def test_ended(self, test, results):
 
         self.test = None
 
-        if self.pause_on_unexpected and (subtest_unexpected or is_unexpected):
-            self.logger.info("Got an unexpected result, pausing until the browser exits")
+        if (self.pause_after_test or
+            (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
+            self.logger.info("Pausing until the browser exits")
             self.browser.runner.process_handler.wait()
 
         # Handle starting the next test, with a runner restart if required
@@ -573,7 +575,9 @@ def __exit__(self, *args, **kwargs):
 class ManagerGroup(object):
     def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
                  browser_cls, browser_kwargs,
-                 executor_cls, executor_kwargs, pause_on_unexpected=False,
+                 executor_cls, executor_kwargs,
+                 pause_after_test=False,
+                 pause_on_unexpected=False,
                  debug_args=None):
         """Main thread object that owns all the TestManager threads."""
         self.suite_name = suite_name
@@ -584,6 +588,7 @@ def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
         self.browser_kwargs = browser_kwargs
         self.executor_cls = executor_cls
         self.executor_kwargs = executor_kwargs
+        self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.debug_args = debug_args
 
@@ -621,6 +626,7 @@ def run(self, test_type, tests):
                                             self.executor_cls,
                                             self.executor_kwargs,
                                             self.stop_flag,
+                                            self.pause_after_test,
                                             self.pause_on_unexpected,
                                             self.debug_args)
                 manager.start()

wptrunner/wptcommandline.py

@@ -108,6 +108,12 @@ def create_parser(product_choices=None):
     debugging_group.add_argument('--debugger',
                                  help="run under a debugger, e.g. gdb or valgrind")
     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
+
+    debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
+                                 help="Halt the test runner after each test (this happens by default if only a single test is run)")
+    debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
+                                 help="Don't halt the test runner irrespective of the number of tests run")
+
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
                                  help="Halt the test runner when an unexpected result is encountered")