Merge pull request #73 from w3c/jgraham/reftest_changes_2

Implement new reftest semantics

.gitignore

@@ -2,4 +2,6 @@
 *~
 *#
 \#*
-_virtualenv
+_virtualenv
+test/test.cfg
+test/metadata/MANIFEST.json

test/metadata/reftest/reftest_and_fail.html.ini

@@ -0,0 +1,3 @@
+[reftest_and_fail.html]
+  type: reftest
+  expected: FAIL

test/metadata/reftest/reftest_cycle_fail.html.ini

@@ -0,0 +1,3 @@
+[reftest_cycle_fail.html]
+  type: reftest
+  expected: FAIL

test/metadata/reftest/reftest_match_fail.html.ini

@@ -0,0 +1,3 @@
+[reftest_match_fail.html]
+  type: reftest
+  expected: FAIL

test/metadata/reftest/reftest_mismatch_fail.html.ini

@@ -0,0 +1,3 @@
+[reftest_mismatch_fail.html]
+  type: reftest
+  expected: FAIL

test/metadata/reftest/reftest_ref_timeout.html.ini

@@ -0,0 +1,3 @@
+[reftest_ref_timeout.html]
+  type: reftest
+  expected: TIMEOUT

test/metadata/reftest/reftest_timeout.html.ini

@@ -0,0 +1,3 @@
+[reftest_timeout.html]
+  type: reftest
+  expected: TIMEOUT

test/metadata/testharness/testharness_0.html.ini

@@ -0,0 +1,4 @@
+[testharness_0.html]
+  type: testharness
+  [Test that should fail]
+    expected: FAIL

test/metadata/testharness/testharness_error.html.ini

@@ -0,0 +1,3 @@
+[testharness_error.html]
+  type: testharness
+  expected: ERROR

test/metadata/testharness/testharness_timeout.html.ini

@@ -0,0 +1,3 @@
+[testharness_timeout.html]
+  type: testharness
+  expected: TIMEOUT

test/test.cfg.example

@@ -0,0 +1,16 @@
+[general]
+tests=/path/to/web-platform-tests/
+metadata=/path/to/web-platform-tests/
+ssl-type=none
+
+# [firefox]
+# binary=/path/to/firefox
+# prefs-root=/path/to/gecko-src/testing/profiles/
+
+# [servo]
+# binary=/path/to/servo-src/components/servo/target/servo
+# exclude=testharness # Because it needs a special testharness.js
+
+# [chrome]
+# binary=/path/to/chrome
+# webdriver-binary=/path/to/chromedriver

test/test.py

@@ -0,0 +1,161 @@
+import ConfigParser
+import argparse
+import json
+import os
+import sys
+import tempfile
+import threading
+import time
+from StringIO import StringIO
+
+from mozlog.structured import structuredlog, reader
+from mozlog.structured.handlers import BaseHandler, StreamHandler, StatusHandler
+from mozlog.structured.formatters import MachFormatter
+from wptrunner import wptcommandline, wptrunner
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+def setup_wptrunner_logging(logger):
+    structuredlog.set_default_logger(logger)
+    wptrunner.logger = logger
+    wptrunner.setup_stdlib_logger()
+
+class ResultHandler(BaseHandler):
+    def __init__(self, verbose=False, logger=None):
+        self.inner = StreamHandler(sys.stdout, MachFormatter())
+        BaseHandler.__init__(self, self.inner)
+        self.product = None
+        self.verbose = verbose
+        self.logger = logger
+
+        self.register_message_handlers("wptrunner-test", {"set-product": self.set_product})
+
+    def set_product(self, product):
+        self.product = product
+
+    def __call__(self, data):
+        if self.product is not None and data["action"] in ["suite_start", "suite_end"]:
+            # Hack: mozlog sets some internal state to prevent multiple suite_start or
+            # suite_end messages. We actually want that here (one from the metaharness
+            # and one from the individual test type harness), so override that internal
+            # state (a better solution might be to not share loggers, but this works well
+            # enough)
+            self.logger._state.suite_started = True
+            return
+
+        if (not self.verbose and
+            (data["action"] == "process_output" or
+             data["action"] == "log" and data["level"] not in ["error", "critical"])):
+            return
+
+        if "test" in data:
+            data = data.copy()
+            data["test"] = "%s: %s" % (self.product, data["test"])
+
+        return self.inner(data)
+
+def test_settings():
+    return {
+        "include": "_test",
+        "manifest-update": "",
+        "no-capture-stdio": ""
+    }
+
+def read_config():
+    parser = ConfigParser.ConfigParser()
+    parser.read("test.cfg")
+
+    rv = {"general":{},
+          "products":{}}
+
+    rv["general"].update(dict(parser.items("general")))
+
+    # This only allows one product per whatever for now
+    for product in parser.sections():
+        if product != "general":
+            dest = rv["products"][product] = {}
+            for key, value in parser.items(product):
+                rv["products"][product][key] = value
+
+    return rv
+
+def run_tests(product, kwargs):
+    kwargs["test_paths"]["/_test/"] = {"tests_path": os.path.join(here, "testdata"),
+                                       "metadata_path": os.path.join(here, "metadata")}
+
+    wptrunner.run_tests(**kwargs)
+
+def settings_to_argv(settings):
+    rv = []
+    for name, value in settings.iteritems():
+        key = "--%s" % name
+        if not value:
+            rv.append(key)
+        elif isinstance(value, list):
+            for item in value:
+                rv.extend([key, item])
+        else:
+            rv.extend([key, value])
+    return rv
+
+def set_from_args(settings, args):
+    if args.test:
+        settings["include"] = args.test
+
+def run(config, args):
+    logger = structuredlog.StructuredLogger("web-platform-tests")
+    logger.add_handler(ResultHandler(logger=logger, verbose=args.verbose))
+    setup_wptrunner_logging(logger)
+
+    parser = wptcommandline.create_parser()
+
+    logger.suite_start(tests=[])
+
+    for product, product_settings in config["products"].iteritems():
+        if args.product and product not in args.product:
+            continue
+
+        settings = test_settings()
+        settings.update(config["general"])
+        settings.update(product_settings)
+        settings["product"] = product
+        set_from_args(settings, args)
+
+        kwargs = vars(parser.parse_args(settings_to_argv(settings)))
+        wptcommandline.check_args(kwargs)
+
+        logger.send_message("wptrunner-test", "set-product", product)
+
+        run_tests(product, kwargs)
+
+    logger.send_message("wptrunner-test", "set-product", None)
+    logger.suite_end()
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="verbose log output")
+    parser.add_argument("--product", action="append",
+                        help="Specific product to include in test run")
+    parser.add_argument("--pdb", action="store_true",
+                        help="Invoke pdb on uncaught exception")
+    parser.add_argument("test", nargs="*", type=wptcommandline.slash_prefixed,
+                        help="Specific tests to include in test run")
+    return parser
+
+def main():
+    config = read_config()
+
+    args = get_parser().parse_args()
+
+    try:
+        run(config, args)
+    except Exception:
+        if args.pdb:
+            import pdb, traceback
+            print traceback.format_exc()
+            pdb.post_mortem()
+
+
+if __name__ == "__main__":
+    main()

test/testdata/reftest/green-ref.html

@@ -0,0 +1,4 @@
+<link rel=match href=green.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/green.html

@@ -0,0 +1,3 @@
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/red.html

@@ -0,0 +1,3 @@
+<style>
+:root {background-color:red}
+</style>

test/testdata/reftest/reftest_and_fail.html

@@ -0,0 +1,5 @@
+<title>Reftest chain that should fail</title>
+<link rel=match href=reftest_and_fail_0-ref.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_and_fail_0-ref.html

@@ -0,0 +1,5 @@
+<title>Reftest chain that should fail</title>
+<link rel=match href=red.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_cycle.html

@@ -0,0 +1,5 @@
+<title>Reftest with cycle, all match</title>
+<link rel=match href=reftest_cycle_0-ref.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_cycle_0-ref.html

@@ -0,0 +1,5 @@
+<title>OR match that should pass</title>
+<link rel=match href=reftest_cycle_1-ref.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_cycle_1-ref.html

@@ -0,0 +1,5 @@
+<title>Reftest with cycle, all match</title>
+<link rel=match href=reftest_cycle.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_cycle_fail.html

@@ -0,0 +1,5 @@
+<title>Reftest with cycle, fails</title>
+<link rel=match href=reftest_cycle_fail_0-ref.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_cycle_fail_0-ref.html

@@ -0,0 +1,5 @@
+<title>Reftest with cycle, fails</title>
+<link rel=mismatch href=reftest_cycle_fail.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_match.html

@@ -0,0 +1,5 @@
+<title>rel=match that should pass</title>
+<link rel=match href=green.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_match_fail.html

@@ -0,0 +1,5 @@
+<title>rel=match that should fail</title>
+<link rel=match href=red.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_mismatch.html

@@ -0,0 +1,5 @@
+<title>rel=mismatch that should pass</title>
+<link rel=mismatch href=red.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_mismatch_fail.html

@@ -0,0 +1,5 @@
+<title>rel=mismatch that should fail</title>
+<link rel=mismatch href=green.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_or_0.html

@@ -0,0 +1,6 @@
+<title>OR match that should pass</title>
+<link rel=match href=red.html>
+<link rel=match href=green.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_ref_timeout-ref.html

@@ -0,0 +1,6 @@
+<html class="reftest-wait">
+<title>rel=match that should time out in the ref</title>
+<link rel=match href=reftest_ref_timeout-ref.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_ref_timeout.html

@@ -0,0 +1,6 @@
+<html>
+<title>rel=match that should time out in the ref</title>
+<link rel=match href=reftest_ref_timeout-ref.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_timeout.html

@@ -0,0 +1,6 @@
+<html class="reftest-wait">
+<title>rel=match that should timeout</title>
+<link rel=match href=green.html>
+<style>
+:root {background-color:green}
+</style>

test/testdata/reftest/reftest_wait_0.html

@@ -0,0 +1,11 @@
+<title>rel=match that should fail</title>
+<link rel=match href=red.html>
+<style>
+:root {background-color:red}
+</style>
+<body class="reftest-wait">
+<script>
+setTimeout(function() {
+  document.documentElement.style.backgroundColor = "green";
+  body.className = "";
+}, 2000);

test/testdata/testharness/testharness_0.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<title>Simple testharness.js usage</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+test(function() {
+  assert_true(true);
+}, "Test that should pass");
+
+test(function() {
+  assert_true(false);
+}, "Test that should fail");
+</script>

test/testdata/testharness/testharness_error.html

@@ -0,0 +1,7 @@
+<!doctype html>
+<title>testharness.js test that should error</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+undefined_function()
+</script>

test/testdata/testharness/testharness_long_timeout.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<title>testharness.js test with long timeout</title>
+<meta name=timeout content=long>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+var t = async_test("Long timeout test");
+setTimeout(t.step_func_done(function() {assert_true(true)}), 15*1000);
+</script>