Merge pull request #108 from w3c/jgraham/metadata_improvements

Add support for inheriting metadata from directory files, and for --tags for test selection
web-platform-tests · Jun 3, 2015 · 6258375 · 6258375
2 parents cabaf16 + b24c241
commit 6258375
Show file tree

Hide file tree

Showing 34 changed files with 453 additions and 125 deletions.
diff --git a/test/metadata/testharness/firefox/__dir__.ini b/test/metadata/testharness/firefox/__dir__.ini
@@ -0,0 +1,2 @@
+prefs: ["browser.display.foreground_color:#FF0000",
+        "browser.display.background_color:#000000"]
diff --git a/test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini b/test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini
@@ -0,0 +1,2 @@
+[test_pref_reset.html]
+  prefs: [@Reset]
diff --git a/test/metadata/testharness/subdir/__dir__.ini b/test/metadata/testharness/subdir/__dir__.ini
@@ -0,0 +1 @@
+disabled: true
diff --git a/test/metadata/testharness/subdir/testharness_1.html.ini b/test/metadata/testharness/subdir/testharness_1.html.ini
@@ -0,0 +1,2 @@
+[testharness_1.html]
+  disabled: @False
diff --git a/test/metadata/testharness/tags/__dir__.ini b/test/metadata/testharness/tags/__dir__.ini
@@ -0,0 +1 @@
+tags: [dir-tag-1, dir-tag-2]
diff --git a/test/metadata/testharness/tags/testharness_0.html.ini b/test/metadata/testharness/tags/testharness_0.html.ini
@@ -0,0 +1,4 @@
+tags: [file-tag]
+
+[testharness_0.html]
+  tags: [test-tag]
diff --git a/test/metadata/testharness/tags/testharness_1.html.ini b/test/metadata/testharness/tags/testharness_1.html.ini
@@ -0,0 +1,2 @@
+[testharness_0.html]
+  tags: [test-1-tag]
diff --git a/test/metadata/testharness/tags/testharness_2.html.ini b/test/metadata/testharness/tags/testharness_2.html.ini
@@ -0,0 +1,4 @@
+tags: [file-tag]
+
+[testharness_2.html]
+  tags: [test-2-tag, @Reset]
diff --git a/test/test.py b/test/test.py
@@ -101,6 +101,8 @@ def settings_to_argv(settings):
 def set_from_args(settings, args):
     if args.test:
         settings["include"] = args.test
+    if args.tags:
+        settings["tags"] = args.tags
 
 def run(config, args):
     logger = structuredlog.StructuredLogger("web-platform-tests")
@@ -139,6 +141,8 @@ def get_parser():
                         help="Specific product to include in test run")
     parser.add_argument("--pdb", action="store_true",
                         help="Invoke pdb on uncaught exception")
+    parser.add_argument("--tag", action="append", dest="tags",
+                        help="tags to select tests")
     parser.add_argument("test", nargs="*",
                         help="Specific tests to include in test run")
     return parser

diff --git a/test/testdata/testharness/firefox/subdir/test_pref_inherit.html b/test/testdata/testharness/firefox/subdir/test_pref_inherit.html
@@ -0,0 +1,10 @@
+<!doctype html>
+<title>Example pref test</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
+<script>
+test(function() {
+  assert_equals(getComputedStyle(document.body).color, "rgb(255, 0, 0)");
+}, "Test that pref was set");
+</script>
diff --git a/test/testdata/testharness/firefox/subdir/test_pref_reset.html b/test/testdata/testharness/firefox/subdir/test_pref_reset.html
@@ -0,0 +1,10 @@
+<!doctype html>
+<title>Example pref test</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
+<script>
+test(function() {
+  assert_equals(getComputedStyle(document.body).color, "rgb(0, 0, 0)");
+}, "Test that pref was reset");
+</script>
diff --git a/test/testdata/testharness/firefox/test_pref_dir.html b/test/testdata/testharness/firefox/test_pref_dir.html
@@ -0,0 +1,10 @@
+<!doctype html>
+<title>Example pref test</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<p>Test requires the pref browser.display.foreground_color to be set to #FF0000</p>
+<script>
+test(function() {
+  assert_equals(getComputedStyle(document.body).color, "rgb(255, 0, 0)");
+}, "Test that pref was set");
+</script>
diff --git a/test/testdata/testharness/firefox/test_pref_set.html b/test/testdata/testharness/firefox/test_pref_set.html
@@ -1,10 +1,10 @@
 <!doctype html>
-<title>Example https test</title>
+<title>Example pref test</title>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
 <script>
 test(function() {
   assert_equals(getComputedStyle(document.body).color, "rgb(0, 255, 0)");
 }, "Test that pref was set");
-</script>
+</script>
diff --git a/test/testdata/testharness/subdir/testharness_1.html b/test/testdata/testharness/subdir/testharness_1.html
@@ -0,0 +1,9 @@
+<!doctype html>
+<title>Test should be enabled</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+test(function() {
+  assert_true(true);
+}, "Test that should pass");
+</script>
diff --git a/test/testdata/testharness/tags/testharness_0.html b/test/testdata/testharness/tags/testharness_0.html
@@ -0,0 +1,9 @@
+<!doctype html>
+<title>Test</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+test(function() {
+  assert_true(true);
+}, "Test that should pass");
+</script>
diff --git a/test/testdata/testharness/tags/testharness_1.html b/test/testdata/testharness/tags/testharness_1.html
@@ -0,0 +1,9 @@
+<!doctype html>
+<title>Test</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+test(function() {
+  assert_true(true);
+}, "Test that should pass");
+</script>
diff --git a/test/testdata/testharness/tags/testharness_2.html b/test/testdata/testharness/tags/testharness_2.html
@@ -0,0 +1,9 @@
+<!doctype html>
+<title>Test</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+test(function() {
+  assert_true(true);
+}, "Test that should pass");
+</script>
diff --git a/test/testdata/testharness/testharness_0.html b/test/testdata/testharness/testharness_0.html
@@ -1,13 +1,9 @@
 <!doctype html>
-<title>Simple testharness.js usage</title>
+<title>Test should be disabled</title>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
-test(function() {
-  assert_true(true);
-}, "Test that should pass");
-
 test(function() {
   assert_true(false);
 }, "Test that should fail");
-</script>
+</script>
diff --git a/wptrunner/executors/base.py b/wptrunner/executors/base.py
@@ -99,7 +99,7 @@ def __init__(self, browser, server_config, timeout_multiplier=1,
         self.timeout_multiplier = timeout_multiplier
         self.debug_info = debug_info
         self.last_environment = {"protocol": "http",
-                                 "prefs": []}
+                                 "prefs": {}}
         self.protocol = None # This must be set in subclasses
 
     @property

diff --git a/wptrunner/executors/executormarionette.py b/wptrunner/executors/executormarionette.py
@@ -139,55 +139,62 @@ def wait(self):
 
     def on_environment_change(self, old_environment, new_environment):
         #Unset all the old prefs
-        for name, _ in old_environment.get("prefs", []):
+        for name in old_environment.get("prefs", {}).iterkeys():
             value = self.executor.original_pref_values[name]
             if value is None:
                 self.clear_user_pref(name)
             else:
                 self.set_pref(name, value)
 
-        for name, value in new_environment.get("prefs", []):
+        for name, value in new_environment.get("prefs", {}).iteritems():
             self.executor.original_pref_values[name] = self.get_pref(name)
             self.set_pref(name, value)
 
     def set_pref(self, name, value):
+        if value.lower() not in ("true", "false"):
+            try:
+                int(value)
+            except ValueError:
+                value = "'%s'" % value
+        else:
+            value = value.lower()
+
         self.logger.info("Setting pref %s (%s)" % (name, value))
-        self.marionette.set_context(self.marionette.CONTEXT_CHROME)
+
         script = """
             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
                                           .getService(Components.interfaces.nsIPrefBranch);
             let pref = '%s';
             let type = prefInterface.getPrefType(pref);
+            let value = %s;
             switch(type) {
                 case prefInterface.PREF_STRING:
-                    prefInterface.setCharPref(pref, '%s');
+                    prefInterface.setCharPref(pref, value);
                     break;
                 case prefInterface.PREF_BOOL:
-                    prefInterface.setBoolPref(pref, %s);
+                    prefInterface.setBoolPref(pref, value);
                     break;
                 case prefInterface.PREF_INT:
-                    prefInterface.setIntPref(pref, %s);
+                    prefInterface.setIntPref(pref, value);
                     break;
             }
-            """ % (name, value, value, value)
-        self.marionette.execute_script(script)
-        self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
+            """ % (name, value)
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            self.marionette.execute_script(script)
 
     def clear_user_pref(self, name):
         self.logger.info("Clearing pref %s" % (name))
-        self.marionette.set_context(self.marionette.CONTEXT_CHROME)
         script = """
             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
                                           .getService(Components.interfaces.nsIPrefBranch);
             let pref = '%s';
             prefInterface.clearUserPref(pref);
             """ % name
-        self.marionette.execute_script(script)
-        self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            self.marionette.execute_script(script)
 
     def get_pref(self, name):
-        self.marionette.set_context(self.marionette.CONTEXT_CHROME)
-        self.marionette.execute_script("""
+        script = """
             let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
                                           .getService(Components.interfaces.nsIPrefBranch);
             let pref = '%s';
@@ -202,8 +209,9 @@ def get_pref(self, name):
                 case prefInterface.PREF_INVALID:
                     return null;
             }
-            """ % (name))
-        self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
+            """ % name
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            self.marionette.execute_script(script)
 
 class MarionetteRun(object):
     def __init__(self, logger, func, marionette, url, timeout):

diff --git a/wptrunner/manifestexpected.py b/wptrunner/manifestexpected.py
@@ -29,6 +29,42 @@ def data_cls_getter(output_node, visited_node):
     raise ValueError
 
 
+def disabled(node):
+    """Boolean indicating whether the test is disabled"""
+    try:
+        return node.get("disabled")
+    except KeyError:
+        return None
+
+
+def tags(node):
+    """Set of tags that have been applied to the test"""
+    try:
+        value = node.get("tags")
+        if isinstance(value, (str, unicode)):
+            return {value}
+        return set(value)
+    except KeyError:
+        return set()
+
+
+def prefs(node):
+    def value(ini_value):
+        if isinstance(ini_value, (str, unicode)):
+            return tuple(ini_value.split(":", 1))
+        else:
+            return (ini_value, None)
+
+    try:
+        node_prefs = node.get("prefs")
+        if type(node_prefs) in (str, unicode):
+            prefs = {value(node_prefs)}
+        rv = dict(value(item) for item in node_prefs)
+    except KeyError:
+        rv = {}
+    return rv
+
+
 class ExpectedManifest(ManifestItem):
     def __init__(self, name, test_path, url_base):
         """Object representing all the tests in a particular manifest
@@ -71,6 +107,32 @@ def url(self):
         return urlparse.urljoin(self.url_base,
                                 "/".join(self.test_path.split(os.path.sep)))
 
+    @property
+    def disabled(self):
+        return disabled(self)
+
+    @property
+    def tags(self):
+        return tags(self)
+
+    @property
+    def prefs(self):
+        return prefs(self)
+
+
+class DirectoryManifest(ManifestItem):
+    @property
+    def disabled(self):
+        return disabled(self)
+
+    @property
+    def tags(self):
+        return tags(self)
+
+    @property
+    def prefs(self):
+        return prefs(self)
+
 
 class TestNode(ManifestItem):
     def __init__(self, name):
@@ -100,21 +162,17 @@ def test_type(self):
     def id(self):
         return urlparse.urljoin(self.parent.url, self.name)
 
+    @property
     def disabled(self):
-        """Boolean indicating whether the test is disabled"""
-        try:
-            return self.get("disabled")
-        except KeyError:
-            return False
+        return disabled(self)
 
+    @property
+    def tags(self):
+        return tags(self)
+
+    @property
     def prefs(self):
-        try:
-            prefs = self.get("prefs")
-            if type(prefs) in (str, unicode):
-                prefs = [prefs]
-            return [item.split(":", 1) for item in prefs]
-        except KeyError:
-            return []
+        return prefs(self)
 
     def append(self, node):
         """Add a subtest to the current test
@@ -159,9 +217,28 @@ def get_manifest(metadata_root, test_path, url_base, run_info):
     manifest_path = expected.expected_path(metadata_root, test_path)
     try:
         with open(manifest_path) as f:
-            return static.compile(f, run_info,
+            return static.compile(f,
+                                  run_info,
                                   data_cls_getter=data_cls_getter,
                                   test_path=test_path,
                                   url_base=url_base)
     except IOError:
         return None
+
+def get_dir_manifest(metadata_root, path, run_info):
+    """Get the ExpectedManifest for a particular test path, or None if there is no
+    metadata stored for that test path.
+
+    :param metadata_root: Absolute path to the root of the metadata directory
+    :param path: Path to the ini file relative to the metadata root
+    :param run_info: Dictionary of properties of the test run for which the expectation
+                     values should be computed.
+    """
+    full_path = os.path.join(metadata_root, path)
+    try:
+        with open(full_path) as f:
+            return static.compile(f,
+                                  run_info,
+                                  data_cls_getter=lambda x,y: DirectoryManifest)
+    except IOError:
+        return None
diff --git a/wptrunner/reduce.py b/wptrunner/reduce.py
@@ -56,8 +56,8 @@ def __init__(self, target, **kwargs):
         self.test_loader = wptrunner.TestLoader(kwargs["tests_root"],
                                                 kwargs["metadata_root"],
                                                 [self.test_type],
-                                                test_filter,
-                                                run_info)
+                                                run_info,
+                                                manifest_filer=test_filter)
         if kwargs["repeat"] == 1:
             logger.critical("Need to specify --repeat with more than one repetition")
             sys.exit(1)