Extend the multi-global test (.any.js) infrastructure to support othe…

…r scopes.

This commit is based on ideas and code by Anne van Kesteren and James Graham.

The infrastructure is extended to allow tests to opt-in to being run in shared
workers and service workers, while the default remains as window and dedicated
worker scopes. (This default may be changed in the future.)

docs/_writing-tests/file-names.md

@@ -52,7 +52,7 @@ themselves precede any test type flag, but are otherwise unordered.
 
 `.any`
  : (js files only) Indicates that the file generates tests in which it
-    is run in Window and dedicated worker environments.
+    is [run in multiple scopes][multi-global-tests].
 
 `.tentative`
  : Indicates that a test makes assertions not yet required by any specification,
@@ -63,3 +63,4 @@ themselves precede any test type flag, but are otherwise unordered.
 
 
 [server-side substitution]: https://wptserve.readthedocs.io/en/latest/pipes.html#sub
+[multi-global-tests]: {{ site.baseurl }}{% link _writing-tests/testharness.md %}#multi-global-tests

docs/_writing-tests/testharness.md

@@ -59,13 +59,11 @@ This test could then be run from `FileAPI/FileReaderSync.worker.html`.
 
 ### Multi-global tests
 
-Tests for features that exist in multiple global scopes can be written
-in a way that they are automatically run in a window scope and a
-worker scope.
+Tests for features that exist in multiple global scopes can be written in a way
+that they are automatically run in several scopes. In this case, the test is a
+JavaScript file with extension `.any.js`, which can use all the usual APIs.
 
-In this case, the test is a JavaScript file with extension `.any.js`.
-The test can then use all the usual APIs, and can be run from the path to the
-JavaScript file with the `.js` replaced by `.worker.html` or `.html`.
+By default, the test runs in a window scope and a dedicated worker scope.
 
 For example, one could write a test for the `Blob` constructor by
 creating a `FileAPI/Blob-constructor.any.js` as follows:
@@ -80,6 +78,30 @@ creating a `FileAPI/Blob-constructor.any.js` as follows:
 This test could then be run from `FileAPI/Blob-constructor.any.worker.html` as well
 as `FileAPI/Blob-constructor.any.html`.
 
+It is possible to customize the set of scopes with a metadata comment, such as
+
+    // META: global=sharedworker
+    //       ==> would run in the default window and dedicated worker scopes,
+    //           as well as the shared worker scope
+    // META: global=!default,serviceworker
+    //       ==> would only run in the service worker scope
+    // META: global=!window
+    //       ==> would run in the default dedicated worker scope, but not the
+    //           window scope
+    // META: global=worker
+    //       ==> would run in the default window scope, as well as in the
+    //           dedicated, shared and service worker scopes
+
+For a test file <code><var>x</var>.any.js</code>, the available scope keywords
+are:
+
+* `window` (default): to be run at <code><var>x</var>.html</code>
+* `dedicatedworker` (default): to be run at <code><var>x</var>.worker.html</code>
+* `serviceworker`: to be run at <code><var>x</var>.https.any.serviceworker.html</code>
+* `sharedworker`: to be run at <code><var>x</var>.any.sharedworker.html</code>
+* `default`: shorthand for the default scopes
+* `worker`: shorthand for the dedicated, shared and service worker scopes
+
 To check if your test is run from a window or worker you can use the following two methods that will
 be made available by the framework:
 

tools/lint/lint.py

@@ -18,7 +18,7 @@
 from ..gitignore.gitignore import PathFilter
 from ..wpt import testfiles
 
-from manifest.sourcefile import SourceFile, js_meta_re, python_meta_re, space_chars
+from manifest.sourcefile import SourceFile, js_meta_re, python_meta_re, space_chars, get_any_variants, get_default_any_variants
 from six import binary_type, iteritems, itervalues
 from six.moves import range
 from six.moves.urllib.parse import urlsplit, urljoin
@@ -616,6 +616,31 @@ def check_python_ast(repo_root, path, f):
 broken_python_metadata = re.compile(b"#\s*META:")
 
 
+def check_global_metadata(value):
+    global_values = {item.strip() for item in value.split(b",") if item.strip()}
+
+    included_variants = set.union(get_default_any_variants(),
+                                  *(get_any_variants(v) for v in global_values if not v.startswith(b"!")))
+
+    for global_value in global_values:
+        if global_value.startswith(b"!"):
+            excluded_value = global_value[1:]
+            if not get_any_variants(excluded_value):
+                yield ("UNKNOWN-GLOBAL-METADATA", "Unexpected value for global metadata")
+
+            elif excluded_value in global_values:
+                yield ("BROKEN-GLOBAL-METADATA", "Cannot specify both %s and %s" % (global_value, excluded_value))
+
+            else:
+                excluded_variants = get_any_variants(excluded_value)
+                if not (excluded_variants & included_variants):
+                    yield ("BROKEN-GLOBAL-METADATA", "Cannot exclude %s if it is not included" % (excluded_value,))
+
+        else:
+            if not get_any_variants(global_value):
+                yield ("UNKNOWN-GLOBAL-METADATA", "Unexpected value for global metadata")
+
+
 def check_script_metadata(repo_root, path, f):
     if path.endswith((".worker.js", ".any.js")):
         meta_re = js_meta_re
@@ -634,7 +659,9 @@ def check_script_metadata(repo_root, path, f):
         m = meta_re.match(line)
         if m:
             key, value = m.groups()
-            if key == b"timeout":
+            if key == b"global":
+                errors.extend((kind, message, path, idx + 1) for (kind, message) in check_global_metadata(value))
+            elif key == b"timeout":
                 if value != b"long":
                     errors.append(("UNKNOWN-TIMEOUT-METADATA", "Unexpected value for timeout metadata", path, idx + 1))
             elif key == b"script":

tools/lint/tests/test_file_lints.py

@@ -559,6 +559,39 @@ def test_script_metadata(filename, input, error):
         assert errors == []
 
 
+@pytest.mark.parametrize("globals,error", [
+    (b"", None),
+    (b"default", None),
+    (b"!default", None),
+    (b"window", None),
+    (b"!window", None),
+    (b"!dedicatedworker", None),
+    (b"window, !window", "BROKEN-GLOBAL-METADATA"),
+    (b"!serviceworker", "BROKEN-GLOBAL-METADATA"),
+    (b"serviceworker, !serviceworker", "BROKEN-GLOBAL-METADATA"),
+    (b"worker, !dedicatedworker", None),
+    (b"worker, !serviceworker", None),
+    (b"!worker", None),
+    (b"foo", "UNKNOWN-GLOBAL-METADATA"),
+    (b"!foo", "UNKNOWN-GLOBAL-METADATA"),
+])
+def test_script_globals_metadata(globals, error):
+    filename = "foo.any.js"
+    input = b"""// META: global=%s\n""" % globals
+    errors = check_file_contents("", filename, six.BytesIO(input))
+    check_errors(errors)
+
+    if error is not None:
+        errors = [(k, f, l) for (k, _, f, l) in errors]
+        assert errors == [
+            (error,
+             filename,
+             1),
+        ]
+    else:
+        assert errors == []
+
+
 @pytest.mark.parametrize("input,error", [
     (b"""#META: timeout=long\n""", None),
     (b"""# META: timeout=long\n""", None),

tools/manifest/sourcefile.py

@@ -49,6 +49,87 @@ def read_script_metadata(f, regexp):
         yield (m.groups()[0], m.groups()[1])
 
 
+_any_variants = {
+    b"default": {"longhand": {b"window", b"dedicatedworker"}},
+    b"window": {"suffix": ".any.html"},
+    b"serviceworker": {"force_https": True},
+    b"sharedworker": {},
+    b"dedicatedworker": {"suffix": ".any.worker.html"},
+    b"worker": {"longhand": {b"dedicatedworker", b"sharedworker", b"serviceworker"}}
+}
+
+
+def get_any_variants(item):
+    """
+    Returns a set of variants (bytestrings) defined by the given keyword.
+    """
+    assert isinstance(item, binary_type), item
+    assert not item.startswith(b"!"), item
+
+    variant = _any_variants.get(item, None)
+    if variant is None:
+        return set()
+
+    return variant.get("longhand", {item})
+
+
+def get_default_any_variants():
+    """
+    Returns a set of variants (bytestrings) that will be used by default.
+    """
+    return set(_any_variants[b"default"]["longhand"])
+
+
+def parse_variants(value):
+    """
+    Returns a set of variants (bytestrings) defined by a comma-separated value.
+    """
+    assert isinstance(value, binary_type), value
+
+    globals = get_default_any_variants()
+
+    for item in value.split(b","):
+        item = item.strip()
+        if item.startswith(b"!"):
+            globals -= get_any_variants(item[1:])
+        else:
+            globals |= get_any_variants(item)
+
+    return globals
+
+
+def global_suffixes(value):
+    """
+    Yields the relevant filename suffixes (strings) for the variants defined by
+    the given comma-separated value.
+    """
+    assert isinstance(value, binary_type), value
+
+    rv = set()
+
+    global_types = parse_variants(value)
+    for global_type in global_types:
+        variant = _any_variants[global_type]
+        suffix = variant.get("suffix", ".any.%s.html" % global_type.decode("utf-8"))
+        if variant.get("force_https", False):
+            suffix = ".https" + suffix
+        rv.add(suffix)
+
+    return rv
+
+
+def global_variant_url(url, suffix):
+    """
+    Returns a url created from the given url and suffix (all strings).
+    """
+    url = url.replace(".any.", ".")
+    # If the url must be loaded over https, ensure that it will have
+    # the form .https.any.js
+    if ".https." in url and suffix.startswith(".https."):
+        url = url.replace(".https.", ".")
+    return replace_end(url, ".js", suffix)
+
+
 class SourceFile(object):
     parsers = {"html":lambda x:html5lib.parse(x, treebuilder="etree"),
                "xhtml":lambda x:ElementTree.parse(x, XMLParser.XMLParser()),
@@ -510,12 +591,17 @@ def manifest_items(self):
             rv = VisualTest.item_type, [VisualTest(self, self.url)]
 
         elif self.name_is_multi_global:
-            rv = TestharnessTest.item_type, [
-                TestharnessTest(self, replace_end(self.url, ".any.js", ".any.html"),
-                                timeout=self.timeout),
-                TestharnessTest(self, replace_end(self.url, ".any.js", ".any.worker.html"),
-                                timeout=self.timeout),
+            globals = b""
+            for (key, value) in self.script_metadata:
+                if key == b"global":
+                    globals = value
+                    break
+
+            tests = [
+                TestharnessTest(self, global_variant_url(self.url, suffix), timeout=self.timeout)
+                for suffix in sorted(global_suffixes(globals))
             ]
+            rv = TestharnessTest.item_type, tests
 
         elif self.name_is_worker:
             rv = (TestharnessTest.item_type,

tools/manifest/tests/test_sourcefile.py

@@ -3,6 +3,7 @@
 import pytest
 
 from six import BytesIO
+from ...lint.lint import check_global_metadata
 from ..sourcefile import SourceFile, read_script_metadata, js_meta_re, python_meta_re
 
 def create(filename, contents=b""):
@@ -263,6 +264,61 @@ def test_multi_global_long_timeout():
         assert item.timeout == "long"
 
 
+@pytest.mark.parametrize("input,expected", [
+    (b"", {"dedicatedworker", "window"}),
+    (b"default", {"dedicatedworker", "window"}),
+    (b"!default", {}),
+    (b"!default,window", {"window"}),
+    (b"window,!default", {}),
+    (b"!default,dedicatedworker", {"dedicatedworker"}),
+    (b"dedicatedworker,!default", {}),
+    (b"!default,worker", {"dedicatedworker", "serviceworker", "sharedworker"}),
+    (b"worker,!default", {"serviceworker", "sharedworker"}),
+    (b"!dedicatedworker", {"window"}),
+    (b"!worker", {"window"}),
+    (b"!window", {"dedicatedworker"}),
+    (b"!window,worker", {"dedicatedworker", "serviceworker", "sharedworker"}),
+    (b"worker,!dedicatedworker", {"serviceworker", "sharedworker", "window"}),
+    (b"!dedicatedworker,worker", {"dedicatedworker", "serviceworker", "sharedworker", "window"}),
+    (b"worker,!sharedworker", {"dedicatedworker", "serviceworker", "window"}),
+    (b"!sharedworker,worker", {"dedicatedworker", "serviceworker", "sharedworker", "window"}),
+    (b"sharedworker", {"dedicatedworker", "sharedworker", "window"}),
+    (b"sharedworker,serviceworker", {"dedicatedworker", "serviceworker", "sharedworker", "window"}),
+])
+def test_multi_global_with_custom_globals(input, expected):
+    contents = b"""// META: global=%s
+test()""" % input
+
+    assert list(check_global_metadata(input)) == []
+
+    s = create("html/test.any.js", contents=contents)
+    assert not s.name_is_non_test
+    assert not s.name_is_manual
+    assert not s.name_is_visual
+    assert s.name_is_multi_global
+    assert not s.name_is_worker
+    assert not s.name_is_reference
+
+    assert not s.content_is_testharness
+
+    item_type, items = s.manifest_items()
+    assert item_type == "testharness"
+
+    urls = {
+        "dedicatedworker": "/html/test.any.worker.html",
+        "serviceworker": "/html/test.https.any.serviceworker.html",
+        "sharedworker": "/html/test.any.sharedworker.html",
+        "window": "/html/test.any.html",
+    }
+
+    expected_urls = sorted(urls[ty] for ty in expected)
+    assert len(items) == len(expected_urls)
+
+    for item, url in zip(items, expected_urls):
+        assert item.url == url
+        assert item.timeout is None
+
+
 @pytest.mark.parametrize("input,expected", [
     (b"""//META: foo=bar\n""", [(b"foo", b"bar")]),
     (b"""// META: foo=bar\n""", [(b"foo", b"bar")]),