From 545f4c69a3b8ed08dd618186546e79740d6c7930 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 5 May 2015 11:55:03 +0100 Subject: [PATCH 1/4] Add --tags command line argument for slecting groups of tests. Tests can be tagged by providing a 'tags: [tag1, tag2]' entry in the ini file. There is also a default set of tags of the form dir:top-level-dir-name so that you can e.g. run all dom tests with dir:dom --- test/metadata/testharness/tags/__dir__.ini | 1 + .../testharness/tags/testharness_0.html.ini | 4 ++ .../testharness/tags/testharness_1.html.ini | 2 + .../testharness/tags/testharness_2.html.ini | 4 ++ test/test.py | 4 ++ .../testharness/tags/testharness_0.html | 9 +++++ .../testharness/tags/testharness_1.html | 9 +++++ .../testharness/tags/testharness_2.html | 9 +++++ wptrunner/manifestexpected.py | 10 +++++ wptrunner/reduce.py | 4 +- wptrunner/testloader.py | 37 +++++++++++++++---- wptrunner/wptcommandline.py | 2 + wptrunner/wptrunner.py | 17 ++++++--- wptrunner/wpttest.py | 13 +++++++ 14 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 test/metadata/testharness/tags/__dir__.ini create mode 100644 test/metadata/testharness/tags/testharness_0.html.ini create mode 100644 test/metadata/testharness/tags/testharness_1.html.ini create mode 100644 test/metadata/testharness/tags/testharness_2.html.ini create mode 100644 test/testdata/testharness/tags/testharness_0.html create mode 100644 test/testdata/testharness/tags/testharness_1.html create mode 100644 test/testdata/testharness/tags/testharness_2.html diff --git a/test/metadata/testharness/tags/__dir__.ini b/test/metadata/testharness/tags/__dir__.ini new file mode 100644 index 00000000000000..f599adda92b54d --- /dev/null +++ b/test/metadata/testharness/tags/__dir__.ini @@ -0,0 +1 @@ +tags: [dir-tag-1, dir-tag-2] \ No newline at end of file diff --git a/test/metadata/testharness/tags/testharness_0.html.ini b/test/metadata/testharness/tags/testharness_0.html.ini new file mode 100644 index 00000000000000..fe8ffa48dd0bbe --- /dev/null +++ b/test/metadata/testharness/tags/testharness_0.html.ini @@ -0,0 +1,4 @@ +tags: [file-tag] + +[testharness_0.html] + tags: [test-tag] diff --git a/test/metadata/testharness/tags/testharness_1.html.ini b/test/metadata/testharness/tags/testharness_1.html.ini new file mode 100644 index 00000000000000..d6006a1551c6f4 --- /dev/null +++ b/test/metadata/testharness/tags/testharness_1.html.ini @@ -0,0 +1,2 @@ +[testharness_0.html] + tags: [test-1-tag] diff --git a/test/metadata/testharness/tags/testharness_2.html.ini b/test/metadata/testharness/tags/testharness_2.html.ini new file mode 100644 index 00000000000000..25fbf55362dea9 --- /dev/null +++ b/test/metadata/testharness/tags/testharness_2.html.ini @@ -0,0 +1,4 @@ +tags: [file-tag] + +[testharness_2.html] + tags: [test-2-tag, @Reset] diff --git a/test/test.py b/test/test.py index 234cb671353e2a..ed3a92a6e32fa1 100644 --- a/test/test.py +++ b/test/test.py @@ -101,6 +101,8 @@ def settings_to_argv(settings): def set_from_args(settings, args): if args.test: settings["include"] = args.test + if args.tags: + settings["tags"] = args.tags def run(config, args): logger = structuredlog.StructuredLogger("web-platform-tests") @@ -139,6 +141,8 @@ def get_parser(): help="Specific product to include in test run") parser.add_argument("--pdb", action="store_true", help="Invoke pdb on uncaught exception") + parser.add_argument("--tag", action="append", + help="tags to select tests") parser.add_argument("test", nargs="*", help="Specific tests to include in test run") return parser diff --git a/test/testdata/testharness/tags/testharness_0.html b/test/testdata/testharness/tags/testharness_0.html new file mode 100644 index 00000000000000..5daf02a77d9e99 --- /dev/null +++ b/test/testdata/testharness/tags/testharness_0.html @@ -0,0 +1,9 @@ + +Test + + + diff --git a/test/testdata/testharness/tags/testharness_1.html b/test/testdata/testharness/tags/testharness_1.html new file mode 100644 index 00000000000000..5daf02a77d9e99 --- /dev/null +++ b/test/testdata/testharness/tags/testharness_1.html @@ -0,0 +1,9 @@ + +Test + + + diff --git a/test/testdata/testharness/tags/testharness_2.html b/test/testdata/testharness/tags/testharness_2.html new file mode 100644 index 00000000000000..5daf02a77d9e99 --- /dev/null +++ b/test/testdata/testharness/tags/testharness_2.html @@ -0,0 +1,9 @@ + +Test + + + diff --git a/wptrunner/manifestexpected.py b/wptrunner/manifestexpected.py index beebdd35ab68cd..a7eb33036ed97e 100644 --- a/wptrunner/manifestexpected.py +++ b/wptrunner/manifestexpected.py @@ -107,6 +107,16 @@ def disabled(self): except KeyError: return False + def tags(self): + """Set of tags that have been applied to the test""" + try: + value = self.get("tags") + if isinstance(value, (str, unicode)): + return {value} + return set(value) + except KeyError: + return set() + def prefs(self): try: prefs = self.get("prefs") diff --git a/wptrunner/reduce.py b/wptrunner/reduce.py index 50683cf4b28f63..9f50dbec28469e 100644 --- a/wptrunner/reduce.py +++ b/wptrunner/reduce.py @@ -56,8 +56,8 @@ def __init__(self, target, **kwargs): self.test_loader = wptrunner.TestLoader(kwargs["tests_root"], kwargs["metadata_root"], [self.test_type], - test_filter, - run_info) + run_info, + manifest_filer=test_filter) if kwargs["repeat"] == 1: logger.critical("Need to specify --repeat with more than one repetition") sys.exit(1) diff --git a/wptrunner/testloader.py b/wptrunner/testloader.py index 01ed0a81bb2d28..989823170013d0 100644 --- a/wptrunner/testloader.py +++ b/wptrunner/testloader.py @@ -343,6 +343,14 @@ def __call__(self, manifest_iter): if include_tests: yield test_path, include_tests +class TagFilter(object): + def __init__(self, tags): + self.tags = set(tags) + + def __call__(self, test_iter): + for test in test_iter: + if test.tags() & self.tags: + yield test class ManifestLoader(object): def __init__(self, test_paths, force_manifest_update=False): @@ -405,20 +413,30 @@ def load_manifest(self, tests_path, metadata_path, url_base="/"): return manifest_file +def iterfilter(filters, iter): + for f in filters: + iter = f(iter) + for item in iter: + yield item + class TestLoader(object): def __init__(self, test_manifests, test_types, - test_filter, run_info, + manifest_filters=None, + meta_filters=None, chunk_type="none", total_chunks=1, chunk_number=1, include_https=True): self.test_types = test_types - self.test_filter = test_filter self.run_info = run_info + + self.manifest_filters = manifest_filters if manifest_filters is not None else [] + self.meta_filters = meta_filters if meta_filters is not None else [] + self.manifests = test_manifests self.tests = None self.disabled_tests = None @@ -460,7 +478,9 @@ def iter_tests(self): manifest_items = [] for manifest in self.manifests.keys(): - manifest_items.extend(self.test_filter(manifest.itertypes(*self.test_types))) + manifest_iter = iterfilter(self.manifest_filters, + manifest.itertypes(*self.test_types)) + manifest_items.extend(manifest_iter) if self.chunker is not None: manifest_items = self.chunker(manifest_items) @@ -470,10 +490,13 @@ def iter_tests(self): metadata_path = self.manifests[manifest_file]["metadata_path"] expected_file = self.load_expected_manifest(manifest_file, metadata_path, test_path) - for manifest_test in tests: - test = self.get_test(manifest_test, expected_file) - test_type = manifest_test.item_type - yield test_path, test_type, test + for test in iterfilter(self.meta_filters, + self.iter_wpttest(expected_file, tests)): + yield test_path, test.test_type, test + + def iter_wpttest(self, expected_file, tests): + for manifest_test in tests: + yield self.get_test(manifest_test, expected_file) def _load_tests(self): """Read in the tests from the manifest file and add them to a queue""" diff --git a/wptrunner/wptcommandline.py b/wptrunner/wptcommandline.py index 5586eb9815dd30..933468a777d813 100644 --- a/wptrunner/wptcommandline.py +++ b/wptrunner/wptcommandline.py @@ -105,6 +105,8 @@ def create_parser(product_choices=None): help="URL prefix to exclude") test_selection_group.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") + test_selection_group.add_argument("--tag", action="append", dest="tags", + help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.") debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument('--debugger', const="__default__", nargs="?", diff --git a/wptrunner/wptrunner.py b/wptrunner/wptrunner.py index 5f712a4daa6290..42923c2ae0df0d 100644 --- a/wptrunner/wptrunner.py +++ b/wptrunner/wptrunner.py @@ -45,15 +45,22 @@ def get_loader(test_paths, product, ssl_env, debug=None, **kwargs): test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load() - test_filter = testloader.TestFilter(include=kwargs["include"], - exclude=kwargs["exclude"], - manifest_path=kwargs["include_manifest"], - test_manifests=test_manifests) + manifest_filters = [] + meta_filters = [] + + if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]: + manifest_filters.append(testloader.TestFilter(include=kwargs["include"], + exclude=kwargs["exclude"], + manifest_path=kwargs["include_manifest"], + test_manifests=test_manifests)) + if kwargs["tags"]: + meta_filters.append(testloader.TagFilter(tags=kwargs["tags"])) test_loader = testloader.TestLoader(test_manifests, kwargs["test_types"], - test_filter, run_info, + manifest_filters=manifest_filters, + meta_filters=meta_filters, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"], diff --git a/wptrunner/wpttest.py b/wptrunner/wpttest.py index c5b8ae1f1d4823..a6067db014b35c 100644 --- a/wptrunner/wpttest.py +++ b/wptrunner/wpttest.py @@ -86,6 +86,7 @@ def __init__(self, *args, **kwargs): class Test(object): result_cls = None subtest_result_cls = None + test_type = None def __init__(self, url, expected_metadata, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): @@ -137,6 +138,14 @@ def disabled(self, subtest=None): return metadata.disabled() + def tags(self): + tags = set(["dir:%s" % self.id.lstrip("/").split("/")[0]]) + metadata = self._get_metadata(None) + if metadata is None: + return tags + + return tags | metadata.tags() + def expected(self, subtest=None): if subtest is None: default = self.result_cls.default_expected @@ -156,6 +165,7 @@ def expected(self, subtest=None): class TestharnessTest(Test): result_cls = TestharnessResult subtest_result_cls = TestharnessSubtestResult + test_type = "testharness" @property def id(self): @@ -163,6 +173,8 @@ def id(self): class ManualTest(Test): + test_type = "manual" + @property def id(self): return self.url @@ -170,6 +182,7 @@ def id(self): class ReftestTest(Test): result_cls = ReftestResult + test_type = "reftest" def __init__(self, url, expected, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): Test.__init__(self, url, expected, timeout, path, protocol) From d83cc93c9bdc5ff8fb712a02d8a3bab7563b82db Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 5 May 2015 17:25:00 +0100 Subject: [PATCH 2/4] Support inheriting metadata from directories. This change means that tags, disabled and prefs can be set in specific directories, or in specific test paths, as well as in specific tests. In order to set it in directories create a /path/to_dir/__dir__.ini file. These apply to each directory and all subdirectories. Occasionally one wishes to override the metadata set in per-directory ini file e.g. in order to enable one specific test whilst all others are disabled, or to unset the prefs specified in higher-level directories. To achieve this the concept of atoms has been introduced to the parser, which are used to represent constant values. In particular: disabled: @False Will ensure that a test is enabled, whilst prefs: [@Reset] will ensure that no prefs from higher-level directories are used. --- test/metadata/testharness/firefox/__dir__.ini | 2 + .../firefox/subdir/test_pref_reset.html.ini | 2 + test/metadata/testharness/subdir/__dir__.ini | 1 + .../testharness/subdir/testharness_1.html.ini | 2 + .../firefox/subdir/test_pref_inherit.html | 10 ++ .../firefox/subdir/test_pref_reset.html | 10 ++ .../testharness/firefox/test_pref_dir.html | 10 ++ .../testharness/firefox/test_pref_set.html | 4 +- .../testharness/subdir/testharness_1.html | 9 ++ test/testdata/testharness/testharness_0.html | 8 +- wptrunner/executors/base.py | 2 +- wptrunner/executors/executormarionette.py | 22 ++-- wptrunner/manifestexpected.py | 109 ++++++++++++++---- wptrunner/testloader.py | 47 +++++--- wptrunner/testrunner.py | 8 +- wptrunner/wptmanifest/backends/conditional.py | 3 + wptrunner/wptmanifest/backends/static.py | 3 + wptrunner/wptmanifest/node.py | 4 + wptrunner/wptmanifest/parser.py | 85 +++++++++----- wptrunner/wptmanifest/serializer.py | 7 +- wptrunner/wptmanifest/tests/test_parser.py | 8 ++ .../wptmanifest/tests/test_serializer.py | 16 +++ wptrunner/wpttest.py | 99 ++++++++++------ 23 files changed, 351 insertions(+), 120 deletions(-) create mode 100644 test/metadata/testharness/firefox/__dir__.ini create mode 100644 test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini create mode 100644 test/metadata/testharness/subdir/__dir__.ini create mode 100644 test/metadata/testharness/subdir/testharness_1.html.ini create mode 100644 test/testdata/testharness/firefox/subdir/test_pref_inherit.html create mode 100644 test/testdata/testharness/firefox/subdir/test_pref_reset.html create mode 100644 test/testdata/testharness/firefox/test_pref_dir.html create mode 100644 test/testdata/testharness/subdir/testharness_1.html diff --git a/test/metadata/testharness/firefox/__dir__.ini b/test/metadata/testharness/firefox/__dir__.ini new file mode 100644 index 00000000000000..c9d164cd418a2a --- /dev/null +++ b/test/metadata/testharness/firefox/__dir__.ini @@ -0,0 +1,2 @@ +prefs: ["browser.display.foreground_color:#FF0000", + "browser.display.background_color:#000000"] \ No newline at end of file diff --git a/test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini b/test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini new file mode 100644 index 00000000000000..6c9198d9bbf1c7 --- /dev/null +++ b/test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini @@ -0,0 +1,2 @@ +[test_pref_reset.html] + prefs: [@Reset] diff --git a/test/metadata/testharness/subdir/__dir__.ini b/test/metadata/testharness/subdir/__dir__.ini new file mode 100644 index 00000000000000..a9157fbc6a9f4a --- /dev/null +++ b/test/metadata/testharness/subdir/__dir__.ini @@ -0,0 +1 @@ +disabled: true \ No newline at end of file diff --git a/test/metadata/testharness/subdir/testharness_1.html.ini b/test/metadata/testharness/subdir/testharness_1.html.ini new file mode 100644 index 00000000000000..db9393987b6c73 --- /dev/null +++ b/test/metadata/testharness/subdir/testharness_1.html.ini @@ -0,0 +1,2 @@ +[testharness_1.html] + disabled: @False \ No newline at end of file diff --git a/test/testdata/testharness/firefox/subdir/test_pref_inherit.html b/test/testdata/testharness/firefox/subdir/test_pref_inherit.html new file mode 100644 index 00000000000000..10b285194b4ae0 --- /dev/null +++ b/test/testdata/testharness/firefox/subdir/test_pref_inherit.html @@ -0,0 +1,10 @@ + +Example pref test + + +

Test requires the pref browser.display.foreground_color to be set to #00FF00

+ diff --git a/test/testdata/testharness/firefox/subdir/test_pref_reset.html b/test/testdata/testharness/firefox/subdir/test_pref_reset.html new file mode 100644 index 00000000000000..5c75c1160522ce --- /dev/null +++ b/test/testdata/testharness/firefox/subdir/test_pref_reset.html @@ -0,0 +1,10 @@ + +Example pref test + + +

Test requires the pref browser.display.foreground_color to be set to #00FF00

+ diff --git a/test/testdata/testharness/firefox/test_pref_dir.html b/test/testdata/testharness/firefox/test_pref_dir.html new file mode 100644 index 00000000000000..105d9070c9dcea --- /dev/null +++ b/test/testdata/testharness/firefox/test_pref_dir.html @@ -0,0 +1,10 @@ + +Example pref test + + +

Test requires the pref browser.display.foreground_color to be set to #FF0000

+ diff --git a/test/testdata/testharness/firefox/test_pref_set.html b/test/testdata/testharness/firefox/test_pref_set.html index 9a783ed78c934e..8e5e2989bf7299 100644 --- a/test/testdata/testharness/firefox/test_pref_set.html +++ b/test/testdata/testharness/firefox/test_pref_set.html @@ -1,5 +1,5 @@ -Example https test +Example pref test

Test requires the pref browser.display.foreground_color to be set to #00FF00

@@ -7,4 +7,4 @@ test(function() { assert_equals(getComputedStyle(document.body).color, "rgb(0, 255, 0)"); }, "Test that pref was set"); - \ No newline at end of file + diff --git a/test/testdata/testharness/subdir/testharness_1.html b/test/testdata/testharness/subdir/testharness_1.html new file mode 100644 index 00000000000000..fd2fc431d39998 --- /dev/null +++ b/test/testdata/testharness/subdir/testharness_1.html @@ -0,0 +1,9 @@ + +Test should be enabled + + + diff --git a/test/testdata/testharness/testharness_0.html b/test/testdata/testharness/testharness_0.html index be612a976a28a5..ff0654cb9a0925 100644 --- a/test/testdata/testharness/testharness_0.html +++ b/test/testdata/testharness/testharness_0.html @@ -1,13 +1,9 @@ -Simple testharness.js usage +Test should be disabled \ No newline at end of file + diff --git a/wptrunner/executors/base.py b/wptrunner/executors/base.py index 4aff10f0a77d12..4e983fb663f616 100644 --- a/wptrunner/executors/base.py +++ b/wptrunner/executors/base.py @@ -99,7 +99,7 @@ def __init__(self, browser, server_config, timeout_multiplier=1, self.timeout_multiplier = timeout_multiplier self.debug_info = debug_info self.last_environment = {"protocol": "http", - "prefs": []} + "prefs": {}} self.protocol = None # This must be set in subclasses @property diff --git a/wptrunner/executors/executormarionette.py b/wptrunner/executors/executormarionette.py index 0e9da816b7f275..495132827e81d2 100644 --- a/wptrunner/executors/executormarionette.py +++ b/wptrunner/executors/executormarionette.py @@ -139,37 +139,45 @@ def wait(self): def on_environment_change(self, old_environment, new_environment): #Unset all the old prefs - for name, _ in old_environment.get("prefs", []): + for name, _ in old_environment.get("prefs", {}).iteritems(): value = self.executor.original_pref_values[name] if value is None: self.clear_user_pref(name) else: self.set_pref(name, value) - for name, value in new_environment.get("prefs", []): + for name, value in new_environment.get("prefs", {}).iteritems(): self.executor.original_pref_values[name] = self.get_pref(name) self.set_pref(name, value) def set_pref(self, name, value): + if value not in ("true", "false"): + try: + int(value) + except ValueError: + value = "'%s'" % value + self.logger.info("Setting pref %s (%s)" % (name, value)) - self.marionette.set_context(self.marionette.CONTEXT_CHROME) + script = """ let prefInterface = Components.classes["@mozilla.org/preferences-service;1"] .getService(Components.interfaces.nsIPrefBranch); let pref = '%s'; let type = prefInterface.getPrefType(pref); + let value = %s; switch(type) { case prefInterface.PREF_STRING: - prefInterface.setCharPref(pref, '%s'); + prefInterface.setCharPref(pref, value); break; case prefInterface.PREF_BOOL: - prefInterface.setBoolPref(pref, %s); + prefInterface.setBoolPref(pref, value); break; case prefInterface.PREF_INT: - prefInterface.setIntPref(pref, %s); + prefInterface.setIntPref(pref, value); break; } - """ % (name, value, value, value) + """ % (name, value) + self.marionette.set_context(self.marionette.CONTEXT_CHROME) self.marionette.execute_script(script) self.marionette.set_context(self.marionette.CONTEXT_CONTENT) diff --git a/wptrunner/manifestexpected.py b/wptrunner/manifestexpected.py index a7eb33036ed97e..b46a1ef9597574 100644 --- a/wptrunner/manifestexpected.py +++ b/wptrunner/manifestexpected.py @@ -29,6 +29,42 @@ def data_cls_getter(output_node, visited_node): raise ValueError +def disabled(node): + """Boolean indicating whether the test is disabled""" + try: + return node.get("disabled") + except KeyError: + return None + + +def tags(node): + """Set of tags that have been applied to the test""" + try: + value = node.get("tags") + if isinstance(value, (str, unicode)): + return {value} + return set(value) + except KeyError: + return set() + + +def prefs(node): + def value(ini_value): + if isinstance(ini_value, (str, unicode)): + return tuple(ini_value.split(":", 1)) + else: + return (ini_value, None) + + try: + node_prefs = node.get("prefs") + if type(node_prefs) in (str, unicode): + prefs = {value(node_prefs)} + rv = dict(value(item) for item in node_prefs) + except KeyError: + rv = {} + return rv + + class ExpectedManifest(ManifestItem): def __init__(self, name, test_path, url_base): """Object representing all the tests in a particular manifest @@ -71,6 +107,32 @@ def url(self): return urlparse.urljoin(self.url_base, "/".join(self.test_path.split(os.path.sep))) + @property + def disabled(self): + return disabled(self) + + @property + def tags(self): + return tags(self) + + @property + def prefs(self): + return prefs(self) + + +class DirectoryManifest(ManifestItem): + @property + def disabled(self): + return disabled(self) + + @property + def tags(self): + return tags(self) + + @property + def prefs(self): + return prefs(self) + class TestNode(ManifestItem): def __init__(self, name): @@ -100,31 +162,17 @@ def test_type(self): def id(self): return urlparse.urljoin(self.parent.url, self.name) + @property def disabled(self): - """Boolean indicating whether the test is disabled""" - try: - return self.get("disabled") - except KeyError: - return False + return disabled(self) + @property def tags(self): - """Set of tags that have been applied to the test""" - try: - value = self.get("tags") - if isinstance(value, (str, unicode)): - return {value} - return set(value) - except KeyError: - return set() + return tags(self) + @property def prefs(self): - try: - prefs = self.get("prefs") - if type(prefs) in (str, unicode): - prefs = [prefs] - return [item.split(":", 1) for item in prefs] - except KeyError: - return [] + return prefs(self) def append(self, node): """Add a subtest to the current test @@ -169,9 +217,28 @@ def get_manifest(metadata_root, test_path, url_base, run_info): manifest_path = expected.expected_path(metadata_root, test_path) try: with open(manifest_path) as f: - return static.compile(f, run_info, + return static.compile(f, + run_info, data_cls_getter=data_cls_getter, test_path=test_path, url_base=url_base) except IOError: return None + +def get_dir_manifest(metadata_root, path, run_info): + """Get the ExpectedManifest for a particular test path, or None if there is no + metadata stored for that test path. + + :param metadata_root: Absolute path to the root of the metadata directory + :param path: Path to the ini file relative to the metadata root + :param run_info: Dictionary of properties of the test run for which the expectation + values should be computed. + """ + full_path = os.path.join(metadata_root, path) + try: + with open(full_path) as f: + return static.compile(f, + run_info, + data_cls_getter=lambda x,y: DirectoryManifest) + except IOError: + return None diff --git a/wptrunner/testloader.py b/wptrunner/testloader.py index 989823170013d0..c431c840d1208f 100644 --- a/wptrunner/testloader.py +++ b/wptrunner/testloader.py @@ -1,5 +1,6 @@ import json import os +import sys import urlparse from abc import ABCMeta, abstractmethod from Queue import Empty @@ -349,7 +350,7 @@ def __init__(self, tags): def __call__(self, test_iter): for test in test_iter: - if test.tags() & self.tags: + if test.tags & self.tags: yield test class ManifestLoader(object): @@ -452,6 +453,9 @@ def __init__(self, chunk_number) self._test_ids = None + + self.directory_manifests = {} + self._load_tests() @property @@ -463,16 +467,31 @@ def test_ids(self): self._test_ids += [item.id for item in test_dict[test_type]] return self._test_ids - def get_test(self, manifest_test, expected_file): - if expected_file is not None: - expected = expected_file.get_test(manifest_test.id) - else: - expected = None - - return wpttest.from_manifest(manifest_test, expected) + def get_test(self, manifest_test, inherit_metadata, test_metadata): + if test_metadata is not None: + inherit_metadata.append(test_metadata) + test_metadata = test_metadata.get_test(manifest_test.id) + + return wpttest.from_manifest(manifest_test, inherit_metadata, test_metadata) + + def load_dir_metadata(self, test_manifest, metadata_path, test_path): + rv = [] + path_parts = os.path.dirname(test_path).split(os.path.sep) + for i in xrange(1,len(path_parts) + 1): + path = os.path.join(os.path.sep.join(path_parts[:i]), "__dir__.ini") + if path not in self.directory_manifests: + self.directory_manifests[path] = manifestexpected.get_dir_manifest( + metadata_path, path, self.run_info) + manifest = self.directory_manifests[path] + if manifest is not None: + rv.append(manifest) + return rv - def load_expected_manifest(self, test_manifest, metadata_path, test_path): - return manifestexpected.get_manifest(metadata_path, test_path, test_manifest.url_base, self.run_info) + def load_metadata(self, test_manifest, metadata_path, test_path): + inherit_metadata = self.load_dir_metadata(test_manifest, metadata_path, test_path) + test_metadata = manifestexpected.get_manifest( + metadata_path, test_path, test_manifest.url_base, self.run_info) + return inherit_metadata, test_metadata def iter_tests(self): manifest_items = [] @@ -488,15 +507,15 @@ def iter_tests(self): for test_path, tests in manifest_items: manifest_file = iter(tests).next().manifest metadata_path = self.manifests[manifest_file]["metadata_path"] - expected_file = self.load_expected_manifest(manifest_file, metadata_path, test_path) + inherit_metadata, test_metadata = self.load_metadata(manifest_file, metadata_path, test_path) for test in iterfilter(self.meta_filters, - self.iter_wpttest(expected_file, tests)): + self.iter_wpttest(inherit_metadata, test_metadata, tests)): yield test_path, test.test_type, test - def iter_wpttest(self, expected_file, tests): + def iter_wpttest(self, inherit_metadata, test_metadata, tests): for manifest_test in tests: - yield self.get_test(manifest_test, expected_file) + yield self.get_test(manifest_test, inherit_metadata, test_metadata) def _load_tests(self): """Read in the tests from the manifest file and add them to a queue""" diff --git a/wptrunner/testrunner.py b/wptrunner/testrunner.py index 063fa7d159e9f9..e665b349eac911 100644 --- a/wptrunner/testrunner.py +++ b/wptrunner/testrunner.py @@ -293,8 +293,8 @@ def run(self): # reason # Need to consider the unlikely case where one test causes the # runner process to repeatedly die - self.logger.info("Last test did not complete, requeueing") - self.requeue_test() + self.logger.critical("Last test did not complete") + break self.logger.warning( "More tests found, but runner process died, restarting") self.restart_count += 1 @@ -466,10 +466,6 @@ def stop_runner(self): def start_next_test(self): self.send_message("run_test") - def requeue_test(self): - self.test_source.requeue(self.test) - self.test = None - def test_start(self, test): self.test = test self.logger.test_start(test.id) diff --git a/wptrunner/wptmanifest/backends/conditional.py b/wptrunner/wptmanifest/backends/conditional.py index eed7c8aabac231..f7921c67ad22ed 100644 --- a/wptrunner/wptmanifest/backends/conditional.py +++ b/wptrunner/wptmanifest/backends/conditional.py @@ -115,6 +115,9 @@ def visit_ListNode(self, node): def visit_ValueNode(self, node): return (lambda x: True, node.data) + def visit_AtomNode(self, node): + return (lambda x: True, node.data) + def visit_ConditionalNode(self, node): return self.visit(node.children[0]), self.visit(node.children[1]) diff --git a/wptrunner/wptmanifest/backends/static.py b/wptrunner/wptmanifest/backends/static.py index 7120e89de04302..7221fce7246699 100644 --- a/wptrunner/wptmanifest/backends/static.py +++ b/wptrunner/wptmanifest/backends/static.py @@ -68,6 +68,9 @@ def visit_KeyValueNode(self, node): def visit_ValueNode(self, node): return node.data + def visit_AtomNode(self, node): + return node.data + def visit_ListNode(self, node): return [self.visit(child) for child in node.children] diff --git a/wptrunner/wptmanifest/node.py b/wptrunner/wptmanifest/node.py index bb40535936e4d0..e260eeaf765e83 100644 --- a/wptrunner/wptmanifest/node.py +++ b/wptrunner/wptmanifest/node.py @@ -93,6 +93,10 @@ def append(self, other): raise TypeError +class AtomNode(ValueNode): + pass + + class ConditionalNode(Node): pass diff --git a/wptrunner/wptmanifest/parser.py b/wptrunner/wptmanifest/parser.py index 28f9666ac559ce..eeac66d31801d2 100644 --- a/wptrunner/wptmanifest/parser.py +++ b/wptrunner/wptmanifest/parser.py @@ -44,6 +44,9 @@ def __init__(self, filename, line, detail): operators = ["==", "!=", "not", "and", "or"] +atoms = {"True": True, + "False": False, + "Reset": object()} def decode(byte_str): return byte_str.decode("utf8") @@ -55,7 +58,7 @@ def precedence(operator_node): class TokenTypes(object): def __init__(self): - for type in ["group_start", "group_end", "paren", "list_start", "list_end", "separator", "ident", "string", "number", "eof"]: + for type in ["group_start", "group_end", "paren", "list_start", "list_end", "separator", "ident", "string", "number", "atom", "eof"]: setattr(self, type, type) token_types = TokenTypes() @@ -232,6 +235,8 @@ def list_value_start_state(self): self.state = self.eol_state elif self.char() == ",": raise ParseError(self.filename, self.line_number, "List item started with separator") + elif self.char() == "@": + self.state = self.list_value_atom_state else: self.state = self.list_value_state @@ -267,6 +272,11 @@ def list_value_state(self): if rv: yield (token_types.string, decode(rv)) + def list_value_atom_state(self): + self.consume() + for _, value in self.list_value_state(): + yield token_types.atom, value + def list_end_state(self): self.consume() yield (token_types.list_end, "]") @@ -282,29 +292,36 @@ def value_state(self): self.state = self.comment_state else: self.state = self.line_end_state + elif self.char() == "@": + self.consume() + for _, value in self.value_inner_state(): + yield token_types.atom, value else: - rv = "" - spaces = 0 - while True: - c = self.char() - if c == "\\": - rv += self.consume_escape() - elif c == "#": - self.state = self.comment_state - break - elif c == " ": - # prevent whitespace before comments from being included in the value - spaces += 1 - self.consume() - elif c == eol: - self.state = self.line_end_state - break - else: - rv += " " * spaces - spaces = 0 - rv += c - self.consume() - yield (token_types.string, decode(rv)) + self.state = self.value_inner_state + + def value_inner_state(self): + rv = "" + spaces = 0 + while True: + c = self.char() + if c == "\\": + rv += self.consume_escape() + elif c == "#": + self.state = self.comment_state + break + elif c == " ": + # prevent whitespace before comments from being included in the value + spaces += 1 + self.consume() + elif c == eol: + self.state = self.line_end_state + break + else: + rv += " " * spaces + spaces = 0 + rv += c + self.consume() + yield (token_types.string, decode(rv)) def comment_state(self): while self.char() is not eol: @@ -544,13 +561,18 @@ def value_block(self): if self.token[0] == token_types.string: self.value() self.eof_or_end_group() + elif self.token[0] == token_types.atom: + self.atom() else: raise ParseError def list_value(self): self.tree.append(ListNode()) - while self.token[0] == token_types.string: - self.value() + while self.token[0] in (token_types.atom, token_types.string): + if self.token[0] == token_types.atom: + self.atom() + else: + self.value() self.expect(token_types.list_end) self.tree.pop() @@ -571,6 +593,13 @@ def value(self): self.consume() self.tree.pop() + def atom(self): + if self.token[1] not in atoms: + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Unrecognised symbol @%s" % self.token[1]) + self.tree.append(AtomNode(atoms[self.token[1]])) + self.consume() + self.tree.pop() + def expr_start(self): self.expr_builder = ExpressionBuilder(self.tokenizer) self.expr_builders.append(self.expr_builder) @@ -605,21 +634,21 @@ def expr_operand(self): elif self.token[0] == token_types.number: self.expr_number() else: - raise ParseError + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Unrecognised operand") def expr_unary_op(self): if self.token[1] in unary_operators: self.expr_builder.push_operator(UnaryOperatorNode(self.token[1])) self.consume() else: - raise ParseError(self.filename, self.tokenizer.line_number, "Expected unary operator") + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Expected unary operator") def expr_bin_op(self): if self.token[1] in binary_operators: self.expr_builder.push_operator(BinaryOperatorNode(self.token[1])) self.consume() else: - raise ParseError(self.filename, self.tokenizer.line_number, "Expected binary operator") + raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Expected binary operator") def expr_value(self): node_type = {token_types.string: StringNode, diff --git a/wptrunner/wptmanifest/serializer.py b/wptrunner/wptmanifest/serializer.py index d87ca0f3be90f3..efa839d8dbf45c 100644 --- a/wptrunner/wptmanifest/serializer.py +++ b/wptrunner/wptmanifest/serializer.py @@ -3,7 +3,9 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from node import NodeVisitor, ValueNode, ListNode, BinaryExpressionNode -from parser import precedence +from parser import atoms, precedence + +atom_names = {v:"@%s" % k for (k,v) in atoms.iteritems()} named_escapes = set(["\a", "\b", "\f", "\n", "\r", "\t", "\v"]) @@ -80,6 +82,9 @@ def visit_ValueNode(self, node): quote = "" return [quote + escape(node.data, extras=quote) + quote] + def visit_AtomNode(self, node): + return [atom_names[node.data]] + def visit_ConditionalNode(self, node): return ["if %s: %s" % tuple(self.visit(item)[0] for item in node.children)] diff --git a/wptrunner/wptmanifest/tests/test_parser.py b/wptrunner/wptmanifest/tests/test_parser.py index 1f197c37443811..6e8e6e6bea934b 100644 --- a/wptrunner/wptmanifest/tests/test_parser.py +++ b/wptrunner/wptmanifest/tests/test_parser.py @@ -67,5 +67,13 @@ def test_expr_1(self): ]]]]]] ) + def test_atom_0(self): + with self.assertRaises(parser.ParseError): + self.parse("key: @Unknown") + + def test_atom_1(self): + with self.assertRaises(parser.ParseError): + self.parse("key: @true") + if __name__ == "__main__": unittest.main() diff --git a/wptrunner/wptmanifest/tests/test_serializer.py b/wptrunner/wptmanifest/tests/test_serializer.py index 236b56cc7fcc27..ec4d6e2d7373d4 100644 --- a/wptrunner/wptmanifest/tests/test_serializer.py +++ b/wptrunner/wptmanifest/tests/test_serializer.py @@ -209,3 +209,19 @@ def test_escape_10(self): def test_escape_11(self): self.compare(r"""key: \\ab """) + + def test_atom_1(self): + self.compare(r"""key: @True +""") + + def test_atom_2(self): + self.compare(r"""key: @False +""") + + def test_atom_3(self): + self.compare(r"""key: @Reset +""") + + def test_atom_4(self): + self.compare(r"""key: [a, @Reset, b] +""") diff --git a/wptrunner/wpttest.py b/wptrunner/wpttest.py index a6067db014b35c..635b1a2298e022 100644 --- a/wptrunner/wpttest.py +++ b/wptrunner/wpttest.py @@ -9,6 +9,9 @@ import mozinfo +from wptmanifest.parser import atoms + +atom_reset = atoms["Reset"] class Result(object): def __init__(self, status, message, expected=None, extra=None): @@ -88,26 +91,24 @@ class Test(object): subtest_result_cls = None test_type = None - def __init__(self, url, expected_metadata, timeout=DEFAULT_TIMEOUT, path=None, + def __init__(self, url, inherit_metadata, test_metadata, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): self.url = url - self._expected_metadata = expected_metadata + self._inherit_metadata = inherit_metadata + self._test_metadata = test_metadata self.timeout = timeout self.path = path - if expected_metadata: - prefs = expected_metadata.prefs() - else: - prefs = [] - self.environment = {"protocol": protocol, "prefs": prefs} + self.environment = {"protocol": protocol, "prefs": self.prefs} def __eq__(self, other): return self.id == other.id @classmethod - def from_manifest(cls, manifest_item, expected_metadata): + def from_manifest(cls, manifest_item, inherit_metadata, test_metadata): timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT return cls(manifest_item.url, - expected_metadata, + inherit_metadata, + test_metadata, timeout=timeout, path=manifest_item.path, protocol="https" if hasattr(manifest_item, "https") and manifest_item.https else "http") @@ -121,30 +122,57 @@ def id(self): def keys(self): return tuple() - def _get_metadata(self, subtest): - if self._expected_metadata is None: - return None - - if subtest is not None: - metadata = self._expected_metadata.get_subtest(subtest) + def _get_metadata(self, subtest=None): + if self._test_metadata is not None and subtest is not None: + return self._test_metadata.get_subtest(subtest) else: - metadata = self._expected_metadata - return metadata + return self._test_metadata + + def itermeta(self, subtest=None): + for metadata in self._inherit_metadata: + yield metadata + + if self._test_metadata is not None: + yield self._get_metadata() + if subtest is not None: + subtest_meta = self._get_metadata(subtest) + if subtest_meta is not None: + yield subtest_meta - def disabled(self, subtest=None): - metadata = self._get_metadata(subtest) - if metadata is None: - return False - return metadata.disabled() + def disabled(self, subtest=None): + for meta in self.itermeta(subtest): + disabled = meta.disabled + if disabled is not None: + return disabled + return None + @property def tags(self): - tags = set(["dir:%s" % self.id.lstrip("/").split("/")[0]]) - metadata = self._get_metadata(None) - if metadata is None: - return tags + tags = set() + for meta in self.itermeta(): + meta_tags = meta.tags + if atom_reset in meta_tags: + tags = meta_tags.copy() + tags.remove(atom_reset) + else: + tags |= meta_tags - return tags | metadata.tags() + tags.add("dir:%s" % self.id.lstrip("/").split("/")[0]) + + return tags + + @property + def prefs(self): + prefs = {} + for meta in self.itermeta(): + meta_prefs = meta.prefs + if atom_reset in prefs: + prefs = meta_prefs.copy() + del prefs[atom_reset] + else: + prefs.update(meta_prefs) + return prefs def expected(self, subtest=None): if subtest is None: @@ -184,8 +212,8 @@ class ReftestTest(Test): result_cls = ReftestResult test_type = "reftest" - def __init__(self, url, expected, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): - Test.__init__(self, url, expected, timeout, path, protocol) + def __init__(self, url, inherit_metadata, test_metadata, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): + Test.__init__(self, url, inherit_metadata, test_metadata, timeout, path, protocol) for _, ref_type in references: if ref_type not in ("==", "!="): @@ -196,7 +224,8 @@ def __init__(self, url, expected, references, timeout=DEFAULT_TIMEOUT, path=None @classmethod def from_manifest(cls, manifest_test, - expected_metadata, + inherit_metadata, + test_metadata, nodes=None, references_seen=None): @@ -210,7 +239,8 @@ def from_manifest(cls, url = manifest_test.url node = cls(manifest_test.url, - expected_metadata, + inherit_metadata, + test_metadata, [], timeout=timeout, path=manifest_test.path, @@ -235,11 +265,12 @@ def from_manifest(cls, manifest_node = manifest_test.manifest.get_reference(ref_url) if manifest_node: reference = ReftestTest.from_manifest(manifest_node, + [], None, nodes, references_seen) else: - reference = ReftestTest(ref_url, None, []) + reference = ReftestTest(ref_url, [], None, []) node.references.append((reference, ref_type)) @@ -259,7 +290,7 @@ def keys(self): "manual": ManualTest} -def from_manifest(manifest_test, expected_metadata): +def from_manifest(manifest_test, inherit_metadata, test_metadata): test_cls = manifest_test_cls[manifest_test.item_type] - return test_cls.from_manifest(manifest_test, expected_metadata) + return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata) From 6084bbfecc78d96831c532f0e24b2fe1b2bd61c4 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 27 May 2015 16:53:29 +0100 Subject: [PATCH 3/4] Use context manager for switching to CONTEXT_CHROME --- wptrunner/executors/executormarionette.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/wptrunner/executors/executormarionette.py b/wptrunner/executors/executormarionette.py index 495132827e81d2..527a6f6071c5f3 100644 --- a/wptrunner/executors/executormarionette.py +++ b/wptrunner/executors/executormarionette.py @@ -177,25 +177,22 @@ def set_pref(self, name, value): break; } """ % (name, value) - self.marionette.set_context(self.marionette.CONTEXT_CHROME) - self.marionette.execute_script(script) - self.marionette.set_context(self.marionette.CONTEXT_CONTENT) + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(script) def clear_user_pref(self, name): self.logger.info("Clearing pref %s" % (name)) - self.marionette.set_context(self.marionette.CONTEXT_CHROME) script = """ let prefInterface = Components.classes["@mozilla.org/preferences-service;1"] .getService(Components.interfaces.nsIPrefBranch); let pref = '%s'; prefInterface.clearUserPref(pref); """ % name - self.marionette.execute_script(script) - self.marionette.set_context(self.marionette.CONTEXT_CONTENT) + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(script) def get_pref(self, name): - self.marionette.set_context(self.marionette.CONTEXT_CHROME) - self.marionette.execute_script(""" + script = """ let prefInterface = Components.classes["@mozilla.org/preferences-service;1"] .getService(Components.interfaces.nsIPrefBranch); let pref = '%s'; @@ -210,8 +207,9 @@ def get_pref(self, name): case prefInterface.PREF_INVALID: return null; } - """ % (name)) - self.marionette.set_context(self.marionette.CONTEXT_CONTENT) + """ % name + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.marionette.execute_script(script) class MarionetteRun(object): def __init__(self, logger, func, marionette, url, timeout): From b24c241b070ce9e84a124508afeda1d8fb18a0ed Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 3 Jun 2015 16:42:31 +0100 Subject: [PATCH 4/4] Fix review issues --- test/test.py | 2 +- wptrunner/executors/executormarionette.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test.py b/test/test.py index ed3a92a6e32fa1..6d90ba9355578b 100644 --- a/test/test.py +++ b/test/test.py @@ -141,7 +141,7 @@ def get_parser(): help="Specific product to include in test run") parser.add_argument("--pdb", action="store_true", help="Invoke pdb on uncaught exception") - parser.add_argument("--tag", action="append", + parser.add_argument("--tag", action="append", dest="tags", help="tags to select tests") parser.add_argument("test", nargs="*", help="Specific tests to include in test run") diff --git a/wptrunner/executors/executormarionette.py b/wptrunner/executors/executormarionette.py index 527a6f6071c5f3..975c1391455afe 100644 --- a/wptrunner/executors/executormarionette.py +++ b/wptrunner/executors/executormarionette.py @@ -139,7 +139,7 @@ def wait(self): def on_environment_change(self, old_environment, new_environment): #Unset all the old prefs - for name, _ in old_environment.get("prefs", {}).iteritems(): + for name in old_environment.get("prefs", {}).iterkeys(): value = self.executor.original_pref_values[name] if value is None: self.clear_user_pref(name) @@ -151,11 +151,13 @@ def on_environment_change(self, old_environment, new_environment): self.set_pref(name, value) def set_pref(self, name, value): - if value not in ("true", "false"): + if value.lower() not in ("true", "false"): try: int(value) except ValueError: value = "'%s'" % value + else: + value = value.lower() self.logger.info("Setting pref %s (%s)" % (name, value))