[WPT/common/security-features] Merge multiple subtests into one file

This CL merges subtests that only differ in subresource request's
origin/redirection into a single file.

In `/common/security-features/tools/spec.src.json`,
`origin` and `redirection` fields are removed from
`test_file_path_pattern`, so that scenarios that
only differ in subresource request's origin/redirection
result in the same file name.

__eq__() is added to some classes, as this CL adds
some assert()s that indirectly requires the __eq__()s.

Diffs in generated files:

- An array of scenarios is written, instead of single scenario.
- `<title>`, `<link rel>` to spec URLs, and
  `<meta name="description">` are removed,
  because this CL merges multiple files
  with different titles etc. into one file.
- `<meta name="timeout" content="long">` is added,
  as a single test now can run up to 8 subtests.
  (In my local runs, the max exec time for
  a single generated test file was about 2.8 seconds).
- <link rel="author"> is removed, as the git history log
  is more representative.

Subsequent CLs will actually re-generate files:

- https://chromium-review.googlesource.com/c/chromium/src/+/2103589
- https://chromium-review.googlesource.com/c/chromium/src/+/2102931
- https://chromium-review.googlesource.com/c/chromium/src/+/2102932

which will reduce the number of generated tests from 8548 to 1380.

Bug: 906850
Change-Id: I2c3e847dcbfd89311d0e36da5b91c20f9a3f3a34
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1893828
Commit-Queue: Hiroshige Hayashizaki <hiroshige@chromium.org>
Reviewed-by: Mike West <mkwst@chromium.org>
Reviewed-by: Kenichi Ishibashi <bashi@chromium.org>
Reviewed-by: Hiroki Nakagawa <nhiroki@chromium.org>
Cr-Commit-Position: refs/heads/master@{#750912}

common/security-features/tools/generate.py

@@ -127,9 +127,11 @@ def handle_deliveries(policy_deliveries):
     return {"meta": meta, "headers": headers}
 
 
-def generate_selection(spec_directory, test_helper_filenames, spec_json,
-                       selection, spec, test_html_template_basename):
-    test_filename = get_test_filename(spec_directory, spec_json, selection)
+def generate_selection(spec_json, selection):
+    '''
+    Returns a scenario object (with a top-level source_context_list entry,
+    which will be removed in generate_test_file() later).
+    '''
 
     target_policy_delivery = util.PolicyDelivery(selection['delivery_type'],
                                                  selection['delivery_key'],
@@ -164,30 +166,51 @@ def generate_selection(spec_directory, test_helper_filenames, spec_json,
             target_policy_delivery, spec_json['subresource_schema']
             ['supported_delivery_type'][selection['subresource']])
 
+    # Generate per-scenario test description.
+    selection['test_description'] = spec_json[
+        'test_description_template'] % selection
+
+    return selection
+
+
+def generate_test_file(spec_directory, test_helper_filenames,
+                       test_html_template_basename, test_filename, scenarios):
+    '''
+    Generates a test HTML file (and possibly its associated .headers file)
+    from `scenarios`.
+    '''
+
+    # Scenarios for the same file should have the same `source_context_list`,
+    # including the top-level one.
+    # Note: currently, non-top-level source contexts aren't necessarily required
+    # to be the same, but we set this requirement as it will be useful e.g. when
+    # we e.g. reuse a worker among multiple scenarios.
+    for scenario in scenarios:
+        assert (scenario['source_context_list'] == scenarios[0]
+                ['source_context_list'])
+
     # We process the top source context below, and do not include it in
-    # `scenario` field in JavaScript.
-    top_source_context = selection['source_context_list'].pop(0)
+    # the JSON objects (i.e. `scenarios`) in generated HTML files.
+    top_source_context = scenarios[0]['source_context_list'].pop(0)
     assert (top_source_context.source_context_type == 'top')
+    for scenario in scenarios[1:]:
+        assert (scenario['source_context_list'].pop(0) == top_source_context)
 
-    # Adjust the template for the test invoking JS. Indent it to look nice.
-    indent = "\n" + " " * 8
-    selection['scenario'] = dump_test_parameters(selection).replace(
-        "\n", indent)
+    parameters = {}
 
-    selection['test_page_title'] = spec_json['test_page_title_template'] % spec
-    selection['spec_description'] = spec['description']
-    selection['spec_specification_url'] = spec['specification_url']
+    parameters['scenarios'] = dump_test_parameters(scenarios).replace(
+        "\n", "\n" + " " * 8)
 
     test_directory = os.path.dirname(test_filename)
 
-    selection['helper_js'] = ""
+    parameters['helper_js'] = ""
     for test_helper_filename in test_helper_filenames:
-        selection['helper_js'] += '    <script src="%s"></script>\n' % (
+        parameters['helper_js'] += '    <script src="%s"></script>\n' % (
             os.path.relpath(test_helper_filename, test_directory))
-    selection['sanity_checker_js'] = os.path.relpath(
+    parameters['sanity_checker_js'] = os.path.relpath(
         os.path.join(spec_directory, 'generic', 'sanity-checker.js'),
         test_directory)
-    selection['spec_json_js'] = os.path.relpath(
+    parameters['spec_json_js'] = os.path.relpath(
         os.path.join(spec_directory, 'generic', 'spec_json.js'),
         test_directory)
 
@@ -205,11 +228,7 @@ def generate_selection(spec_directory, test_helper_filenames, spec_json,
                                              util.test_root_directory)}
 
     # Adjust the template for the test invoking JS. Indent it to look nice.
-    selection['generated_disclaimer'] = generated_disclaimer.rstrip()
-    selection['test_description'] = spec_json[
-        'test_description_template'] % selection
-    selection['test_description'] = \
-        selection['test_description'].rstrip().replace("\n", "\n" + " " * 33)
+    parameters['generated_disclaimer'] = generated_disclaimer.rstrip()
 
     # Directory for the test files.
     try:
@@ -224,14 +243,14 @@ def generate_selection(spec_directory, test_helper_filenames, spec_json,
             for header in delivery['headers']:
                 f.write('%s: %s\n' % (header, delivery['headers'][header]))
 
-    selection['meta_delivery_method'] = delivery['meta']
+    parameters['meta_delivery_method'] = delivery['meta']
     # Obey the lint and pretty format.
-    if len(selection['meta_delivery_method']) > 0:
-        selection['meta_delivery_method'] = "\n    " + \
-                                            selection['meta_delivery_method']
+    if len(parameters['meta_delivery_method']) > 0:
+        parameters['meta_delivery_method'] = "\n    " + \
+                                            parameters['meta_delivery_method']
 
     # Write out the generated HTML file.
-    util.write_file(test_filename, test_html_template % selection)
+    util.write_file(test_filename, test_html_template % parameters)
 
 
 def generate_test_source_files(spec_directory, test_helper_filenames,
@@ -268,6 +287,10 @@ def generate_test_source_files(spec_directory, test_helper_filenames,
             excluded_selection['delivery_key'] = spec_json['delivery_key']
             exclusion_dict.add(excluded_selection_pattern % excluded_selection)
 
+    # `scenarios[filename]` represents the list of scenario objects to be
+    # generated into `filename`.
+    scenarios = {}
+
     for spec in specification:
         # Used to make entries with expansion="override" override preceding
         # entries with the same |selection_path|.
@@ -295,11 +318,18 @@ def generate_test_source_files(spec_directory, test_helper_filenames,
                 print('Excluding selection:', selection_path)
                 continue
             try:
-                generate_selection(spec_directory, test_helper_filenames,
-                                   spec_json, selection, spec, html_template)
+                test_filename = get_test_filename(spec_directory, spec_json,
+                                                  selection)
+                scenario = generate_selection(spec_json, selection)
+                scenarios[test_filename] = scenarios.get(test_filename,
+                                                         []) + [scenario]
             except util.ShouldSkip:
                 continue
 
+    for filename in scenarios:
+        generate_test_file(spec_directory, test_helper_filenames,
+                           html_template, filename, scenarios[filename])
+
 
 def merge_json(base, child):
     for key in child:

common/security-features/tools/spec.src.json

@@ -1,6 +1,6 @@
 {
   "selection_pattern": "%(source_context_list)s.%(delivery_type)s/%(delivery_value)s/%(subresource)s/%(origin)s.%(redirection)s.%(source_scheme)s",
-  "test_file_path_pattern": "gen/%(source_context_list)s.%(delivery_type)s/%(delivery_value)s/%(subresource)s/%(origin)s.%(redirection)s.%(source_scheme)s.html",
+  "test_file_path_pattern": "gen/%(source_context_list)s.%(delivery_type)s/%(delivery_value)s/%(subresource)s.%(source_scheme)s.html",
   "excluded_tests": [
     {
       // Workers are same-origin only

common/security-features/tools/template/test.debug.html.template

@@ -2,12 +2,8 @@
 %(generated_disclaimer)s
 <html>
   <head>
-    <title>%(test_page_title)s</title>
-    <meta charset='utf-8'>
-    <meta name="description" content="%(spec_description)s">
-    <link rel="author" title="Kristijan Burnik" href="burnik@chromium.org">
-    <link rel="help" href="%(spec_specification_url)s">
-    <meta name="assert" content="%(test_description)s">%(meta_delivery_method)s
+    <meta charset="utf-8">
+    <meta name="timeout" content="long">%(meta_delivery_method)s
     <script src="/resources/testharness.js"></script>
     <script src="/resources/testharnessreport.js"></script>
     <script src="/common/security-features/resources/common.sub.js"></script>
@@ -19,11 +15,10 @@
   <body>
     <script>
       TestCase(
-        %(scenario)s,
-        document.querySelector("meta[name=assert]").content,
+        %(scenarios)s,
         new SanityChecker()
       ).start();
-      </script>
+    </script>
     <div id="log"></div>
   </body>
 </html>

common/security-features/tools/template/test.release.html.template

@@ -2,24 +2,19 @@
 %(generated_disclaimer)s
 <html>
   <head>
-    <title>%(test_page_title)s</title>
-    <meta charset='utf-8'>
-    <meta name="description" content="%(spec_description)s">
-    <link rel="author" title="Kristijan Burnik" href="burnik@chromium.org">
-    <link rel="help" href="%(spec_specification_url)s">
-    <meta name="assert" content="%(test_description)s">%(meta_delivery_method)s
+    <meta charset="utf-8">
+    <meta name="timeout" content="long">%(meta_delivery_method)s
     <script src="/resources/testharness.js"></script>
     <script src="/resources/testharnessreport.js"></script>
     <script src="/common/security-features/resources/common.sub.js"></script>
 %(helper_js)s  </head>
   <body>
     <script>
       TestCase(
-        %(scenario)s,
-        document.querySelector("meta[name=assert]").content,
+        %(scenarios)s,
         new SanityChecker()
       ).start();
-      </script>
+    </script>
     <div id="log"></div>
   </body>
 </html>

common/security-features/tools/util.py

@@ -68,6 +68,9 @@ def __init__(self, delivery_type, key, value):
         self.key = key
         self.value = value
 
+    def __eq__(self, other):
+        return type(self) is type(other) and self.__dict__ == other.__dict__
+
     @classmethod
     def list_from_json(cls, list, target_policy_delivery,
                        supported_delivery_types):
@@ -168,6 +171,9 @@ def __init__(self, source_context_type, policy_deliveries):
         self.source_context_type = source_context_type
         self.policy_deliveries = policy_deliveries
 
+    def __eq__(self, other):
+        return type(self) is type(other) and self.__dict__ == other.__dict__
+
     @classmethod
     def from_json(cls, obj, target_policy_delivery, source_context_schema):
         '''