Submission/rhauck

tools/coverage/README.md

@@ -7,17 +7,40 @@ In order to update the data on the tests that we have, run:
 
 That will produce `test-data.json`.
 
+test-data.js does not need to be run for CSS specs.
+
 To update the mapping between test data and spec sections (this depends on
 the previous data being correct), run:
 
+    For HTML5 & Canvas:
     node tests-per-section.js
 
 That will produce `tests-per-section.json`.
 
+    For CSS:
+    node test-per-section.js [shortname]
+
+    where [shortname] is the shepherd API name for the spec. The full list of
+    shortnames can be seen in the raw JSON data here:
+    http://test.csswg.org/shepherd/api/coverage
+
+That will produce `[shortname]-tests-per-section.json`.
+
+    To collect all the CSS test data in one json file:
+    node test-per-section.js css-all
+
+That will produce `css-all-tests-per-section.json`.
+
+
 Finally, in order to update the full mapping data (which depends on the previous
 step), run:
 
-    node analyse.specs.js
+    node analyse-specs.js
+
+    For CSS:
+    node analyse-specs.js [shortname]
+
+    where [shortname] is the same as above
 
 That will produce `spec-data-*.json`. Those files are used by the coverage page.
 

tools/coverage/analyse-specs.js

@@ -1,48 +1,80 @@
-
 var fs = require("fs")
 ,   pth = require("path")
 ,   exec = require("child_process").execFile
 ,   phanthomScript = pth.join(__dirname, "get-analysis-for.phjs")
-,   specs = {
-        html:       "http://www.w3.org/html/wg/drafts/html/master/single-page.html"
-    ,   canvas2d:   "http://www.w3.org/html/wg/drafts/2dcontext/html5_canvas/Overview.html"
-    // ,   microdata:  "http://www.w3.org/html/wg/drafts/microdata/master/Overview.html"
+,   allSpecs = {
+		html:       		"http://www.w3.org/html/wg/drafts/html/master/single-page.html"
+,		canvas2d:   		"http://www.w3.org/html/wg/drafts/2dcontext/html5_canvas/Overview.html"
+,		microdata:  		"http://www.w3.org/html/wg/drafts/microdata/master/Overview.html"
+,		css21:      		"http://www.w3.org/TR/CSS21/Overview.html"
+,		css3_animations: 	"http://dev.w3.org/csswg/css3-animations/Overview.html"
+,		css3_background: 	"http://dev.w3.org/csswg/css3-background/Overview.html"
+,		css3_color:  		"http://www.w3.org/TR/css3-color/Overview.html"
+,		css3_fonts:  		"http://dev.w3.org/csswg/css3-fonts/Overview.html"
+,		css3_transforms: 	"http://dev.w3.org/csswg/css3-transforms/Overview.html"
+,		css3_transitions:	"http://dev.w3.org/csswg/css3-transitions/Overview.html"
+,		cssom:      		"http://dev.w3.org/csswg/cssom/Overview.html"
+,		cssom_view:  		"http://dev.w3.org/csswg/cssom-view/Overview.html"
+,		css_device_adapt: 	"http://dev.w3.org/csswg/css-device-adapt/Overview.html"
+,		css3_flexbox:    	"http://www.w3.org/TR/css3-flexbox/Overview.html"
+,		css3_images: 		"http://dev.w3.org/csswg/css3-images/Overview.html"
+,		css3_mediaqueries: 	"http://dev.w3.org/csswg/css3-mediaqueries/Overview.html"
+,		css3_selectors:  	"http://www.w3.org/TR/css3-selectors/Overview.html"
+,		css3_text:   		"http://dev.w3.org/csswg/css3-text/Overview.html"
+,		css3_ui:     		"http://dev.w3.org/csswg/css3-ui/Overview.html"
+,		css3_values: 		"http://dev.w3.org/csswg/css3-values/Overview.html"
     }
-,   perSec = JSON.parse(fs.readFileSync(pth.join(__dirname, "tests-per-section.json"), "utf8"))
+,   specName = "html"
+,   specUrl = allSpecs["html"]
+,   specSectionData = "tests-per-section.json"
 ,   data = {}
 ,   ranges = {}
 ;
 
-function analyseSpec (spec, cb) {
-    exec("/usr/local/bin/phantomjs", [phanthomScript, specs[spec]], { timeout: 30000 }, function (err, stdout, stderr) {
+if(process.argv.length > 2) {
+	specName = process.argv[2];
+	specSectionData = specName +"-tests-per-section.json";
+	specKey = specName.replace(/-/g,'_');
+	specUrl = allSpecs[specKey];
+}
+
+try {
+	perSec = JSON.parse(fs.readFileSync(pth.join(__dirname, specSectionData), "utf8"));
+} catch (err) {
+	console.log("Unable to find " + specSectionData);
+	process.exit(1);
+}
+
+console.log("Launching extraction for " + specUrl);
+data[specName] = [];
+ranges[specName] = {};
+
+analyseSpec(specUrl, specName, function (err, spec) {
+	if (err) return console.log("ERROR: " + err);
+	for (var i = 0, n = data[spec].length; i < n; i++) {
+		var sec = data[spec][i];
+		var tests = perSec[spec];
+        sec.tests = tests[sec.id] || 0;
+	}
+    fs.writeFileSync(pth.join(__dirname, "spec-data-" + spec + ".json"), JSON.stringify(data[spec], null, 4), "utf8");
+});
+
+function analyseSpec (url, name, cb) {
+    exec("/usr/local/bin/phantomjs", [phanthomScript, url], { timeout: 30000 }, function (err, stdout, stderr) {
         if (err || stderr) {
-            console.log("[ERROR] " + (err || stderr) + " (" + specs[spec] + ")");
+            console.log("[ERROR] " + (err || stderr) + " (" + url + ")");
         }
         else {
             try {
                 var res = JSON.parse(stdout);
-                data[spec] = res;
-                console.log(spec + " OK");
+                data[name] = res;
+                console.log(url + " OK");
             }
             catch (e) {
-                console.log("[ERROR] " + e + " (" + spec + ") received: <<<" + stdout + ">>>");
+                console.log("[ERROR] " + e + " (" + url + ") received: <<<" + stdout + ">>>");
             }
         }
-        cb(null, spec);
-    });
-}
-
-for (var spec in specs) {
-    console.log("Launching extraction for " + spec);
-    data[spec] = [];
-    ranges[spec] = {};
-    analyseSpec(spec, function (err, spec) {
-        if (err) return console.log("ERROR: " + err);
-        for (var i = 0, n = data[spec].length; i < n; i++) {
-            var sec = data[spec][i];
-            sec.tests = perSec[spec][sec.id] || 0;
-        }
-        fs.writeFileSync(pth.join(__dirname, "spec-data-" + spec + ".json"), JSON.stringify(data[spec], null, 4), "utf8");
+        cb(null, name);
     });
 }
 

tools/coverage/css-device-adapt-tests-per-section.json

@@ -0,0 +1,32 @@
+{
+    "css-device-adapt": {
+        "introduction": 0,
+        "values": 0,
+        "the-viewport": 0,
+        "the-viewport-rule": 0,
+        "syntax": 0,
+        "viewport-properties": 0,
+        "the-lsquomin-widthrsquo-and-lsquomax-wid": 0,
+        "the-lsquowidthrsquo-shorthand-property": 0,
+        "the-lsquomin-heightrsquo-and-lsquomax-he": 0,
+        "the-lsquoheightrsquo-shorthand-property": 0,
+        "the-lsquozoomrsquo-property": 0,
+        "the-lsquomin-zoomrsquo-property": 0,
+        "the-lsquomax-zoomrsquo-property": 0,
+        "the-lsquouser-zoomrsquo-property": 0,
+        "the-lsquoorientationrsquo-property": 0,
+        "the-lsquoresolutionrsquo-property": 0,
+        "constraining-viewport-property-values": 0,
+        "definitions": 0,
+        "constraining-procedure": 0,
+        "media-queries": 0,
+        "cssom": 0,
+        "conformance": 0,
+        "viewport-meta-element": 0,
+        "ua-stylesheet": 0,
+        "meta-properties": 0,
+        "parsing-algorithm": 0,
+        "translation-into-viewport-properties": 0,
+        "handling-auto-zoom": 0
+    }
+}

tools/coverage/css-index.html

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html lang='en'>
+  <head>
+    <meta charset='utf-8'>
+    <title>Test Suite Coverage Analysis</title>
+    <link rel='stylesheet' href='coverage.css'>
+  </head>
+  <body>
+    <h1>Test Suite Coverage Analysis</h1>
+    <p>
+      The following tables summarise test coverage for the specifications contained in this
+      test suite.
+    </p>
+    <div>
+      <h4>Show levels:</h4>
+      <p>
+        The report analyses sections at a maximal depth of 3. If you want to see an overview
+        at section levels 1 or 2 instead, you can choose those options below.
+      </p>
+      <ul>
+        <li>Level 1 <input type='radio' name='level' value='1'></li>
+        <li>Level 2 <input type='radio' name='level' value='2'></li>
+        <li>Level 3 <input type='radio' name='level' value='3'></li>
+      </ul>
+    </div>
+    <div>
+      <h4>Thresholds:</h4>
+      <p>
+        Picking a good ratio of tests to something (or something to test) is difficult.
+        You can set values for specific aspects below. I assume that at some point we
+        will come up with a more or less definitive set of values to roughly aim for.
+      </p>
+      <ul>
+        <li>Words per test: <input type='number' name='words' size='4' value='10'></li>
+        <li>Tests per RFC2119: <input type='number' name='rfc2119' size='4' value='5'></li>
+        <li>Tests per algorithm step: <input type='number' name='algos' size='4' value='1'></li>
+        <li>Tests per IDL item: <input type='number' name='idl' size='4' value='5'></li>
+      </ul>
+      <button id='update'>Update</button>
+    </div>
+    <div id='output'></div>
+
+    <!-- 
+      - add a filter that can colour the cells based on user-defined thresholds
+      - hide levels
+    -->
+    <script src='http://ajax.googleapis.com/ajax/libs/jquery/1.9.0/jquery.min.js'></script>
+    <script src='coverage.js'></script>
+    <script>
+      cover(["css3-animations", "css3-background",
+              "css3-color", "css3-fonts",
+              "css3-transforms", "css3-transitions",
+              "cssom", "cssom-view",
+              "css-device-adapt", "css3-flexbox",
+              "css3-images", "css3-mediaqueries",
+              "css3-selectors", "css3-text",
+              "css3-values", "css3-ui"]
+          , [ "CSS Animations", "CSS Backgrounds & Borders",
+              "CSS Color Level 3", "CSS Fonts Level 3",
+              "CSS Transforms", "CSS Transitions",
+              "CSSOM", "CSSOM View Module",
+              "CSS Device Adaptation", "CSS Flexible Box Layout",
+              "CSS Image Values and Replaced Content", "CSS Media Queries",
+              "CSS Selectors Level 3", "CSS Text Level 3",
+              "CSS Values and Units", "CSS Basic User Interface Module"]
+          , $("#output"));
+    </script>
+  </body>
+</html>