From 81181d633946dbde15c924a07f4a3ec9b1449d9a Mon Sep 17 00:00:00 2001 From: Johannes Henkel Date: Mon, 23 Nov 2015 12:22:36 -0800 Subject: [PATCH] Open source the feature tests for the AMP Validator. This is still not integrated with the rest of the AMP build, so it's piling onto build.py for now, taking the shortest route to open source this rather than do a lot of internal refactoring first (to be fair even this took some refactoring ...). validator.js: Contains the logic for the test. This uses jasmin and assert.js, the library that comes with NodeJS. I think this library is fairly weak but for this particular purpose it was the easiest to use because it keeps our node dependencies inside Google smaller and the main usage here is to compare with the output of golden files. On a positive note, validator_test.js is run through the closure compiler so we have that going for us. The diff in build.py makes it fairly easy to see what's going on: * CompileValidatorTestMinified runs closure to build validator_test.js with its transitive closure of dependencies, and the actual test method exported. * GenerateValidatorTest refines this into a script with #! line for nodejs at the top, and a relatively trivial snippet of Javascript to invoke the test, with the testdata dir as argument. * RunValidatorTest runs the test. The tests in testdata/feature_tests are structured so that the .html file contains descriptive comments explaining what the test is about, and the .out files contain the validator output for the respective test. --- validator/build.py | 113 ++++++++--- validator/package.json | 3 +- .../testdata/feature_tests/amp_font.html | 44 ++++ validator/testdata/feature_tests/amp_font.out | 1 + .../testdata/feature_tests/bad_viewport.html | 32 +++ .../testdata/feature_tests/bad_viewport.out | 5 + .../testdata/feature_tests/css_length.html | 35 ++++ .../testdata/feature_tests/css_length.out | 1 + .../testdata/feature_tests/dog_doc_type.html | 35 ++++ .../testdata/feature_tests/dog_doc_type.out | 3 + ...plicate_unique_tags_and_wrong_parents.html | 37 ++++ ...uplicate_unique_tags_and_wrong_parents.out | 4 + validator/testdata/feature_tests/empty.html | 0 validator/testdata/feature_tests/empty.out | 10 + .../feature_tests/empty_stylesheet.html | 33 +++ .../feature_tests/empty_stylesheet.out | 1 + .../incorrect_mandatory_style.html | 33 +++ .../incorrect_mandatory_style.out | 3 + .../testdata/feature_tests/lang_attr.html | 32 +++ .../testdata/feature_tests/lang_attr.out | 1 + .../feature_tests/link_meta_values.html | 43 ++++ .../feature_tests/link_meta_values.out | 3 + .../feature_tests/mandatory_dimensions.html | 132 ++++++++++++ .../feature_tests/mandatory_dimensions.out | 20 ++ .../mandatory_style_without_spaces.html | 33 +++ .../mandatory_style_without_spaces.out | 1 + .../minimum_valid_amp.html | 11 +- .../feature_tests/minimum_valid_amp.out | 1 + .../feature_tests/several_errors.html | 37 ++++ .../testdata/feature_tests/several_errors.out | 9 + .../testdata/feature_tests/spec_example.html | 59 ++++++ .../testdata/feature_tests/spec_example.out | 1 + validator/testdata/feature_tests/svg.html | 71 +++++++ validator/testdata/feature_tests/svg.out | 1 + validator/testdata/feature_tests/youtube.html | 44 ++++ validator/testdata/feature_tests/youtube.out | 3 + validator/validator_test.js | 188 ++++++++++++++++++ 37 files changed, 1054 insertions(+), 29 deletions(-) create mode 100644 validator/testdata/feature_tests/amp_font.html create mode 100644 validator/testdata/feature_tests/amp_font.out create mode 100644 validator/testdata/feature_tests/bad_viewport.html create mode 100644 validator/testdata/feature_tests/bad_viewport.out create mode 100644 validator/testdata/feature_tests/css_length.html create mode 100644 validator/testdata/feature_tests/css_length.out create mode 100644 validator/testdata/feature_tests/dog_doc_type.html create mode 100644 validator/testdata/feature_tests/dog_doc_type.out create mode 100644 validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.html create mode 100644 validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.out create mode 100644 validator/testdata/feature_tests/empty.html create mode 100644 validator/testdata/feature_tests/empty.out create mode 100644 validator/testdata/feature_tests/empty_stylesheet.html create mode 100644 validator/testdata/feature_tests/empty_stylesheet.out create mode 100644 validator/testdata/feature_tests/incorrect_mandatory_style.html create mode 100644 validator/testdata/feature_tests/incorrect_mandatory_style.out create mode 100644 validator/testdata/feature_tests/lang_attr.html create mode 100644 validator/testdata/feature_tests/lang_attr.out create mode 100644 validator/testdata/feature_tests/link_meta_values.html create mode 100644 validator/testdata/feature_tests/link_meta_values.out create mode 100644 validator/testdata/feature_tests/mandatory_dimensions.html create mode 100644 validator/testdata/feature_tests/mandatory_dimensions.out create mode 100644 validator/testdata/feature_tests/mandatory_style_without_spaces.html create mode 100644 validator/testdata/feature_tests/mandatory_style_without_spaces.out rename validator/testdata/{ => feature_tests}/minimum_valid_amp.html (81%) create mode 100644 validator/testdata/feature_tests/minimum_valid_amp.out create mode 100644 validator/testdata/feature_tests/several_errors.html create mode 100644 validator/testdata/feature_tests/several_errors.out create mode 100644 validator/testdata/feature_tests/spec_example.html create mode 100644 validator/testdata/feature_tests/spec_example.out create mode 100644 validator/testdata/feature_tests/svg.html create mode 100644 validator/testdata/feature_tests/svg.out create mode 100644 validator/testdata/feature_tests/youtube.html create mode 100644 validator/testdata/feature_tests/youtube.out create mode 100644 validator/validator_test.js diff --git a/validator/build.py b/validator/build.py index 5ac5c7a5b38e..6a8c8c7ec5cf 100755 --- a/validator/build.py +++ b/validator/build.py @@ -17,6 +17,7 @@ """A build script which (thus far) works on Ubuntu 14.""" +import logging import os import platform import re @@ -36,6 +37,7 @@ def Die(msg): def CheckPrereqs(): """Checks that various prerequisites for this script are satisfied.""" + logging.info('entering ...') if platform.system() != 'Linux': Die('Sorry, this script assumes Linux thus far, e.g. Ubuntu 14. ' @@ -43,7 +45,8 @@ def CheckPrereqs(): # Ensure source files are available. for f in ['validator.protoascii', 'validator.proto', 'validator_gen.py', - 'package.json']: + 'package.json', 'validator.js', 'validator_test.js', + 'validator-in-browser.js', 'tokenize-css.js', 'parse-css.js']: if not os.path.exists(f): Die('%s not found. Must run in amp_validator source directory.' % f) @@ -83,6 +86,7 @@ def CheckPrereqs(): subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) except: Die('Java missing. Try "apt-get install openjdk-7-jre"') + logging.info('... done') def SetupOutDir(out_dir): @@ -92,17 +96,21 @@ def SetupOutDir(out_dir): out_dir: directory name of the output directory. Must not have slashes, dots, etc. """ + logging.info('entering ...') assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir if os.path.exists(out_dir): subprocess.check_call(['rm', '-rf', out_dir]) os.mkdir(out_dir) + logging.info('... done') def InstallNodeDependencies(): + logging.info('entering ...') # Install the project dependencies specified in package.json into # node_modules. subprocess.check_call(['npm', 'install']) + logging.info('... done') def GenValidatorPb2Py(out_dir): @@ -112,11 +120,13 @@ def GenValidatorPb2Py(out_dir): out_dir: directory name of the output directory. Must not have slashes, dots, etc. """ + logging.info('entering ...') assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir subprocess.check_call(['protoc', 'validator.proto', '--python_out=%s' % out_dir]) - open('codegen/__init__.py', 'w').close() + open('%s/__init__.py' % out_dir, 'w').close() + logging.info('... done') def GenValidatorGeneratedJs(out_dir): @@ -126,6 +136,7 @@ def GenValidatorGeneratedJs(out_dir): out_dir: directory name of the output directory. Must not have slashes, dots, etc. """ + logging.info('entering ...') assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir # These imports happen late, within this method because they don't necessarily @@ -142,34 +153,40 @@ def GenValidatorGeneratedJs(out_dir): descriptor=descriptor, out=out) out.append('') - f = open('codegen/validator-generated.js', 'w') + f = open('%s/validator-generated.js' % out_dir, 'w') f.write('\n'.join(out)) f.close() + logging.info('... done') + + +def CompileWithClosure(js_files, closure_entry_points, output_file): + cmd = ['java', '-jar', 'node_modules/google-closure-compiler/compiler.jar', + '--language_in=ECMASCRIPT6_STRICT', '--language_out=ES5_STRICT', + '--js_output_file=%s' % output_file, + '--only_closure_dependencies'] + cmd += ['--closure_entry_point=%s' % e for e in closure_entry_points] + cmd += ['node_modules/google-closure-library/closure/**.js', + '!node_modules/google-closure-library/closure/**_test.js', + 'node_modules/google-closure-library/third_party/closure/**.js', + '!node_modules/google-closure-library/third_party/closure/**_test.js'] + cmd += js_files + subprocess.check_call(cmd) def CompileValidatorMinified(out_dir): - GOOG = 'node_modules/google-closure-library/closure/goog/' - subprocess.check_call([ - 'node_modules/google-closure-library/closure/bin/build/closurebuilder.py', - '--output_mode=compiled', - '--compiler_jar=node_modules/google-closure-compiler/compiler.jar', - '--root=node_modules/google-closure-library/closure', - '--root=node_modules/google-closure-library/third_party/closure', - '--output_file=codegen/validator_minified.js', - '--input=codegen/validator-generated.js', - '--input=validator-in-browser.js', - '--input=validator.js', - '--compiler_flags=--language_in=ECMASCRIPT6_STRICT', - '--compiler_flags=--language_out=ES5_STRICT', - 'htmlparser.js', - 'parse-css.js', - 'tokenize-css.js', - 'codegen/validator-generated.js', - 'validator-in-browser.js', - 'validator.js']) + logging.info('entering ...') + CompileWithClosure( + js_files=['htmlparser.js', 'parse-css.js', 'tokenize-css.js', + '%s/validator-generated.js' % out_dir, + 'validator-in-browser.js', 'validator.js'], + closure_entry_points=['amp.validator.validateString', + 'amp.validator.renderValidationResult'], + output_file='%s/validator_minified.js' % out_dir) + logging.info('... done') def GenerateValidateBin(out_dir): + logging.info('entering ...') f = open('%s/validate' % out_dir, 'w') f.write('#!/usr/bin/nodejs\n') for l in open('%s/validator_minified.js' % out_dir): @@ -208,11 +225,14 @@ def GenerateValidateBin(out_dir): } """) os.chmod('%s/validate' % out_dir, 0750) + logging.info('... done') def RunSmokeTest(out_dir): + logging.info('entering ...') # Run codegen/validate on the minimum valid amp and observe that it passes. - p = subprocess.Popen(['codegen/validate', 'testdata/minimum_valid_amp.html'], + p = subprocess.Popen(['%s/validate' % out_dir, + 'testdata/feature_tests/minimum_valid_amp.html'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() if ('PASS\n', '', p.returncode) != (stdout, stderr, 0): @@ -221,15 +241,56 @@ def RunSmokeTest(out_dir): # Run codegen/validate on an empty file and observe that it fails. open('%s/empty.html' % out_dir, 'w').close() - p = subprocess.Popen(['codegen/validate', '%s/empty.html' % out_dir], + p = subprocess.Popen(['%s/validate' % out_dir, '%s/empty.html' % out_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() if p.returncode != 1: Die('smoke test failed. Expected p.returncode==1, saw: %s' % p.returncode) if not stderr.startswith('FAIL\nempty.html:1:0 MANDATORY_TAG_MISSING'): Die('smoke test failed; stderr was: "%s"' % stdout) + logging.info('... done') + + +def CompileValidatorTestMinified(out_dir): + logging.info('entering ...') + CompileWithClosure( + js_files=['htmlparser.js', 'parse-css.js', 'tokenize-css.js', + '%s/validator-generated.js' % out_dir, + 'validator-in-browser.js', 'validator.js', 'validator_test.js'], + closure_entry_points=['amp.validator.validatorTest'], + output_file='%s/validator_test_minified.js' % out_dir) + logging.info('... success') + + +def GenerateValidatorTest(out_dir): + logging.info('entering ...') + f = open('%s/validator_test' % out_dir, 'w') + f.write('#!/usr/bin/nodejs\n') + for l in open('%s/validator_test_minified.js' % out_dir): + f.write(l) + f.write(""" + var assert = require('assert'); + var fs = require('fs'); + var path = require('path'); + var JasmineRunner = require('jasmine'); + var jasmine = new JasmineRunner(); + amp.validator.validatorTest(['testdata'], assert, fs, path, describe, it); + jasmine.onComplete(function (passed) { process.exit(passed ? 0 : 1); }); + jasmine.execute(); + """) + os.chmod('%s/validator_test' % out_dir, 0750) + logging.info('... success') + + +def RunValidatorTest(out_dir): + logging.info('entering ...') + # Run codegen/validate on the minimum valid amp and observe that it passes. + subprocess.check_call(['%s/validator_test' % out_dir]) + logging.info('... success') +logging.basicConfig(format='[[%(filename)s %(funcName)s]] - %(message)s', + level=logging.INFO) CheckPrereqs() InstallNodeDependencies() SetupOutDir(out_dir='codegen') @@ -238,4 +299,6 @@ def RunSmokeTest(out_dir): CompileValidatorMinified(out_dir='codegen') GenerateValidateBin(out_dir='codegen') RunSmokeTest(out_dir='codegen') -print 'Success - codegen/validate built and tested.' +CompileValidatorTestMinified(out_dir='codegen') +GenerateValidatorTest(out_dir='codegen') +RunValidatorTest(out_dir='codegen') diff --git a/validator/package.json b/validator/package.json index 1c7c62af79d3..b3b855cdf710 100644 --- a/validator/package.json +++ b/validator/package.json @@ -14,6 +14,7 @@ "dependencies": {}, "devDependencies": { "google-closure-compiler": "20151015.0.0", - "google-closure-library": "20151015.0.0" + "google-closure-library": "20151015.0.0", + "jasmine": "2.3.2" } } diff --git a/validator/testdata/feature_tests/amp_font.html b/validator/testdata/feature_tests/amp_font.html new file mode 100644 index 000000000000..68c266ea01ea --- /dev/null +++ b/validator/testdata/feature_tests/amp_font.html @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + diff --git a/validator/testdata/feature_tests/amp_font.out b/validator/testdata/feature_tests/amp_font.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/amp_font.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/bad_viewport.html b/validator/testdata/feature_tests/bad_viewport.html new file mode 100644 index 000000000000..eb6978c65ff6 --- /dev/null +++ b/validator/testdata/feature_tests/bad_viewport.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + +Hello, world. + + diff --git a/validator/testdata/feature_tests/bad_viewport.out b/validator/testdata/feature_tests/bad_viewport.out new file mode 100644 index 000000000000..c664564808a7 --- /dev/null +++ b/validator/testdata/feature_tests/bad_viewport.out @@ -0,0 +1,5 @@ +FAIL +feature_tests/bad_viewport.html:25:2 DISALLOWED_PROPERTY_IN_ATTR_VALUE content="...foo=..." (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) +feature_tests/bad_viewport.html:25:2 INVALID_PROPERTY_VALUE_IN_ATTR_VALUE content="...minimum-scale=not-a-number..." (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) +feature_tests/bad_viewport.html:25:2 MANDATORY_PROPERTY_MISSING_FROM_ATTR_VALUE content="...width=..." (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) +feature_tests/bad_viewport.html:32:7 MANDATORY_TAG_MISSING viewport declaration (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) diff --git a/validator/testdata/feature_tests/css_length.html b/validator/testdata/feature_tests/css_length.html new file mode 100644 index 000000000000..f5e4affec227 --- /dev/null +++ b/validator/testdata/feature_tests/css_length.html @@ -0,0 +1,35 @@ + + + + + + + + + + + + + +Hello, world. + + diff --git a/validator/testdata/feature_tests/css_length.out b/validator/testdata/feature_tests/css_length.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/css_length.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/dog_doc_type.html b/validator/testdata/feature_tests/dog_doc_type.html new file mode 100644 index 000000000000..6448431872c5 --- /dev/null +++ b/validator/testdata/feature_tests/dog_doc_type.html @@ -0,0 +1,35 @@ + + + + + + + + + + + + +Hello this is dog. + + diff --git a/validator/testdata/feature_tests/dog_doc_type.out b/validator/testdata/feature_tests/dog_doc_type.out new file mode 100644 index 000000000000..710eeed32516 --- /dev/null +++ b/validator/testdata/feature_tests/dog_doc_type.out @@ -0,0 +1,3 @@ +FAIL +feature_tests/dog_doc_type.html:23:0 DISALLOWED_ATTR 🐶 +feature_tests/dog_doc_type.html:35:7 MANDATORY_TAG_MISSING html ⚡ for top-level html (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#ampd) \ No newline at end of file diff --git a/validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.html b/validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.html new file mode 100644 index 000000000000..fd8b9eea7ba3 --- /dev/null +++ b/validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + diff --git a/validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.out b/validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.out new file mode 100644 index 000000000000..f869314aa486 --- /dev/null +++ b/validator/testdata/feature_tests/duplicate_unique_tags_and_wrong_parents.out @@ -0,0 +1,4 @@ +FAIL +feature_tests/duplicate_unique_tags_and_wrong_parents.html:31:2 DUPLICATE_UNIQUE_TAG author stylesheet (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#stylesheets) +feature_tests/duplicate_unique_tags_and_wrong_parents.html:35:0 DISALLOWED_ATTR amp-custom (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) +feature_tests/duplicate_unique_tags_and_wrong_parents.html:35:0 WRONG_PARENT_TAG body > style (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) diff --git a/validator/testdata/feature_tests/empty.html b/validator/testdata/feature_tests/empty.html new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/validator/testdata/feature_tests/empty.out b/validator/testdata/feature_tests/empty.out new file mode 100644 index 000000000000..c07dfdcc9675 --- /dev/null +++ b/validator/testdata/feature_tests/empty.out @@ -0,0 +1,10 @@ +FAIL +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING html ⚡ for top-level html (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#ampd) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING head (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#crps) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING charset utf-8 declaration (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#chrs) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING viewport declaration (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING mandatory style (js enabled) opacity 0 (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING mandatory style (noscript) opacity 1 (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING noscript enclosure for mandatory style (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING body (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#crps) +feature_tests/empty.html:1:0 MANDATORY_TAG_MISSING amphtml engine v1.js script (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#scrpt) diff --git a/validator/testdata/feature_tests/empty_stylesheet.html b/validator/testdata/feature_tests/empty_stylesheet.html new file mode 100644 index 000000000000..9782f2355a9d --- /dev/null +++ b/validator/testdata/feature_tests/empty_stylesheet.html @@ -0,0 +1,33 @@ + + + + + + + + + + + + + +Hello, world. + + diff --git a/validator/testdata/feature_tests/empty_stylesheet.out b/validator/testdata/feature_tests/empty_stylesheet.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/empty_stylesheet.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/incorrect_mandatory_style.html b/validator/testdata/feature_tests/incorrect_mandatory_style.html new file mode 100644 index 000000000000..274839531c6b --- /dev/null +++ b/validator/testdata/feature_tests/incorrect_mandatory_style.html @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + diff --git a/validator/testdata/feature_tests/incorrect_mandatory_style.out b/validator/testdata/feature_tests/incorrect_mandatory_style.out new file mode 100644 index 000000000000..62206679f58d --- /dev/null +++ b/validator/testdata/feature_tests/incorrect_mandatory_style.out @@ -0,0 +1,3 @@ +FAIL +feature_tests/incorrect_mandatory_style.html:28:2 MANDATORY_CDATA_MISSING_OR_INCORRECT mandatory style (js enabled) opacity 0 (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) +feature_tests/incorrect_mandatory_style.html:28:74 MANDATORY_CDATA_MISSING_OR_INCORRECT mandatory style (noscript) opacity 1 (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#opacity) diff --git a/validator/testdata/feature_tests/lang_attr.html b/validator/testdata/feature_tests/lang_attr.html new file mode 100644 index 000000000000..40262903cf48 --- /dev/null +++ b/validator/testdata/feature_tests/lang_attr.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + +Hello, world. + + diff --git a/validator/testdata/feature_tests/lang_attr.out b/validator/testdata/feature_tests/lang_attr.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/lang_attr.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/link_meta_values.html b/validator/testdata/feature_tests/link_meta_values.html new file mode 100644 index 000000000000..df18efe5bf2c --- /dev/null +++ b/validator/testdata/feature_tests/link_meta_values.html @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + +Hello, world. + + diff --git a/validator/testdata/feature_tests/link_meta_values.out b/validator/testdata/feature_tests/link_meta_values.out new file mode 100644 index 000000000000..1929a2e1893c --- /dev/null +++ b/validator/testdata/feature_tests/link_meta_values.out @@ -0,0 +1,3 @@ +FAIL +feature_tests/link_meta_values.html:32:2 INVALID_ATTR_VALUE name=content-disposition (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) +feature_tests/link_meta_values.html:33:2 INVALID_ATTR_VALUE rel=unknown (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#canon) diff --git a/validator/testdata/feature_tests/mandatory_dimensions.html b/validator/testdata/feature_tests/mandatory_dimensions.html new file mode 100644 index 000000000000..16ddbda9300a --- /dev/null +++ b/validator/testdata/feature_tests/mandatory_dimensions.html @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/validator/testdata/feature_tests/mandatory_dimensions.out b/validator/testdata/feature_tests/mandatory_dimensions.out new file mode 100644 index 000000000000..639e8059c0ac --- /dev/null +++ b/validator/testdata/feature_tests/mandatory_dimensions.out @@ -0,0 +1,20 @@ +FAIL +feature_tests/mandatory_dimensions.html:102:4 MANDATORY_ATTR_MISSING width (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:102:4 MANDATORY_ATTR_MISSING height (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:103:4 MANDATORY_ATTR_MISSING width (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:103:4 MANDATORY_ATTR_MISSING height (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:104:4 MANDATORY_ATTR_MISSING height (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:105:4 MANDATORY_ATTR_MISSING width (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:108:4 MANDATORY_ATTR_MISSING src (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/mandatory_dimensions.html:109:4 MANDATORY_ATTR_MISSING src (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-anim/amp-anim.md) +feature_tests/mandatory_dimensions.html:112:4 DISALLOWED_ATTR srcset (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-audio/amp-audio.md) +feature_tests/mandatory_dimensions.html:113:4 DISALLOWED_ATTR srcset (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-ad.md) +feature_tests/mandatory_dimensions.html:116:4 MANDATORY_ATTR_MISSING src or srcdoc (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-iframe/amp-iframe.md) +feature_tests/mandatory_dimensions.html:117:4 MANDATORY_ATTR_MISSING src (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-pixel.md) +feature_tests/mandatory_dimensions.html:120:4 DISALLOWED_ATTR src (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-fit-text/amp-fit-text.md) +feature_tests/mandatory_dimensions.html:121:4 DISALLOWED_ATTR src (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-carousel/amp-carousel.md) +feature_tests/mandatory_dimensions.html:122:4 DISALLOWED_ATTR srcset (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-youtube/amp-youtube.md) +feature_tests/mandatory_dimensions.html:123:4 DISALLOWED_ATTR srcset (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-twitter/amp-twitter.md) +feature_tests/mandatory_dimensions.html:124:4 DISALLOWED_ATTR srcset (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-instagram/amp-instagram.md) +feature_tests/mandatory_dimensions.html:125:4 DISALLOWED_ATTR src (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-lightbox/amp-lightbox.md) +feature_tests/mandatory_dimensions.html:128:4 DISALLOWED_ATTR foo (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) diff --git a/validator/testdata/feature_tests/mandatory_style_without_spaces.html b/validator/testdata/feature_tests/mandatory_style_without_spaces.html new file mode 100644 index 000000000000..0ea112782e7d --- /dev/null +++ b/validator/testdata/feature_tests/mandatory_style_without_spaces.html @@ -0,0 +1,33 @@ + + + + + + + + + + + + +Hello, world. + + diff --git a/validator/testdata/feature_tests/mandatory_style_without_spaces.out b/validator/testdata/feature_tests/mandatory_style_without_spaces.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/mandatory_style_without_spaces.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/minimum_valid_amp.html b/validator/testdata/feature_tests/minimum_valid_amp.html similarity index 81% rename from validator/testdata/minimum_valid_amp.html rename to validator/testdata/feature_tests/minimum_valid_amp.html index 806d7919cb17..71f66774aba3 100644 --- a/validator/testdata/minimum_valid_amp.html +++ b/validator/testdata/feature_tests/minimum_valid_amp.html @@ -1,5 +1,3 @@ - - + + + - + Hello, world. diff --git a/validator/testdata/feature_tests/minimum_valid_amp.out b/validator/testdata/feature_tests/minimum_valid_amp.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/minimum_valid_amp.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/several_errors.html b/validator/testdata/feature_tests/several_errors.html new file mode 100644 index 000000000000..07b9fd2c2dcf --- /dev/null +++ b/validator/testdata/feature_tests/several_errors.html @@ -0,0 +1,37 @@ + + + + + + + + + + + +
Tables are allowed
+ + + + + + diff --git a/validator/testdata/feature_tests/several_errors.out b/validator/testdata/feature_tests/several_errors.out new file mode 100644 index 000000000000..03cc0e379320 --- /dev/null +++ b/validator/testdata/feature_tests/several_errors.out @@ -0,0 +1,9 @@ +FAIL +feature_tests/several_errors.html:23:2 INVALID_ATTR_VALUE charset=pick-a-key (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#chrs) +feature_tests/several_errors.html:26:2 INVALID_ATTR_VALUE type=javascript +feature_tests/several_errors.html:32:2 MANDATORY_ATTR_MISSING width (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/several_errors.html:32:2 MANDATORY_ATTR_MISSING height (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-img.md) +feature_tests/several_errors.html:33:0 DISALLOWED_ATTR made_up_attribute (see https://github.com/ampproject/amphtml/blob/master/builtins/amp-ad.md) +feature_tests/several_errors.html:37:7 MANDATORY_TAG_MISSING charset utf-8 declaration (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#chrs) +feature_tests/several_errors.html:37:7 MANDATORY_TAG_MISSING viewport declaration (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#vprt) +feature_tests/several_errors.html:37:7 MANDATORY_TAG_MISSING amphtml engine v1.js script (see https://github.com/ampproject/amphtml/blob/master/spec/amp-html-format.md#scrpt) diff --git a/validator/testdata/feature_tests/spec_example.html b/validator/testdata/feature_tests/spec_example.html new file mode 100644 index 000000000000..ba73a481be59 --- /dev/null +++ b/validator/testdata/feature_tests/spec_example.html @@ -0,0 +1,59 @@ + + + + + + + Sample document + + + + + + + + + +

Sample document

+

+ Some text + +

+ + + + diff --git a/validator/testdata/feature_tests/spec_example.out b/validator/testdata/feature_tests/spec_example.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/spec_example.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/svg.html b/validator/testdata/feature_tests/svg.html new file mode 100644 index 000000000000..1b4960b14aff --- /dev/null +++ b/validator/testdata/feature_tests/svg.html @@ -0,0 +1,71 @@ + + + + + + + + + + + + +

SVG

+ + + + + + + + + + + + + + + + + + + + + + + + Layer 1 + Oh Hi I'm an SVG. + + + + + + + + + + + + diff --git a/validator/testdata/feature_tests/svg.out b/validator/testdata/feature_tests/svg.out new file mode 100644 index 000000000000..7ef22e9a431a --- /dev/null +++ b/validator/testdata/feature_tests/svg.out @@ -0,0 +1 @@ +PASS diff --git a/validator/testdata/feature_tests/youtube.html b/validator/testdata/feature_tests/youtube.html new file mode 100644 index 000000000000..d54867f6a662 --- /dev/null +++ b/validator/testdata/feature_tests/youtube.html @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/validator/testdata/feature_tests/youtube.out b/validator/testdata/feature_tests/youtube.out new file mode 100644 index 000000000000..3286d97269b7 --- /dev/null +++ b/validator/testdata/feature_tests/youtube.out @@ -0,0 +1,3 @@ +FAIL +feature_tests/youtube.html:38:2 MANDATORY_ATTR_MISSING src or data-videoid or video-id (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-youtube/amp-youtube.md) +feature_tests/youtube.html:40:2 MUTUALLY_EXCLUSIVE_ATTRS src or data-videoid or video-id (see https://github.com/ampproject/amphtml/blob/master/extensions/amp-youtube/amp-youtube.md) diff --git a/validator/validator_test.js b/validator/validator_test.js new file mode 100644 index 000000000000..a413bee762ab --- /dev/null +++ b/validator/validator_test.js @@ -0,0 +1,188 @@ +/** + * @license + * Copyright 2015 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the license. + */ +goog.require('amp.validator.validateString'); +goog.require('amp.validator.renderValidationResult'); +goog.provide('amp.validator.validatorTest'); + +/** + * This function allows us to inject NodeJS dependencies into the test. + * It encloses the contents of the remainder of the file. This + * function gets called from a small NodeJS script (e.g. in our github + * project, see build.py, GenerateValidatorTest). + * @param {!Array} testdataDirs + * @param {?} assert The NodeJS assert module. + * @param {?} fs The NodeJS fs module. + * @param {?} path The NodeJS path module. + * @param {?} describe The describe function from jasmin. + * @param {?} it The it function from jasmin. + * @export + */ +amp.validator.validatorTest = function(testdataDirs, assert, fs, + path, describe, it) { + +/** + * Returns the absolute path for a given test file, that is, a file + * underneath a testdata directory. E.g., 'foo/bar/testdata/baz.html' => + * 'baz.html'. + * @param {!string} testFile + * @return {!string} + */ +function absolutePathFor(testFile) { + for (const dir of testdataDirs) { + const candidate = path.join(dir, testFile); + if (fs.existsSync(candidate)) { + return candidate; + } + } + throw 'Could not find ' + testFile; +} + +/** + * Returns all html files underneath the testdata directories. This does + * not traverse the directories recursively but only one level deep + * (e.g., it will find the 'feature_tests' subdir and the .html files inside it. + * @return {!Array} + */ +function findHtmlFilesRelativeToTestdata() { + const testFiles = []; + for (const dir of testdataDirs) { + for (const subdir of /** @type {!Array} */( + fs.readdirSync(path.join(dir)))) { + for (const candidate of /** @type {!Array} */( + fs.readdirSync(path.join(dir, subdir)))) { + if (candidate.match(/^.*.html/g)) { + testFiles.push(path.join(subdir, candidate)); + } + } + } + } + return testFiles; +} + +/** + * An AMP Validator test case. This constructor will load the AMP HTML file + * and also find the adjacent .out file. + * @constructor + */ +const ValidatorTestCase = function(ampHtmlFile) { + /** @type {!string} */ + this.name = ampHtmlFile; + /** @type {!string} */ + this.ampHtmlFile = ampHtmlFile; + /** + * This field can be null, indicating that the expectedOutput did not + * come from a file. + * @type {?string} + */ + this.expectedOutputFile = path.join( + path.dirname(ampHtmlFile), path.basename(ampHtmlFile, '.html') + '.out'); + /** @type {!string} */ + this.ampHtmlFileContents = fs.readFileSync( + absolutePathFor(this.ampHtmlFile), 'utf8'); + /** @type {!string} */ + this.expectedOutput = fs.readFileSync( + absolutePathFor(this.expectedOutputFile), 'utf8').trim(); +}; + +/** + * Runs the test, by executing the AMP Validator, then comparing its output + * against the golden file content. + */ +ValidatorTestCase.prototype.run = function() { + const results = amp.validator.validateString(this.ampHtmlFileContents); + const observed = amp.validator.renderValidationResult( + results, this.ampHtmlFile).join('\n'); + if (observed === this.expectedOutput) { + return; + } + let message = ''; + if (this.expectedOutputFile != null) { + message = '\n' + this.expectedOutputFile + ':1:0\n'; + } + message += 'expected:\n' + this.expectedOutput + '\nsaw:\n' + observed; + assert.fail('', '', message, ''); +}; + +/** + * A strict comparison between two values. + * Note: Unfortunately assert.strictEqual has some drawbacks, including that + * it truncates the provided arguments (and it's not configurable) and + * with the Closure compiler, it requires a message argument to which + * we'd always have to pass undefined. Too messy, so we roll our own. + */ +function assertStrictEqual(expected, saw) { + assert.ok(expected === saw, 'expected: ' + expected + ' saw: ' + saw); +} + +describe('ValidatorFeatures', () => { + for (const htmlFile of findHtmlFilesRelativeToTestdata()) { + const test = new ValidatorTestCase(htmlFile); + it(test.name, () => { test.run(); }); + } +}); + +describe('ValidatorCssLengthValidation', () => { + // Rather than encoding some really long author stylesheets in + // testcases, which would be difficult to read/verify that the + // testcase is valid, we modify a valid testcase + // (features/css_length.html) designed for this purpose in code. + + // We use a blob of length 10 (both bytes and chars) to make it easy to + // construct stylesheets of any length that we want. + const validStyleBlob = 'h1 {a: b}\n'; + assertStrictEqual(10, validStyleBlob.length); + + it('accepts max bytes with exactly 50000 bytes in author stylesheet', () => { + const maxBytes = Array(5001).join(validStyleBlob); + assertStrictEqual(50000, maxBytes.length); + + const test = new ValidatorTestCase('feature_tests/css_length.html'); + test.ampHtmlFileContents = test.ampHtmlFileContents.replace( + '.replaceme {}', maxBytes); + }); + + it('will not accept 50001 bytes in author stylesheet - one too many', () => { + const oneTooMany = Array(5001).join(validStyleBlob) + ' '; + assertStrictEqual(50001, oneTooMany.length); + const test = new ValidatorTestCase('feature_tests/css_length.html'); + test.ampHtmlFileContents = test.ampHtmlFileContents.replace( + '.replaceme {}', oneTooMany); + test.expectedOutputFile = null; + test.expectedOutput = + 'FAIL\n' + + 'feature_tests/css_length.html:28:2 STYLESHEET_TOO_LONG ' + + 'seen: 50001 bytes, limit: 50000 bytes ' + + '(see https://github.com/ampproject/amphtml/blob/master/spec/' + + 'amp-html-format.md#maximum-size)'; + }); + + it('knows utf8 and rejects file w/ 50002 bytes but 49999 characters', () => { + const multiByteSheet = Array(5000).join(validStyleBlob) + 'h {a: 😺}'; + assertStrictEqual(49999, multiByteSheet.length); // character length + const test = new ValidatorTestCase('feature_tests/css_length.html'); + test.ampHtmlFileContents = test.ampHtmlFileContents.replace( + '.replaceme {}', multiByteSheet); + test.expectedOutputFile = null; + test.expectedOutput = + 'FAIL\n' + + 'feature_tests/css_length.html:28:2 STYLESHEET_TOO_LONG ' + + 'seen: 50002 bytes, limit: 50000 bytes ' + + '(see https://github.com/ampproject/amphtml/blob/master/spec/' + + 'amp-html-format.md#maximum-size)'; + }); +}); +};