URL whitelist filter and baseURL processing

Gruntfile.js

@@ -29,7 +29,7 @@ module.exports = function(grunt) {
     },
     browserify: {
       standalone: {
-        src: [ 'src/<%= pkg.name %>.js' ],
+        src: [ 'src/index-node.js' ],
         dest: 'dist/<%= pkg.name %>.js',
         options: {
           browserifyOptions: {
@@ -48,23 +48,61 @@ module.exports = function(grunt) {
           join_vars: true
         }
       },
-      buildBrowserified: {
-        src: 'dist/<%= pkg.name %>.js',
-        dest: 'dist/<%= pkg.name %>.min-browserified.js'
+      // buildBrowserified: {
+      //   src: 'dist/<%= pkg.name %>.js',
+      //   dest: 'dist/<%= pkg.name %>.min-browserified.js'
+      // },
+      buildUrlOnly: {
+        options: {
+          wrap: 'urlFilters'
+        },
+        src: [
+          'src/lib/hostParser.js',
+          'src/lib/urlFilters.js',
+          'src/lib/urlResolver.js'],
+        dest: 'dist/url-filters-only.min.<%= pkg.version %>.js'
+      },
+      buildXssOnly: {
+        options: {
+          wrap: 'xssFilters'
+        },
+        src: [
+          'src/index-browser.js', 
+          'src/lib/htmlDecode.js', 
+          'src/lib/xssFilters.priv.js', 
+          'src/lib/xssFilters.js'
+        ],
+        dest: 'dist/xss-filters.min.<%= pkg.version %>.js'
       },
       buildMin: {
         options: {
           wrap: 'xssFilters'
         },
-        src: 'src/<%= pkg.name %>.js',
-        dest: 'dist/<%= pkg.name %>.min.js'
+        src: [
+          'src/index-browser.js', 
+          'src/lib/hostParser.js',
+          'src/lib/urlFilters.js',
+          'src/lib/urlResolver.js',
+          'src/lib/htmlDecode.js', 
+          'src/lib/xssFilters.priv.js', 
+          'src/lib/xssFilters.js'
+        ],
+        dest: 'dist/all-filters.min.js'
       },
       buildMinWithVersion: {
         options: {
           wrap: 'xssFilters'
         },
-        src: 'src/<%= pkg.name %>.js',
-        dest: 'dist/<%= pkg.name %>.<%= pkg.version %>.min.js'
+        src: [
+          'src/index-browser.js', 
+          'src/lib/hostParser.js',
+          'src/lib/urlFilters.js',
+          'src/lib/urlResolver.js',
+          'src/lib/htmlDecode.js', 
+          'src/lib/xssFilters.priv.js', 
+          'src/lib/xssFilters.js'
+        ],
+        dest: 'dist/all-filters.min.<%= pkg.version %>.js'
       }
     },
     mocha_istanbul: {
@@ -98,7 +136,7 @@ module.exports = function(grunt) {
   });
 
   grunt.loadNpmTasks('grunt-mocha-istanbul');
-  grunt.loadNpmTasks('grunt-browserify');
+  // grunt.loadNpmTasks('grunt-browserify');
   grunt.loadNpmTasks('grunt-contrib-uglify');
   grunt.loadNpmTasks('grunt-contrib-jshint');
   grunt.loadNpmTasks('grunt-contrib-clean');
@@ -112,7 +150,8 @@ module.exports = function(grunt) {
     testSet.push('dist', 'karma:ci');
 
   grunt.registerTask('test', testSet);
-  grunt.registerTask('dist', ['browserify', 'uglify'])
+  // grunt.registerTask('dist', ['browserify', 'uglify'])
+  grunt.registerTask('dist', 'uglify')
   grunt.registerTask('docs', ['jsdoc']);
   grunt.registerTask('default', ['test', 'dist']);
 

karma.conf.js

@@ -2,7 +2,7 @@ module.exports = function(config) {
 
   if (!process.env.SAUCE_USERNAME || !process.env.SAUCE_ACCESS_KEY) {
     console.warn('No SAUCE credentials found (missing SAUCE_USERNAME and SAUCE_ACCESS_KEY env variables). Skipping SauceLabs testing.');
-    return;
+    // return;
   }
 
   // Browsers to run on Sauce Labs
@@ -103,7 +103,7 @@ module.exports = function(config) {
     // list of files / patterns to load in the browser
     files: [
       'node_modules/expect.js/index.js',
-      'dist/xss-filters.min.js',
+      'dist/all-filters.min.js',
       'tests/polyfills.js',
       'tests/utils.js',
       'tests/unit/*.js'

package.json

@@ -1,6 +1,6 @@
 {
   "name": "xss-filters",
-  "version": "1.2.6",
+  "version": "2.0.0",
   "licenses": [
     {
       "type": "BSD",
@@ -23,7 +23,7 @@
       "email": "albertyu@yahoo-inc.com"
     }
   ],
-  "main": "src/xss-filters.js",
+  "main": "src/index-node.js",
   "scripts": {
     "test": "grunt test",
     "hint": "grunt jshint",
@@ -46,8 +46,8 @@
   ],
   "devDependencies": {
     "expect.js": "^0.3.1",
+    "extend": "^3.0.0",
     "grunt": "^0.4.5",
-    "grunt-browserify": "^3.8.0",
     "grunt-cli": "^0.1.13",
     "grunt-contrib-clean": "^0.6.0",
     "grunt-contrib-copy": "^0.7.0",

src/index-browser.js

@@ -0,0 +1,17 @@
+/*
+Copyright (c) 2015, Yahoo! Inc. All rights reserved.
+Copyrights licensed under the New BSD License.
+See the accompanying LICENSE file for terms.
+
+Authors: Adonis Fung <adon@yahoo-inc.com>
+         Nera Liu <neraliu@yahoo-inc.com>
+         Albert Yu <albertyu@yahoo-inc.com>
+*/
+
+// This file is used only for building the min version with uglify
+/*jshint esnext: true *//*jshint -W079 *//*jshint unused:false*/
+const require = false;
+
+exports._privFilters = {
+	urlFilters: (exports.urlFilters = {})
+};

src/index-node.js

@@ -0,0 +1,38 @@
+/*
+Copyright (c) 2015, Yahoo! Inc. All rights reserved.
+Copyrights licensed under the New BSD License.
+See the accompanying LICENSE file for terms.
+
+Authors: Nera Liu <neraliu@yahoo-inc.com>
+         Adonis Fung <adon@yahoo-inc.com>
+         Albert Yu <albertyu@yahoo-inc.com>
+*/
+/**
+ * The following file serves the node.js version
+ */
+
+/*jshint node: true */
+var extend = require('extend');
+
+// populate the xss filters
+module.exports = exports = require('./lib/xssFilters');
+
+// add yHtmlDecode to _privFilters
+extend(exports._privFilters, require('./lib/htmlDecode'));
+
+// the following is largely designed for secure-handlebars-helpers
+exports._getPrivFilters = {
+	toString : function() {
+        var fs = require('fs');
+        return '(function(){var xssFilters={},exports={};' +
+            fs.readFileSync('./src/lib/htmlDecode.js', 'utf8') +
+            fs.readFileSync('./src/lib/xssFilters.priv.js', 'utf8') +
+            'return exports;})';
+    }
+};
+
+// add urlFilters
+exports.urlFilters = extend(
+	require('./lib/urlFilters'), 
+	require('./lib/hostParser'), 
+	require('./lib/urlResolver'));

src/lib/hostParser.js

@@ -0,0 +1,180 @@
+/*
+Copyright (c) 2015, Yahoo! Inc. All rights reserved.
+Copyrights licensed under the New BSD License.
+See the accompanying LICENSE file for terms.
+
+Authors: Nera Liu <neraliu@yahoo-inc.com>
+         Adonis Fung <adon@yahoo-inc.com>
+         Albert Yu <albertyu@yahoo-inc.com>
+*/
+
+var _urlFilters = exports.urlFilters || exports;
+
+// designed according to https://url.spec.whatwg.org/#percent-decode
+var _reHostInvalidSyntax = /[\x00\t\n\r#%\/:?@\[\\\]]/g,
+    _reUpto4HexDigits = /^[\dA-Fa-f]{1,4}$/,
+    _yUrl256Power = [1, 256, 65536, 16777216, 4294967296],
+    _reValidNumber = /^(?:0[xX][\dA-Fa-f]*|0[0-7]*|(?!0)\d*)$/;
+
+function _yIPv4NumberParser(part) {
+    if (!_reValidNumber.test(part)) return NaN;
+
+    var n, len = part.length;
+    return (len > 1 && part.slice(0, 2).toLowerCase() === '0x') ? (len === 2 ? 0 : parseInt(part.slice(2), 16)) :
+           (len > 1 && part.charCodeAt(0) === 48) ? parseInt(part, 8) : 
+           parseInt(part);
+}
+
+
+_urlFilters.hostParser = function (input, options) {
+    /*jshint -W030 */
+    options || (options = {});
+
+    var FAILURE = null, lastIdx;
+
+    // the order of percent decoding is swapped with ipv6 to follow browser behavior
+    try {
+        // Let domain be the result of utf-8 decode without BOM on the percent
+        //   decoding of utf-8 encode on input.
+        input = decodeURIComponent(input);    
+    } catch(e) { return FAILURE; }
+
+    lastIdx = input.length - 1;
+
+    // trigger ipv6 parser
+    // _reUrlAuthHostsPort has excluded those not ending with ]
+    if (input.charCodeAt(0) === 91) {                          // begins with [
+        return input.charCodeAt(lastIdx) === 93 ?              // ends with ]
+            _urlFilters.ipv6Parser(input.slice(1, lastIdx)) :  // && valid ipv6
+            FAILURE;
+    }
+
+    // Let asciiDomain be the result of running domain to ASCII on domain.
+    // If asciiDomain is failure, return failure.
+    options.IDNAtoASCII && (input = punycode.toASCII(input));
+
+    // If asciiDomain contains one of U+0000, U+0009, U+000A, U+000D, U+0020, 
+    //   "#", "%", "/", ":", "?", "@", "[", "\", and "]", syntax violation, 
+    //   return failure.
+    // We follow this except the space character U+0020
+    if (_reHostInvalidSyntax.test(input)) { return FAILURE; }
+
+    return _urlFilters.ipv4Parser(input);
+}
+
+
+_urlFilters.ipv4Parser = function (input) {
+    var chunks = input.split('.'), 
+        len = chunks.length, 
+        i = 0, 
+        FAILURE = null, INPUT = {'domain': 1, 'host': input.toLowerCase()},
+        outputA = '', outputB = '';
+
+    // If the last item in parts is the empty string, remove the last item from
+    //   parts
+    chunks[len - 1].length === 0 && len--;
+
+    // If parts has more than four items, return input
+    if (len > 4) { return INPUT; }
+
+    // parse the number in every item. 
+    for (; i < len; i++) {
+        // If part is the empty string, return input.
+        //   0..0x300 is a domain, not an IPv4 address.
+        if (chunks[i].length === 0 || 
+            // If n is failure, return input.
+            isNaN((chunks[i] = _yIPv4NumberParser(chunks[i])))) { return INPUT; }
+    }
+
+    // If any but the last item in numbers > 255, return failure
+    for (i = 0; i < len - 1; i++) {
+        if ((n = chunks[i]) > 255) { return FAILURE; }
+        // Use them directly as output
+        outputA += n + '.';
+    }
+
+    // If the last item in numbers is greater than or equal to 256^(5 − the 
+    //   number of items in numbers), syntax violation, return failure.
+    if ((n = chunks[i]) >= _yUrl256Power[5 - len]) { return FAILURE; }
+
+    // IPv4 serializer composes anything after outputA
+    for (i = len - 1; i < 4; i++) {
+        // Prepend n % 256, serialized, to output.
+        outputB = (n % 256) + outputB;
+        // Unless this is the fourth time, prepend "." to output.
+        (i !== 3) && (outputB = '.' + outputB);
+        // Set n to n / 256.
+        n = Math.floor(n / 256);
+    }
+
+    // Return output as {'ipv4': 1, 'host': 'IPv4_ADDRESS'}
+    return {'ipv4': 1, 'host': outputA + outputB};
+}
+
+_urlFilters.ipv6Parser = function(input) {
+    if (input === '::') { return {'ipv6': 1, 'host': '[::]'}; }
+
+    var chunks = input.split(':'), 
+        FAILURE = null, compressPtr = null, compressAtEdge = false,
+        i = 0, len = chunks.length, piece, result;
+
+    // too little or many colons than allowed
+    if (len < 3 || len > 9 || 
+        // start with a single colon (except double-colon)
+        chunks[0].length === 0 && chunks[1].length !== 0) { return FAILURE; }
+
+    // capture as many 4-hex-digits as possible
+    for (; i < len; i++) {
+        piece = chunks[i];
+        // empty indicates a colon is found
+        if (piece.length === 0) {
+            // 2nd colon found
+            if (compressPtr !== null) {
+                // double-colon allowed once, and only at either start or end
+                if (!compressAtEdge && (i === 1 || compressPtr === len - 2)) {
+                    compressAtEdge = true;
+                    continue;
+                }
+                return FAILURE;
+            }
+            // if the input ends with a single colon
+            if (i === len - 1) { return FAILURE; }
+            // point to the first colon position
+            compressPtr = i;
+        } 
+        // check if the piece conform to 4 hex digits
+        else if (_reUpto4HexDigits.test(piece)) {
+            // lowercased, and leading zeros are removed
+            chunks[i] = parseInt(piece, 16).toString(16);
+        }
+        // quit the loop once a piece is found not matching 4 hex digits
+        else { break; }
+    }
+
+    // all pieces conform to the 4-hex pattern
+    if (i === len) {}
+    // only the last one doesn't conform to 4 hex digits
+    //   has at most 6 4-hex pieces (7 is due to the leading compression)
+    //   it must has a dot to trigger ipv4 parsing (excluded number only)
+    //   it's a valid IPv4 address
+    else if (i === len - 1 &&
+            i < (compressAtEdge ? 8 : 7) &&
+            piece.indexOf('.') !== -1 &&
+            (result = _urlFilters.ipv4Parser(piece)) && result.ipv4) {
+        // replace the last piece with two pieces of ipv4 in hexadecimal
+        result = result.host.split('.');
+        chunks[i]     = (result[0] * 0x100 + parseInt(result[1])).toString(16);
+        chunks[len++] = (result[2] * 0x100 + parseInt(result[3])).toString(16);
+    }
+    else { return FAILURE; }
+
+    // insert zero for ipv6 that has 7 chunks plus a compressor 
+    if (compressAtEdge) {
+        --len === 8 && chunks.splice(compressPtr, 2, '0');
+    } else if (len === 8 && compressPtr !== null) {
+        chunks[compressPtr] = '0';
+    }
+
+    // return the input in string if there're less than 8 chunks
+    return len === 9 ? FAILURE : {'ipv6': 1, 'host': '[' + chunks.join(':') + ']'};
+}