From 2a8caff2582996cb500037a0c84a7898ab11f0db Mon Sep 17 00:00:00 2001 From: mizzick Date: Tue, 17 Apr 2018 00:29:47 +0700 Subject: [PATCH 1/2] #917 filter directives --- Extension/browser/chrome/background.html | 1 + Extension/browser/firefox/background.html | 1 + .../browser/firefox_webext/background.html | 1 + Extension/browser/safari/background.html | 1 + Extension/lib/filter/antibanner.js | 6 +- Extension/lib/libs/filter-downloader.js | 384 ++++++++++++++++++ Extension/lib/utils/service-client.js | 68 +--- 7 files changed, 408 insertions(+), 54 deletions(-) create mode 100644 Extension/lib/libs/filter-downloader.js diff --git a/Extension/browser/chrome/background.html b/Extension/browser/chrome/background.html index f0fb720345..e3972e8eb0 100644 --- a/Extension/browser/chrome/background.html +++ b/Extension/browser/chrome/background.html @@ -6,6 +6,7 @@ + diff --git a/Extension/browser/firefox/background.html b/Extension/browser/firefox/background.html index f75a48e608..b4828d01db 100644 --- a/Extension/browser/firefox/background.html +++ b/Extension/browser/firefox/background.html @@ -6,6 +6,7 @@ + diff --git a/Extension/browser/firefox_webext/background.html b/Extension/browser/firefox_webext/background.html index 67ac7e7b45..5f08075769 100644 --- a/Extension/browser/firefox_webext/background.html +++ b/Extension/browser/firefox_webext/background.html @@ -6,6 +6,7 @@ + diff --git a/Extension/browser/safari/background.html b/Extension/browser/safari/background.html index 3ec94d16b5..96d0d31f84 100644 --- a/Extension/browser/safari/background.html +++ b/Extension/browser/safari/background.html @@ -6,6 +6,7 @@ + diff --git a/Extension/lib/filter/antibanner.js b/Extension/lib/filter/antibanner.js index 5c0caf11f4..dac417c78e 100644 --- a/Extension/lib/filter/antibanner.js +++ b/Extension/lib/filter/antibanner.js @@ -1004,8 +1004,8 @@ adguard.antiBannerService = (function (adguard) { callback(true); }; - var errorCallback = function (request, cause) { - adguard.console.error("Error retrieved response from server for filter {0}, cause: {1} {2}", filter.filterId, request.statusText, cause || ""); + var errorCallback = function (cause) { + adguard.console.error("Error retrieved response from server for filter {0}, cause: {1}", filter.filterId, cause || ""); delete filter._isDownloading; adguard.listeners.notifyListeners(adguard.listeners.ERROR_DOWNLOAD_FILTER, filter); callback(false); @@ -1606,7 +1606,7 @@ adguard.filters = (function (adguard) { var rules = adguard.userrules.addRules(rulesText); loadCallback(rules.length); }, function (request, cause) { - adguard.console.error("Error download subscription by url {0}, cause: {1} {2}", subscriptionUrl, request.statusText, cause || ""); + adguard.console.error("Error download subscription by url {0}, cause: {1}", subscriptionUrl, cause || ""); }); } }; diff --git a/Extension/lib/libs/filter-downloader.js b/Extension/lib/libs/filter-downloader.js new file mode 100644 index 0000000000..47297c9523 --- /dev/null +++ b/Extension/lib/libs/filter-downloader.js @@ -0,0 +1,384 @@ +/** + * filters-downloader - Compiles filters source files + * @version v1.0.0 + * @link http://adguard.com + */ +/** + * This file is part of Adguard Browser Extension (https://github.com/AdguardTeam/AdguardBrowserExtension). + * + * Adguard Browser Extension is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Adguard Browser Extension is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Adguard Browser Extension. If not, see . + */ + +/* global URL */ +/** + * The utility tool resolves preprocessor directives in filter content. + * + * Directives syntax: + * !#if, !#endif - filters maintainers can use these conditions to supply different rules depending on the ad blocker type. + * condition - just like in some popular programming languages, pre-processor conditions are based on constants declared by ad blockers. Ad blocker authors define on their own what exact constants do they declare. + * !#include - this directive allows to include contents of a specified file into the filter. + * + * Condition constants should be declared in FilterCompilerConditionsConstants + * + * More details: + * https://github.com/AdguardTeam/AdguardBrowserExtension/issues/917 + */ +const FilterDownloader = (() => { + "use strict"; + + const CONDITION_DIRECTIVE_START = "!#if"; + const CONDITION_DIRECTIVE_END = "!#endif"; + + const CONDITION_OPERATOR_NOT = "!"; + const CONDITION_OPERATOR_AND = "&&"; + const CONDITION_OPERATOR_OR = "||"; + const CONDITION_BRACKET_OPEN_CHAR = "("; + const CONDITION_BRACKET_CLOSE_CHAR = ")"; + + const INCLUDE_DIRECTIVE = "!#include"; + + const REGEXP_ABSOLUTE_URL = /^([a-z]+:\/\/|\/\/)/i; + + /** + * Checks brackets in string + * + * @param str + */ + const checkBracketsBalance = (str) => { + let depth = 0; + for (let i in str) { + if (str[i] === CONDITION_BRACKET_OPEN_CHAR) { + // if the char is an opening parenthesis then we increase the depth + depth++; + } else if (str[i] === CONDITION_BRACKET_CLOSE_CHAR) { + // if the char is an closing parenthesis then we decrease the depth + depth--; + } + // if the depth is negative we have a closing parenthesis + // before any matching opening parenthesis + if (depth < 0) { + return false; + } + } + // If the depth is not null then a closing parenthesis is missing + if (depth > 0) { + return false; + } + + return true; + }; + + /** + * Finds end of condition block started with startIndex + * + * @param rules + * @param startIndex + */ + const findConditionEnd = (rules, startIndex) => { + for (let j = startIndex; j < rules.length; j++) { + let internalRule = rules[j]; + + if (internalRule.startsWith(CONDITION_DIRECTIVE_START)) { + throw new Error('Invalid directives: Nested conditions are not supported: ' + internalRule); + } + + if (internalRule.startsWith(CONDITION_DIRECTIVE_END)) { + return j; + } + } + + return -1; + }; + + /** + * Resolves constant expression + * + * @param expression + * @param definedProperties + */ + const resolveConditionConstant = (expression, definedProperties) => { + if (!expression) { + throw new Error('Invalid directives: Empty condition'); + } + + let trim = expression.trim(); + return trim === "true" || definedProperties[trim]; + }; + + /** + * Calculates conditional expression + * + * @param expression + * @param definedProperties + */ + const resolveExpression = (expression, definedProperties) => { + if (!expression) { + throw new Error('Invalid directives: Empty condition'); + } + + expression = expression.trim(); + + if (!checkBracketsBalance(expression)) { + throw new Error('Invalid directives: Incorrect brackets: ' + expression); + } + + //Replace bracketed expressions + const openBracketIndex = expression.lastIndexOf(CONDITION_BRACKET_OPEN_CHAR); + if (openBracketIndex !== -1) { + const endBracketIndex = expression.indexOf(CONDITION_BRACKET_CLOSE_CHAR, openBracketIndex); + const innerExpression = expression.substring(openBracketIndex + 1, endBracketIndex); + const innerResult = resolveExpression(innerExpression, definedProperties); + const resolvedInner = expression.substring(0, openBracketIndex) + + innerResult + expression.substring(endBracketIndex + 1); + + return resolveExpression(resolvedInner, definedProperties); + } + + let result; + + // Resolve logical operators + const indexOfAndOperator = expression.indexOf(CONDITION_OPERATOR_AND); + const indexOfOrOperator = expression.indexOf(CONDITION_OPERATOR_OR); + const indexOfNotOperator = expression.indexOf(CONDITION_OPERATOR_NOT); + + if (indexOfOrOperator !== -1) { + result = resolveExpression(expression.substring(0, indexOfOrOperator - 1), definedProperties) || + resolveExpression(expression.substring(indexOfOrOperator + CONDITION_OPERATOR_OR.length, expression.length), definedProperties); + } else if (indexOfAndOperator !== -1) { + result = resolveExpression(expression.substring(0, indexOfAndOperator - 1), definedProperties) && + resolveExpression(expression.substring(indexOfAndOperator + CONDITION_OPERATOR_AND.length, expression.length), definedProperties); + } else if (indexOfNotOperator === 0) { + result = !resolveExpression(expression.substring(CONDITION_OPERATOR_NOT.length), definedProperties); + } else { + result = resolveConditionConstant(expression, definedProperties); + } + + return result; + }; + + /** + * Validates and resolves condition directive + * + * @param directive + * @param definedProperties + */ + const resolveCondition = (directive, definedProperties) => { + const expression = directive.substring(CONDITION_DIRECTIVE_START.length).trim(); + + return resolveExpression(expression, definedProperties); + }; + + /** + * Resolves conditions directives + * + * @param rules + * @param definedProperties + */ + const resolveConditions = (rules, definedProperties) => { + let result = []; + + for (let i = 0; i < rules.length; i++) { + let rule = rules[i]; + + if (rule.indexOf(CONDITION_DIRECTIVE_START) === 0) { + let endLineIndex = findConditionEnd(rules, i + 1); + if (endLineIndex === -1) { + throw new Error('Invalid directives: Condition end not found: ' + rule); + } + + let conditionValue = resolveCondition(rule, definedProperties); + if (conditionValue) { + result = result.concat(rules.slice(i + 1, endLineIndex)); + } + + // Skip to the end of block + i = endLineIndex; + } else if (rule.indexOf(CONDITION_DIRECTIVE_END) === 0) { + // Found condition end without start + throw new Error('Invalid directives: Found unexpected condition end: ' + rule); + } else { + result.push(rule); + } + } + + return result; + }; + + /** + * Executes async request + * + * @param url Url + * @param contentType Content type + * @returns {Promise} + */ + const executeRequestAsync = (url, contentType) => { + + return new Promise((resolve, reject) => { + const request = new XMLHttpRequest(); + + try { + request.open('GET', url); + request.setRequestHeader('Content-type', contentType); + request.setRequestHeader('Pragma', 'no-cache'); + request.overrideMimeType(contentType); + request.mozBackgroundRequest = true; + request.onload = function () { + resolve(request); + }; + request.onerror = reject; + request.onabort = reject; + request.ontimeout = reject; + + request.send(null); + } catch (ex) { + reject(ex); + } + }); + }; + + /** + * Validates url to be the same origin with original filterUrl + * + * @param url + * @param filterUrlOrigin + */ + const validateUrl = function (url, filterUrlOrigin) { + if (filterUrlOrigin) { + if (REGEXP_ABSOLUTE_URL.test(url)) { + + // Include url is absolute + let origin = new URL(url).origin; + if (origin !== filterUrlOrigin) { + throw new Error('Include url is rejected with origin: ' + origin); + } + } + } + }; + + /** + * Validates and resolves include directive + * + * @param line + * @param filterOrigin + * @param definedProperties + * @returns {Promise} A promise that returns {string} with rules when if resolved and {Error} if rejected. + */ + const resolveInclude = function (line, filterOrigin, definedProperties) { + if (line.indexOf(INCLUDE_DIRECTIVE) !== 0) { + return Promise.resolve([line]); + } else { + const url = line.substring(INCLUDE_DIRECTIVE.length).trim(); + validateUrl(url, filterOrigin); + + return downloadFilterRules(url, filterOrigin, definedProperties); + } + }; + + /** + * Resolves include directives + * + * @param rules + * @param filterOrigin + * @param definedProperties + * @returns {Promise} A promise that returns {string} with rules when if resolved and {Error} if rejected. + */ + const resolveIncludes = (rules, filterOrigin, definedProperties) => { + const dfds = []; + + for (let rule of rules) { + dfds.push(resolveInclude(rule, filterOrigin, definedProperties)); + } + + return Promise.all(dfds).then((values) => { + let result = []; + values.forEach(function (v) { + result = result.concat(v); + }); + + return result; + }); + }; + + /** + * Compiles filter content + * + * @param {Array} rules Array of strings + * @param {?string} filterOrigin Filter file URL origin or null + * @param {Object} definedProperties An object with the defined properties. These properties might be used in pre-processor directives (`#if`, etc) + * @returns {Promise} A promise that returns {string} with rules when if resolved and {Error} if rejected. + */ + const compile = (rules, filterOrigin, definedProperties) => { + try { + // Resolve 'if' conditions + const resolvedConditionsResult = resolveConditions(rules, definedProperties); + + // Resolve 'includes' directives + return resolveIncludes(resolvedConditionsResult, filterOrigin, definedProperties); + } catch (ex) { + return Promise.reject(ex); + } + }; + + /** + * Downloads filter rules from url + * + * @param {string} url Filter file URL + * @param {?string} filterUrlOrigin Filter file URL origin or null + * @param {Object} definedProperties An object with the defined properties. These properties might be used in pre-processor directives (`#if`, etc) + * @returns {Promise} A promise that returns {string} with rules when if resolved and {Error} if rejected. + */ + const downloadFilterRules = (url, filterUrlOrigin, definedProperties) => { + return executeRequestAsync(url, 'text/plain').then((response) => { + if (response.status !== 200) { + throw new Error("Response status is invalid: " + response.status); + } + + const responseText = response.responseText; + if (!responseText) { + throw new Error("Response is empty"); + } + + const lines = responseText.split(/[\r\n]+/); + return compile(lines, filterUrlOrigin, definedProperties); + }); + }; + + /** + * Downloads a specified filter and interpretes all the pre-processor directives from there. + * + * @param {string} url Filter file URL + * @param {Object} definedProperties An object with the defined properties. These properties might be used in pre-processor directives (`#if`, etc) + * @returns {Promise} A promise that returns {string} with rules when if resolved and {Error} if rejected. + */ + const download = (url, definedProperties) => { + try { + let filterUrlOrigin; + if (url && REGEXP_ABSOLUTE_URL.test(url)) { + filterUrlOrigin = new URL(url).origin; + } else { + filterUrlOrigin = null; + } + + return downloadFilterRules(url, filterUrlOrigin, definedProperties); + } catch (ex) { + return Promise.reject(ex); + } + }; + + return { + compile: compile, + download: download + }; +})(); + diff --git a/Extension/lib/utils/service-client.js b/Extension/lib/utils/service-client.js index deb6a22636..3401920f6b 100644 --- a/Extension/lib/utils/service-client.js +++ b/Extension/lib/utils/service-client.js @@ -15,6 +15,7 @@ * along with Adguard Browser Extension. If not, see . */ +/* global FilterDownloader */ adguard.backend = (function (adguard) { 'use strict'; @@ -129,43 +130,21 @@ adguard.backend = (function (adguard) { }; /** - * Loading subscriptions map + * FilterDownloader constants */ - var loadingSubscriptions = Object.create(null); + var FilterCompilerConditionsConstants = { + adguard: true, + adguard_ext_chromium: adguard.utils.browser.isChromium(), + adguard_ext_firefox: adguard.utils.browser.isFirefoxBrowser(), + adguard_ext_edge: adguard.utils.browser.isEdgeBrowser(), + adguard_ext_safari: adguard.utils.browser.isSafariBrowser(), + adguard_ext_opera: adguard.utils.browser.isOperaBrowser(), + }; /** - * Load filter rules. - * Parse header and rules. - * Response format: - * HEADER - * rule1 - * rule2 - * ... - * ruleN - * - * @param filterId Filter identifier - * @param url Url for loading rules - * @param successCallback Success callback (version, rules) - * @param errorCallback Error callback (response, errorText) - * @private + * Loading subscriptions map */ - function doLoadFilterRules(filterId, url, successCallback, errorCallback) { - - var success = function (response) { - - var responseText = response.responseText; - if (!responseText) { - errorCallback(response, "filter rules missing"); - return; - } - - var lines = responseText.split(/[\r\n]+/); - successCallback(lines); - - }; - - executeRequestAsync(url, "text/plain", success, errorCallback); - } + var loadingSubscriptions = Object.create(null); /** * Executes async request @@ -295,7 +274,7 @@ adguard.backend = (function (adguard) { } } - doLoadFilterRules(filterId, url, successCallback, errorCallback); + FilterDownloader.download(url, FilterCompilerConditionsConstants).then(successCallback, errorCallback); }; /** @@ -312,22 +291,9 @@ adguard.backend = (function (adguard) { } loadingSubscriptions[url] = true; - var success = function (response) { - + var success = function (lines) { delete loadingSubscriptions[url]; - if (response.status !== 200) { - errorCallback(response, "wrong status code: " + response.status); - return; - } - - var responseText = (response.responseText || '').trim(); - if (responseText.length === 0) { - errorCallback(response, "filter rules missing"); - return; - } - - var lines = responseText.split(/[\r\n]+/); if (lines[0].indexOf('[') === 0) { //[Adblock Plus 2.0] lines.shift(); @@ -336,12 +302,12 @@ adguard.backend = (function (adguard) { successCallback(lines); }; - var error = function (request, cause) { + var error = function (cause) { delete loadingSubscriptions[url]; - errorCallback(request, cause); + errorCallback(cause); }; - executeRequestAsync(url, "text/plain", success, error); + FilterDownloader.download(url, FilterCompilerConditionsConstants).then(success, error); }; /** From c58f6e9f6195c51b4310f771f4d8670c0eb624a0 Mon Sep 17 00:00:00 2001 From: mizzick Date: Mon, 23 Apr 2018 23:18:51 +0700 Subject: [PATCH 2/2] #917 filter directives --- Extension/lib/libs/filter-downloader.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Extension/lib/libs/filter-downloader.js b/Extension/lib/libs/filter-downloader.js index 47297c9523..2836844711 100644 --- a/Extension/lib/libs/filter-downloader.js +++ b/Extension/lib/libs/filter-downloader.js @@ -1,6 +1,6 @@ /** * filters-downloader - Compiles filters source files - * @version v1.0.0 + * @version v1.0.1 * @link http://adguard.com */ /** @@ -140,7 +140,7 @@ const FilterDownloader = (() => { const innerExpression = expression.substring(openBracketIndex + 1, endBracketIndex); const innerResult = resolveExpression(innerExpression, definedProperties); const resolvedInner = expression.substring(0, openBracketIndex) + - innerResult + expression.substring(endBracketIndex + 1); + innerResult + expression.substring(endBracketIndex + 1); return resolveExpression(resolvedInner, definedProperties); } @@ -276,7 +276,7 @@ const FilterDownloader = (() => { */ const resolveInclude = function (line, filterOrigin, definedProperties) { if (line.indexOf(INCLUDE_DIRECTIVE) !== 0) { - return Promise.resolve([line]); + return Promise.resolve(line); } else { const url = line.substring(INCLUDE_DIRECTIVE.length).trim(); validateUrl(url, filterOrigin); @@ -302,8 +302,13 @@ const FilterDownloader = (() => { return Promise.all(dfds).then((values) => { let result = []; + values.forEach(function (v) { - result = result.concat(v); + if (Array.isArray(v)) { + result = result.concat(v); + } else { + result.push(v); + } }); return result; @@ -340,7 +345,7 @@ const FilterDownloader = (() => { */ const downloadFilterRules = (url, filterUrlOrigin, definedProperties) => { return executeRequestAsync(url, 'text/plain').then((response) => { - if (response.status !== 200) { + if (response.status !== 200 && response.status !== 0) { throw new Error("Response status is invalid: " + response.status); }