Fix #143: Fix uploads for polish ad-block lists and cookie lists

Fix #131 and #130: Remove cpp dat files from ad-block crx

package.json

@@ -3,7 +3,6 @@
   "version": "1.0.0",
   "description": "Packages component and theme extensions used in the Brave browser",
   "dependencies": {
-    "ad-block": "brave/ad-block",
     "adblock-rs": "^0.2.9",
     "ajv": "^6.10.0",
     "autoplay-whitelist": "github:brave/autoplay-whitelist",
@@ -12,12 +11,12 @@
     "ethereum-remote-client": "^0.1.54",
     "extension-whitelist": "github:brave/extension-whitelist",
     "https-everywhere-builder": "brave/https-everywhere-builder",
+    "recursive-readdir-sync": "^1.0.6",
     "referrer-whitelist": "github:brave/referrer-whitelist",
     "request": "^2.88.0",
     "s3-client": "^4.4.2",
     "s3-node-client": "^4.4.4",
     "speedreader": "github:brave-experiments/SpeedReader",
-    "tracking-protection": "brave/tracking-protection",
     "unzip-crx-3": "^0.2.0"
   },
   "devDependencies": {
@@ -35,14 +34,13 @@
     "clean": "rimraf build/",
     "test": "tap test/*.js",
     "data-files-ad-block-rust": "node scripts/generateAdBlockRustDataFiles.js",
-    "data-files-ad-block": "npm run --prefix ./node_modules/ad-block data-files",
     "data-files-https-everywhere": "npm run --prefix ./node_modules/https-everywhere-builder build",
-    "data-files-tracking-protection": "cd ./node_modules/tracking-protection/scripts && node ./genDataFile",
     "data-files-autoplay-whitelist": "npm run --prefix ./node_modules/autoplay-whitelist data-files",
     "data-files-extension-whitelist": "npm run --prefix ./node_modules/extension-whitelist data-files",
-    "data-files-local-data-files": "npm run data-files-tracking-protection && npm run data-files-autoplay-whitelist && npm run data-files-extension-whitelist",
+    "data-files-local-data-files": "npm run data-files-autoplay-whitelist && npm run data-files-extension-whitelist",
     "data-files-speedreader": "npm run --prefix ./node_modules/speedreader data-files",
     "download-ipfs-daemon": "node scripts/downloadIpfsDaemon.js",
+    "generate-ad-block-manifests": "node scripts/generateManifestForRustAdblock.js",
     "generate-ntp-sponsored-images": "node scripts/generateNTPSponsoredImages.js",
     "generate-ntp-super-referrer": "node scripts/generateNTPSuperReferrer.js",
     "generate-ntp-super-referrer-mapping-table": "node scripts/generateNTPSuperReferrerMappingTable.js",

scripts/generateManifestForRustAdblock.js

@@ -0,0 +1,69 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { lists } = require('adblock-rs')
+const fs = require('fs-extra')
+const mkdirp = require('mkdirp')
+const path = require('path')
+
+var outPath = path.join('build', 'ad-block-updater');
+
+const defaultAdblockComponentId = "cffkpbalmllkdoenhmdmpbkajipdjfam"
+const defaultAdblockBase64PublicKey = 
+    "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs0qzJmHSgIiw7IGFCxij" +
+    "1NnB5hJ5ZQ1LKW9htL4EBOaMJvmqaDs/wfq0nw/goBHWsqqkMBynRTu2Hxxirvdb" +
+    "cugn1Goys5QKPgAvKwDHJp9jlnADWm5xQvPQ4GE1mK1/I3ka9cEOCzPW6GI+wGLi" +
+    "VPx9VZrxHHsSBIJRaEB5Tyi5bj0CZ+kcfMnRTsXIBw3C6xJgCVKISQUkd8mawVvG" +
+    "vqOhBOogCdb9qza5eJ1Cgx8RWKucFfaWWxKLOelCiBMT1Hm1znAoVBHG/blhJJOD" +
+    "5HcH/heRrB4MvrE1J76WF3fvZ03aHVcnlLtQeiNNOZ7VbBDXdie8Nomf/QswbBGa" +
+    "VwIDAQAB"
+
+const generateManifestFile = (name, base64PublicKey, uuid) => {
+  var manifest = "{\n" +
+                 "  \"description\": \"Brave Ad Block Updater extension\",\n" +
+                 "  \"key\": \"" + base64PublicKey + "\",\n" +
+                 "  \"manifest_version\": 2,\n" +
+                 "  \"name\": \"Brave Ad Block Updater (" + name + ")\",\n" +
+                 "  \"version\": \"0.0.0\"\n" +
+                 "}\n"
+
+  let filePath = path.join(outPath, uuid , 'manifest.json')
+  let p = fs.writeFile(filePath, manifest);
+  return p
+}
+
+const generateManifestFileForDefaultAdblock =
+  generateManifestFile.bind(null, 'Default', defaultAdblockBase64PublicKey, 'default')  // eslint-disable-line
+
+const generateManifestFilesForAllRegions = () => {
+  let p = Promise.resolve()
+  new lists('regions').forEach((region) => { // eslint-disable-line
+    p = p.then(generateManifestFile.bind(null, region.title,
+      region.base64_public_key, region.uuid))
+  })
+}
+
+if (!fs.existsSync(outPath)){
+    fs.mkdirSync(outPath);
+}
+
+generateManifestFileForDefaultAdblock()
+  .then(generateManifestFilesForAllRegions)
+  .then(() => {
+    console.log('Thank you for updating the data files, don\'t forget to upload them too!')
+  })
+  .catch((e) => {
+    console.error(`Something went wrong, aborting: ${e}`)
+    process.exit(1)
+  })
+
+process.on('uncaughtException', (err) => {
+  console.error('Caught exception:', err)
+  process.exit(1)
+})
+
+process.on('unhandledRejection', (err) => {
+  console.error('Unhandled rejection:', err)
+  process.exit(1)
+})

scripts/packageComponent.js

@@ -10,11 +10,26 @@ const commander = require('commander')
 const fs = require('fs-extra')
 const mkdirp = require('mkdirp')
 const path = require('path')
+const recursive = require("recursive-readdir-sync");
 const replace = require('replace-in-file')
 const util = require('../lib/util')
 
 const stageFiles = (componentType, datFile, version, outputDir) => {
   let datFileName
+
+  // ad-block components are in the correct folder
+  // we don't need to stage the crx files
+  if(componentType == 'ad-block-updater') {
+    const outputManifest = path.join(outputDir, 'manifest.json')
+    const replaceOptions = {
+      files: outputManifest,
+      from: /0\.0\.0/,
+      to: version
+    }
+    replace.sync(replaceOptions)
+    return;
+  }
+
   if (componentNeedsStraightCopyFromUnpackedDir(componentType)) {
     const originalDir = getManifestsDirByComponentType(componentType)
     console.log('Copy dir:', originalDir, ' to:', outputDir)
@@ -58,8 +73,7 @@ const getDATFileVersionByComponentType = (componentType) => {
     case 'ethereum-remote-client':
       return '0'
     case 'ad-block-updater':
-      return fs.readFileSync(path.join('node_modules', 'ad-block', 'data_file_version.h')).toString()
-        .match(/DATA_FILE_VERSION\s*=\s*(\d+)/)[1]
+      return ''
     case 'https-everywhere-updater':
       return '6.0'
     case 'local-data-files-updater':
@@ -82,7 +96,6 @@ const generateManifestFilesByComponentType = (componentType) => {
       // Provides its own manifest file
       break
     case 'ad-block-updater':
-      childProcess.execSync(`npm run --prefix ${path.join('node_modules', 'ad-block')} manifest-files`)
       break
     case 'https-everywhere-updater':
     case 'local-data-files-updater':
@@ -103,7 +116,7 @@ const getManifestsDirByComponentType = (componentType) => {
     case 'ethereum-remote-client':
       return path.join('node_modules', 'ethereum-remote-client')
     case 'ad-block-updater':
-      return path.join('node_modules', 'ad-block', 'out')
+      return path.join('build', 'ad-block-updater')
     case 'https-everywhere-updater':
     case 'local-data-files-updater':
       // TODO(emerick): Make these work like ad-block
@@ -118,27 +131,29 @@ const getManifestsDirByComponentType = (componentType) => {
 const getNormalizedDATFileName = (datFileName) =>
   datFileName === 'ABPFilterParserData' ||
   datFileName === 'httpse.leveldb' ||
-  datFileName === 'TrackingProtection' ||
-  datFileName === 'StorageTrackingProtection' ||
   datFileName === 'ReferrerWhitelist' ||
   datFileName === 'ExtensionWhitelist' ||
   datFileName === 'Greaselion' ||
   datFileName === 'AutoplayWhitelist' ? 'default' : datFileName
 
 const getOriginalManifest = (componentType, datFileName) => {
+  if (componentType == 'ad-block-updater') {
+    return path.join(getManifestsDirByComponentType(componentType), datFileName, 'manifest.json')
+  }
   return path.join(getManifestsDirByComponentType(componentType), datFileName ? `${datFileName}-manifest.json` : 'manifest.json')
 }
+
 const getDATFileListByComponentType = (componentType) => {
   switch (componentType) {
     case 'ethereum-remote-client':
       return ['']
     case 'ad-block-updater':
-      return fs.readdirSync(path.join('node_modules', 'ad-block', 'out'))
+      return recursive(path.join('build', 'ad-block-updater'))
         .filter(file => {
-          return (path.extname(file) === '.dat' && file !== 'SafeBrowsingData.dat')
+          return (path.extname(file) === '.dat' && !file.includes('test-data'))
         })
         .reduce((acc, val) => {
-          acc.push(path.join('node_modules', 'ad-block', 'out', val))
+          acc.push(path.join(val))
           return acc
         }, [])
     case 'https-everywhere-updater':
@@ -147,9 +162,7 @@ const getDATFileListByComponentType = (componentType) => {
       return [path.join('node_modules', 'autoplay-whitelist', 'data', 'AutoplayWhitelist.dat'),
         path.join('node_modules', 'extension-whitelist', 'data', 'ExtensionWhitelist.dat'),
         path.join('node_modules', 'brave-site-specific-scripts', 'Greaselion.json'),
-        path.join('node_modules', 'referrer-whitelist', 'data', 'ReferrerWhitelist.json'),
-        path.join('node_modules', 'tracking-protection', 'data', 'TrackingProtection.dat'),
-        path.join('node_modules', 'tracking-protection', 'data', 'StorageTrackingProtection.dat')]
+        path.join('node_modules', 'referrer-whitelist', 'data', 'ReferrerWhitelist.json')]
     case 'speedreader-updater':
       return path.join('node_modules', 'speedreader', 'data', 'speedreader-updater.dat').split()
     default:
@@ -158,7 +171,14 @@ const getDATFileListByComponentType = (componentType) => {
 }
 
 const processDATFile = (binary, endpoint, region, componentType, key, datFile) => {
-  const datFileName = getNormalizedDATFileName(path.parse(datFile).name)
+  var datFileName = getNormalizedDATFileName(path.parse(datFile).name)
+  if (componentType == 'ad-block-updater') {
+    datFileName = datFileName.replace("rs-", "")
+    if (datFileName == 'ABPFilterParserData') {
+      datFileName = 'default'
+    }
+  }
+
   const originalManifest = getOriginalManifest(componentType, datFileName)
   const parsedManifest = util.parseManifest(originalManifest)
   const id = util.getIDFromBase64PublicKey(parsedManifest.key)

scripts/uploadRawAdblock.js

@@ -56,9 +56,9 @@ const basePath = path.join('build', 'ad-block-updater')
 const dirs = fs.readdirSync(basePath)
 dirs.forEach((dir) => {
   const dataFileDir = path.join(basePath, dir);
-  if (!fs.statSync(dataFileDir).isDirectory()) {
-    return
-  }
+    if (!fs.statSync(dataFileDir).isDirectory()) {
+      return
+    }
   const dataFilenames = fs.readdirSync(dataFileDir)
   dataFilenames.forEach((filename) => {
     const currentPath = path.join(dataFileDir, filename)