Skip to content

Commit

Permalink
added optional lz4 compression for cache storage (uBlockOrigin/uBlock…
Browse files Browse the repository at this point in the history 
…-issues#141)

Squashed commit of the following:

commit 6a84738
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Mon Aug 6 10:56:44 2018 -0400

    remove remnant of snappyjs and spurious instruction

commit 9a4b709
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Mon Aug 6 09:48:58 2018 -0400

    make cache storage compression optionally available on all platforms

    New advanced setting: `cacheStorageCompression`. Default is `false`.

commit 22ee654
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Sun Aug 5 19:16:26 2018 -0400

    remove Chromium from lz4 experiment

commit ee3e201
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Sun Aug 5 18:52:43 2018 -0400

    import lz4-block-codec.wasm library

commit 883a311
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Sun Aug 5 18:50:46 2018 -0400

    implement storage compression through lz4-wasm [draft]

commit 48d1cca
Merge: 8ae77e6 b34c897
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Sat Aug 4 08:56:51 2018 -0400

    Merge branch 'master' of github.com:gorhill/uBlock into lz4

commit 8ae77e6
Author: Raymond Hill <rhill@raymondhill.net>
Date:   Wed Jul 25 18:17:45 2018 -0400

    experiment with compression
  • Loading branch information
@gorhill
gorhill committed Aug 6, 2018
1 parent b34c897 commit e163080
Show file tree
Hide file tree
Showing 5 changed files with 437 additions and 155 deletions.
266 changes: 142 additions & 124 deletions platform/chromium/vapi-cachestorage.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2016-2018 The uBlock Origin authors
Copyright (C) 2016-present The uBlock Origin authors
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
Expand All @@ -19,7 +19,7 @@
Home: https://github.com/gorhill/uBlock
*/

/* global indexedDB, IDBDatabase */
/* global IDBDatabase, indexedDB */

'use strict';

Expand Down Expand Up @@ -48,15 +48,10 @@ vAPI.cacheStorage = (function() {
}

const STORAGE_NAME = 'uBlock0CacheStorage';
var db;
var pending = [];
let db;
let pendingInitialization;

// prime the db so that it's ready asap for next access.
getDb(noopfn);

return { get, set, remove, clear, getBytesInUse };

function get(input, callback) {
let get = function get(input, callback) {
if ( typeof callback !== 'function' ) { return; }
if ( input === null ) {
return getAllFromDb(callback);
Expand All @@ -71,51 +66,48 @@ vAPI.cacheStorage = (function() {
output = input;
}
return getFromDb(toRead, output, callback);
}
};

function set(input, callback) {
let set = function set(input, callback) {
putToDb(input, callback);
}
};

function remove(key, callback) {
let remove = function remove(key, callback) {
deleteFromDb(key, callback);
}
};

function clear(callback) {
let clear = function clear(callback) {
clearDb(callback);
}
};

function getBytesInUse(keys, callback) {
let getBytesInUse = function getBytesInUse(keys, callback) {
// TODO: implement this
callback(0);
}
};

function genericErrorHandler(error) {
console.error('[%s]', STORAGE_NAME, error);
}

function noopfn() {
}
let api = { get, set, remove, clear, getBytesInUse, error: undefined };

function processPendings() {
var cb;
while ( (cb = pending.shift()) ) {
cb(db);
let genericErrorHandler = function(ev) {
let error = ev.target && ev.target.error;
if ( error && error.name === 'QuotaExceededError' ) {
api.error = error.name;
}
console.error('[%s]', STORAGE_NAME, error && error.name);
};

function noopfn() {
}

function getDb(callback) {
if ( pending === undefined ) {
return callback();
let getDb = function getDb() {
if ( db instanceof IDBDatabase ) {
return Promise.resolve(db);
}
if ( pending.length !== 0 ) {
return pending.push(callback);
if ( db === null ) {
return Promise.resolve(null);
}
if ( db instanceof IDBDatabase ) {
return callback(db);
if ( pendingInitialization !== undefined ) {
return pendingInitialization;
}
pending.push(callback);
if ( pending.length !== 1 ) { return; }
// https://github.com/gorhill/uBlock/issues/3156
// I have observed that no event was fired in Tor Browser 7.0.7 +
// medium security level after the request to open the database was
Expand All @@ -125,111 +117,112 @@ vAPI.cacheStorage = (function() {
// necessary when reading the `error` property because we are not
// allowed to read this propery outside of event handlers in newer
// implementation of IDBRequest (my understanding).
var req;
try {
req = indexedDB.open(STORAGE_NAME, 1);
if ( req.error ) {
console.log(req.error);
req = undefined;
pendingInitialization = new Promise(resolve => {
let req;
try {
req = indexedDB.open(STORAGE_NAME, 1);
if ( req.error ) {
console.log(req.error);
req = undefined;
}
} catch(ex) {
}
} catch(ex) {
}
if ( req === undefined ) {
processPendings();
pending = undefined;
return;
}
req.onupgradeneeded = function(ev) {
req = undefined;
db = ev.target.result;
db.onerror = db.onabort = genericErrorHandler;
var table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' });
table.createIndex('value', 'value', { unique: false });
};
req.onsuccess = function(ev) {
req = undefined;
db = ev.target.result;
db.onerror = db.onabort = genericErrorHandler;
processPendings();
};
req.onerror = req.onblocked = function() {
req = undefined;
console.log(this.error);
processPendings();
pending = undefined;
};
}
if ( req === undefined ) {
pendingInitialization = undefined;
db = null;
resolve(null);
return;
}
req.onupgradeneeded = function(ev) {
req = undefined;
let db = ev.target.result;
db.onerror = db.onabort = genericErrorHandler;
let table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' });
table.createIndex('value', 'value', { unique: false });
};
req.onsuccess = function(ev) {
pendingInitialization = undefined;
req = undefined;
db = ev.target.result;
db.onerror = db.onabort = genericErrorHandler;
resolve(db);
};
req.onerror = req.onblocked = function() {
pendingInitialization = undefined;
req = undefined;
db = null;
console.log(this.error);
resolve(null);
};
});
return pendingInitialization;
};

function getFromDb(keys, store, callback) {
let getFromDb = function(keys, keystore, callback) {
if ( typeof callback !== 'function' ) { return; }
if ( keys.length === 0 ) { return callback(store); }
var gotOne = function() {
if ( keys.length === 0 ) { return callback(keystore); }
let gotOne = function() {
if ( typeof this.result === 'object' ) {
store[this.result.key] = this.result.value;
keystore[this.result.key] = this.result.value;
}
};
getDb(function(db) {
getDb().then(( ) => {
if ( !db ) { return callback(); }
var transaction = db.transaction(STORAGE_NAME);
let transaction = db.transaction(STORAGE_NAME);
transaction.oncomplete =
transaction.onerror =
transaction.onabort = function() {
return callback(store);
};
var table = transaction.objectStore(STORAGE_NAME);
for ( var key of keys ) {
var req = table.get(key);
transaction.onabort = ( ) => callback(keystore);
let table = transaction.objectStore(STORAGE_NAME);
for ( let key of keys ) {
let req = table.get(key);
req.onsuccess = gotOne;
req.onerror = noopfn;
req = undefined;
}
});
}
};

function getAllFromDb(callback) {
let getAllFromDb = function(callback) {
if ( typeof callback !== 'function' ) {
callback = noopfn;
}
getDb(function(db) {
getDb().then(( ) => {
if ( !db ) { return callback(); }
var output = {};
var transaction = db.transaction(STORAGE_NAME);
let keystore = {};
let transaction = db.transaction(STORAGE_NAME);
transaction.oncomplete =
transaction.onerror =
transaction.onabort = function() {
callback(output);
};
var table = transaction.objectStore(STORAGE_NAME),
transaction.onabort = ( ) => callback(keystore);
let table = transaction.objectStore(STORAGE_NAME),
req = table.openCursor();
req.onsuccess = function(ev) {
var cursor = ev.target.result;
let cursor = ev.target.result;
if ( !cursor ) { return; }
output[cursor.key] = cursor.value;
keystore[cursor.key] = cursor.value;
cursor.continue();
};
});
}
};

// https://github.com/uBlockOrigin/uBlock-issues/issues/141
// Mind that IDBDatabase.transaction() and IDBObjectStore.put()
// can throw:
// https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction
// https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put

function putToDb(input, callback) {
let putToDb = function(keystore, callback) {
if ( typeof callback !== 'function' ) {
callback = noopfn;
}
let keys = Object.keys(input);
let keys = Object.keys(keystore);
if ( keys.length === 0 ) { return callback(); }
getDb(function(db) {
getDb().then(( ) => {
if ( !db ) { return callback(); }
let finish = () => {
if ( callback !== undefined ) {
let cb = callback;
callback = undefined;
cb();
}
let finish = ( ) => {
if ( callback === undefined ) { return; }
let cb = callback;
callback = undefined;
cb();
};
try {
let transaction = db.transaction(STORAGE_NAME, 'readwrite');
Expand All @@ -240,47 +233,72 @@ vAPI.cacheStorage = (function() {
for ( let key of keys ) {
let entry = {};
entry.key = key;
entry.value = input[key];
entry.value = keystore[key];
table.put(entry);
entry = undefined;
}
} catch (ex) {
finish();
}
});
}
};

function deleteFromDb(input, callback) {
let deleteFromDb = function(input, callback) {
if ( typeof callback !== 'function' ) {
callback = noopfn;
}
var keys = Array.isArray(input) ? input.slice() : [ input ];
let keys = Array.isArray(input) ? input.slice() : [ input ];
if ( keys.length === 0 ) { return callback(); }
getDb(function(db) {
getDb().then(db => {
if ( !db ) { return callback(); }
var transaction = db.transaction(STORAGE_NAME, 'readwrite');
transaction.oncomplete =
transaction.onerror =
transaction.onabort = callback;
var table = transaction.objectStore(STORAGE_NAME);
for ( var key of keys ) {
table.delete(key);
let finish = ( ) => {
if ( callback === undefined ) { return; }
let cb = callback;
callback = undefined;
cb();
};
try {
let transaction = db.transaction(STORAGE_NAME, 'readwrite');
transaction.oncomplete =
transaction.onerror =
transaction.onabort = finish;
let table = transaction.objectStore(STORAGE_NAME);
for ( let key of keys ) {
table.delete(key);
}
} catch (ex) {
finish();
}
});
}
};

function clearDb(callback) {
let clearDb = function(callback) {
if ( typeof callback !== 'function' ) {
callback = noopfn;
}
getDb(function(db) {
getDb().then(db => {
if ( !db ) { return callback(); }
var req = db.transaction(STORAGE_NAME, 'readwrite')
.objectStore(STORAGE_NAME)
.clear();
req.onsuccess = req.onerror = callback;
let finish = ( ) => {
if ( callback === undefined ) { return; }
let cb = callback;
callback = undefined;
cb();
};
try {
let req = db.transaction(STORAGE_NAME, 'readwrite')
.objectStore(STORAGE_NAME)
.clear();
req.onsuccess = req.onerror = finish;
} catch (ex) {
finish();
}
});
}
};

// prime the db so that it's ready asap for next access.
getDb(noopfn);

return api;
}());

/******************************************************************************/
Loading

0 comments on commit e163080

Please sign in to comment.