Skip to content

Commit

Permalink
Merge pull request bitpay#108 from maraoz/feature/realtime-sync-info
Browse files Browse the repository at this point in the history
Feature/realtime sync info
maraoz committed Jan 21, 2014
2 parents 852e390 + 2d176a6 commit 590acff
Showing 5 changed files with 228 additions and 208 deletions.
2 changes: 1 addition & 1 deletion app/controllers/socket.js
Original file line number Diff line number Diff line change
@@ -27,5 +27,5 @@ module.exports.broadcast_address_tx = function(address, tx) {
};

module.exports.broadcastSyncInfo = function(syncInfo) {
ios.sockets.emit('sync', syncInfo);
ios.sockets.in('sync').emit('status', syncInfo);
};
381 changes: 193 additions & 188 deletions lib/HistoricSync.js
Original file line number Diff line number Diff line change
@@ -2,43 +2,39 @@

require('classtool');


function spec() {
var util = require('util');
var RpcClient = require('bitcore/RpcClient').class();
var networks = require('bitcore/networks');
var async = require('async');
var config = require('../config/config');
var Block = require('../app/models/Block');
var Sync = require('./Sync').class();
var sockets = require('../app/controllers/socket.js');
var util = require('util');
var RpcClient = require('bitcore/RpcClient').class();
var networks = require('bitcore/networks');
var async = require('async');
var config = require('../config/config');
var Block = require('../app/models/Block');
var Sync = require('./Sync').class();
var sockets = require('../app/controllers/socket.js');

function HistoricSync(opts) {
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;

var genesisHashReversed = new Buffer(32);
this.network.genesisBlock.hash.copy(genesisHashReversed);
this.genesis = genesisHashReversed.reverse().toString('hex');
this.sync = new Sync(opts);

this.sync = new Sync(opts);

//available status: new / syncing / finished / aborted
this.status = 'new';
this.status = 'new';
this.syncInfo = {};
}

function p() {
var args = [];
Array.prototype.push.apply( args, arguments );

Array.prototype.push.apply(args, arguments);


args.unshift('[historic_sync]');
/*jshint validthis:true */
console.log.apply(this, args);
}

HistoricSync.prototype.init = function(opts,cb) {
HistoricSync.prototype.init = function(opts, cb) {
this.rpc = new RpcClient(config.bitcoind);
this.opts = opts;
this.sync.init(opts, cb);
@@ -48,76 +44,84 @@ function spec() {
this.sync.close();
};


HistoricSync.prototype.showProgress = function() {
var self = this;

var i = self.syncInfo;
var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
if (self.opts.broadcast) {
if (self.opts.shouldBroadcast) {
sockets.broadcastSyncInfo(self.syncInfo);
}
};


HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
var self = this;

// recursion end.
if (!blockHash ) return cb();
if (!blockHash) return cb();

var existed = 0;
var existed = false;
var blockInfo;
var blockObj;

async.series([
// Already got it?
function(c) {
Block.findOne({hash:blockHash}, function(err,block){
if (err) { p(err); return c(err); }
if (block) {
existed =1;
blockObj =block;
}
return c();
});
// Already got it?
function(c) {
Block.findOne({
hash: blockHash
},
//show some (inacurate) status
function(c) {
var step = parseInt(self.syncInfo.blocksToSync / 100);
if (step < 10) step = 10;

if (self.syncInfo.syncedBlocks % step === 1) {
self.showProgress();
function(err, block) {
if (err) {
p(err);
return c(err);
}
if (block) {
existed = true;
blockObj = block;
}
return c();
},
//get Info from RPC
function(c) {

// TODO: if we store prev/next, no need to go to RPC
// if (blockObj && blockObj.nextBlockHash) return c();
});
},
//show some (inacurate) status
function(c) {
if (!self.step) {
var step = parseInt(self.syncInfo.blocksToSync / 100);
if (self.opts.progressStep) {
step = self.opts.progressStep;
}
if (step < 2) step = 2;
self.step = step;
}
if (self.syncInfo.syncedBlocks % self.step === 1) {
self.showProgress();
}
return c();
},
//get Info from RPC
function(c) {

self.rpc.getBlock(blockHash, function(err, ret) {
if (err) return c(err);
// TODO: if we store prev/next, no need to go to RPC
// if (blockObj && blockObj.nextBlockHash) return c();
self.rpc.getBlock(blockHash, function(err, ret) {
if (err) return c(err);

blockInfo = ret;
return c();
});
},
//store it
function(c) {
if (existed) return c();
self.sync.storeBlock(blockInfo.result, function(err) {
blockInfo = ret;
return c();
});
},
//store it
function(c) {
if (existed) return c();
self.sync.storeBlock(blockInfo.result, function(err) {

existed = err && err.toString().match(/E11000/);
existed = err && err.toString().match(/E11000/);

if (err && ! existed) return c(err);
return c();
});
},
/* TODO: Should Start to sync backwards? (this is for partial syncs)
if (err && ! existed) return c(err);
return c();
});
},
/* TODO: Should Start to sync backwards? (this is for partial syncs)
function(c) {
if (blockInfo.result.prevblockhash != current.blockHash) {
@@ -127,176 +131,178 @@ function spec() {
return c();
}
*/
],
function (err){

if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
self.status = 'aborted';
p(self.err);
}
], function(err) {

else {
self.err = null;
self.status = 'syncing';
}
if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
self.status = 'aborted';
p(self.err);
}

if (opts.upToExisting && existed ) {
var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks;
if (diff <= 0) {
self.status = 'finished';
p('DONE. Found existing block: ', blockHash);
return cb(err);
}
else {
self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1;
if ((self.syncInfo.skipped_blocks++ % 1000) === 1 ) {
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks);
}
}
}
else {
self.err = null;
self.status = 'syncing';
}

if (blockEnd && blockEnd === blockHash) {
if (opts.upToExisting && existed) {
var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks;
if (diff <= 0) {
self.status = 'finished';
p('DONE. Found END block: ', blockHash);
p('DONE. Found existing block: ', blockHash);
return cb(err);
}


// Continue
if (blockInfo && blockInfo.result) {
if (! existed) self.syncInfo.syncedBlocks++;
if (opts.prev && blockInfo.result.previousblockhash) {
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
else {
self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1;
if ((self.syncInfo.skipped_blocks++ % 1000) === 1) {
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks);
}

if (opts.next && blockInfo.result.nextblockhash)
return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
}
}

if (blockEnd && blockEnd === blockHash) {
self.status = 'finished';
p('DONE. Found END block: ', blockHash);
return cb(err);
}

// Continue
if (blockInfo && blockInfo.result) {
if (!existed) self.syncInfo.syncedBlocks++;
if (opts.prev && blockInfo.result.previousblockhash) {
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
}

if (opts.next && blockInfo.result.nextblockhash) return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
}
return cb(err);
});
};

HistoricSync.prototype.import_history = function(opts, next) {
var self = this;

var retry_secs = 2;
var retry_secs = 2;

var bestBlock;
var blockChainHeight;

async.series([
function(cb) {
if (opts.destroy) {
p('Deleting DB...');
return self.sync.destroy(cb);
}
function(cb) {
if (opts.destroy) {
p('Deleting DB...');
return self.sync.destroy(cb);
}
return cb();
},
// We are not using getBestBlockHash, because is not available in all clients
function(cb) {
if (!opts.reverse) return cb();

self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
blockChainHeight = res.result;
return cb();
},
// We are not using getBestBlockHash, because is not available in all clients
function(cb) {
if (!opts.reverse) return cb();
});
},
function(cb) {
if (!opts.reverse) return cb();

self.rpc.getBlockHash(blockChainHeight, function(err, res) {
if (err) return cb(err);

self.rpc.getBlockCount(function(err, res) {
bestBlock = res.result;

return cb();
});
},
function(cb) {
// This is only to inform progress.
if (!opts.upToExisting) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
blockChainHeight = res.result;
self.syncInfo.blocksToSync = res.result.blocks;
return cb();
});
},
function(cb) {
if (!opts.reverse) return cb();

self.rpc.getBlockHash(blockChainHeight, function(err, res) {
}
else {
// should be isOrphan = true or null to be more accurate.
Block.count({
isOrphan: null
},
function(err, count) {
if (err) return cb(err);

bestBlock = res.result;

self.syncInfo.blocksToSync = blockChainHeight - count;
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
return cb();
});
},
function(cb) {
// This is only to inform progress.
if (!opts.upToExisting) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
self.syncInfo.blocksToSync = res.result.blocks;
return cb();
});
}
},
], function(err) {

var start, end;
function sync() {
if (opts.reverse) {
start = bestBlock;
end = self.genesis;
opts.prev = true;
}
else {
// should be isOrphan = true or null to be more accurate.
Block.count({ isOrphan: null}, function(err, count) {
if (err) return cb(err);

self.syncInfo.blocksToSync = blockChainHeight - count;
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
return cb();
});
}
},
],
function(err) {
start = self.genesis;
end = null;
opts.next = true;
}

self.syncInfo = util._extend(self.syncInfo, {
start: start,
isStartGenesis: start === self.genesis,
end: end,
isEndGenesis: end === self.genesis,
scanningForward: opts.next,
scanningBackward: opts.prev,
upToExisting: opts.upToExisting,
syncedBlocks: 0,
});

var start, end;
function sync() {
if (opts.reverse) {
start = bestBlock;
end = self.genesis;
opts.prev = true;
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));

self.getPrevNextBlock(start, end, opts, function(err) {
if (err && err.message.match(/ECONNREFUSED/)) {
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
sync();
},
retry_secs * 1000);
}
else {
start = self.genesis;
end = null;
opts.next = true;
}

self.syncInfo = util._extend(self.syncInfo, {
start: start,
isStartGenesis: start === self.genesis,
end: end,
isEndGenesis: end === self.genesis,
scanningForward: opts.next,
scanningBackward: opts.prev,
upToExisting: opts.upToExisting,
syncedBlocks: 0,
});

p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));

self.getPrevNextBlock( start, end, opts , function(err) {
if (err && err.message.match(/ECONNREFUSED/)){
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
sync();
}, retry_secs * 1000);
}
else
return next(err);
});
}
else return next(err);
});
}

if (err) {
self.syncInfo = util._extend(self.syncInfo, { error: err.message });
return next(err, 0);
}
else {
sync();
}
if (err) {
self.syncInfo = util._extend(self.syncInfo, {
error: err.message
});
return next(err, 0);
}
else {
sync();
}
});
};

// upto if we have genesis block?
HistoricSync.prototype.smart_import = function(next) {
var self = this;

Block.findOne({hash:self.genesis}, function(err, b){
Block.findOne({
hash: self.genesis
},
function(err, b) {

if (err) return next(err);


if (!b) {
p('Could not find Genesis block. Running FULL SYNC');
}
@@ -305,15 +311,14 @@ function spec() {
}

var opts = {
reverse: 1,
reverse: true,
upToExisting: b ? true: false,
};

return self.import_history(opts, next);
});
};


return HistoricSync;
}
module.defineClass(spec);
50 changes: 32 additions & 18 deletions public/js/controllers/status.js
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
'use strict';

angular.module('insight.status').controller('StatusController',
function ($scope, $routeParams, $location, $rootScope, Global, Status, Sync) {
function($scope, $routeParams, $location, $rootScope, Global, Status, Sync, get_socket) {
$scope.global = Global;

$scope.getStatus = function(q) {
Status.get({
q: 'get' + q
}, function(d) {
q: 'get' + q
},
function(d) {
$rootScope.infoError = null;
angular.extend($scope, d);
}, function(e) {
},
function(e) {
if (e.status === 503) {
$rootScope.infoError = 'Backend Error. ' + e.data;
}
@@ -20,24 +22,36 @@ angular.module('insight.status').controller('StatusController',
});
};

var on_sync_update = function(sync) {
if (sync.blocksToSync > sync.syncedBlocks) {
var p = parseInt(100*(sync.syncedBlocks) / sync.blocksToSync);
var delta = sync.blocksToSync - sync.syncedBlocks;
sync.message = 'Sync ' + p + '% ['+delta+' blocks remaining]';
sync.style = 'warn';
} else {
sync.message = 'On sync';
sync.style = 'success';
}
sync.tooltip = 'Synced blocks: '+sync.syncedBlocks;
$scope.sync = sync;
};

$scope.getSync = function() {
Sync.get({}, function(sync) {
Sync.get({},
function(sync) {
$rootScope.syncError = null;

if (sync.blocksToSync > sync.syncedBlocks ) {
sync.message = 'Blocks to sync: ' + (sync.blocksToSync - sync.syncedBlocks);
sync.tooltip = 'Skipped blocks:' + sync.skipped_blocks;
}
else {
sync.message = 'On sync';
sync.tooltip = '';
}


$scope.sync = sync;
}, function(e) {
on_sync_update(sync);
},
function(e) {
$rootScope.syncError = 'Could not get sync information' + e;
});
};

var socket = get_socket($scope);
socket.emit('subscribe', 'sync');
socket.on('status', function(sync) {
on_sync_update(sync);
});

});

2 changes: 1 addition & 1 deletion public/views/header.html
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@
<div class="status" data-ng-controller="StatusController">
<span data-ng-init="getSync()">
<span class="small" tooltip="{{sync.tooltip}}" tooltip-placement="down">
<i><strong> {{sync.message}} </strong></i>
<i class="{{sync.style}}"><strong> {{sync.message}} </strong></i>
</span>
</span>
<span data-ng-init="getStatus('Info')">
1 change: 1 addition & 0 deletions server.js
Original file line number Diff line number Diff line change
@@ -60,6 +60,7 @@ if (!config.disableHistoricSync) {
historicSync.init({
skipDbConnection: true,
shouldBroadcast: true,
progressStep: 2,
networkName: config.network
}, function() {
historicSync.smart_import(function(err){

0 comments on commit 590acff

Please sign in to comment.