From 742445eb30f6552ddf9722d1250b8cf78b93f02a Mon Sep 17 00:00:00 2001 From: 4miners Date: Sun, 7 May 2017 03:15:48 +0200 Subject: [PATCH 01/88] Add missing bracket --- modules/blocks/chain.js | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index 4f7e61c1ccf..931bdf078d1 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -581,6 +581,7 @@ Chain.prototype.deleteLastBlock = function (cb) { // Replace last block with previous lastBlock = modules.blocks.lastBlock.set(newLastBlock); return setImmediate(seriesCb); + } }); } }, function (err) { From d676cddc7a68b3de587c355dcc344405b57d7ce2 Mon Sep 17 00:00:00 2001 From: 4miners Date: Sun, 7 May 2017 03:18:52 +0200 Subject: [PATCH 02/88] Drop not needed 'async.series' --- modules/blocks/chain.js | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index 931bdf078d1..c85d29a2391 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -566,25 +566,17 @@ Chain.prototype.deleteLastBlock = function (cb) { library.logger.warn('Deleting last block', lastBlock); if (lastBlock.height === 1) { - return setImmediate(cb, 'Can not delete genesis block'); + return setImmediate(cb, 'Cannot delete genesis block'); } - // FIXME: Not need async.series here - async.series({ - // Delete last block, replace last block with previous block, undo things - popLastBlock: function (seriesCb) { - __private.popLastBlock(lastBlock, function (err, newLastBlock) { - if (err) { - library.logger.error('Error deleting last block', lastBlock); - return setImmediate(seriesCb, err); - } else { - // Replace last block with previous - lastBlock = modules.blocks.lastBlock.set(newLastBlock); - return setImmediate(seriesCb); - } - }); + // Delete last block, replace last block with previous block, undo things + __private.popLastBlock(lastBlock, function (err, newLastBlock) { + if (err) { + library.logger.error('Error deleting last block', lastBlock); + } else { + // Replace last block with previous + lastBlock = modules.blocks.lastBlock.set(newLastBlock); } - }, function (err) { return setImmediate(cb, err, lastBlock); }); }; From 7d90dc84855f7c5b14aef373ee7ee53ca8e4da18 Mon Sep 17 00:00:00 2001 From: 4miners Date: Sun, 7 May 2017 06:59:56 +0200 Subject: [PATCH 03/88] Replace table 'rounds_fees' with 'rounds_rewards', add support for exceptions --- sql/blocks.js | 12 +- .../29170507001337_roundsRewards.sql | 153 ++++++++++++++++++ 2 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 sql/migrations/29170507001337_roundsRewards.sql diff --git a/sql/blocks.js b/sql/blocks.js index 8432bfa217b..7003b6dc821 100644 --- a/sql/blocks.js +++ b/sql/blocks.js @@ -30,19 +30,11 @@ var BlocksSql = { 'WITH', 'delegate AS (SELECT', '1 FROM mem_accounts m WHERE m."isDelegate" = 1 AND m."publicKey" = DECODE(${generatorPublicKey}, \'hex\') LIMIT 1),', - 'rewards AS (SELECT COUNT(1) AS count, SUM(reward) AS rewards FROM blocks WHERE "generatorPublicKey" = DECODE(${generatorPublicKey}, \'hex\')', - (params.start !== undefined ? ' AND timestamp >= ${start}' : ''), - (params.end !== undefined ? ' AND timestamp <= ${end}' : ''), - '),', - 'fees AS (SELECT SUM(fees) AS fees FROM rounds_fees WHERE "publicKey" = DECODE(${generatorPublicKey}, \'hex\')', + 'rewards AS (SELECT COUNT(1) AS count, SUM(reward) AS rewards, SUM(fees) AS fees FROM rounds_rewards WHERE pk = DECODE(${generatorPublicKey}, \'hex\')', (params.start !== undefined ? ' AND timestamp >= ${start}' : ''), (params.end !== undefined ? ' AND timestamp <= ${end}' : ''), ')', - 'SELECT', - '(SELECT * FROM delegate) AS delegate,', - '(SELECT count FROM rewards) AS count,', - '(SELECT fees FROM fees) AS fees,', - '(SELECT rewards FROM rewards) AS rewards' + 'SELECT (SELECT * FROM delegate) AS delegate, * FROM rewards' ].filter(Boolean).join(' '); }, diff --git a/sql/migrations/29170507001337_roundsRewards.sql b/sql/migrations/29170507001337_roundsRewards.sql new file mode 100644 index 00000000000..07095fa5100 --- /dev/null +++ b/sql/migrations/29170507001337_roundsRewards.sql @@ -0,0 +1,153 @@ +/* + * Delete table 'rounds_fees', related functions and triggers + * Create table 'rounds_exceptions', calculate rewards & populate it, set triggers + */ + +BEGIN; + +CREATE TABLE IF NOT EXISTS rounds_exceptions ( + "round" INT PRIMARY KEY, + "rewards_factor" INT NOT NULL, + "fees_factor" INT NOT NULL, + "fees_bonus" BIGINT NOT NULL +); + +-- Add exception for round 27040 +INSERT INTO rounds_exceptions VALUES (27040, 2, 2, 10000000); + +-- Drop existing table, triggers and functions +DROP TABLE IF EXISTS rounds_fees; +DROP FUNCTION IF EXISTS rounds_fees_init(); +DROP TRIGGER IF EXISTS rounds_fees_delete ON "blocks"; +DROP FUNCTION IF EXISTS round_fees_delete(); +DROP TRIGGER IF EXISTS rounds_fees_insert ON "blocks"; +DROP FUNCTION IF EXISTS round_fees_insert(); + +-- Create table 'rounds_rewards' for storing rewards +CREATE TABLE IF NOT EXISTS "rounds_rewards"( + "height" INT NOT NULL, + "timestamp" INT NOT NULL, + "fees" BIGINT NOT NULL, + "reward" BIGINT NOT NULL, + "pk" BYTEA NOT NULL +); + +-- Create function that compute all rewards for previous rounds and insert them to 'rounds_rewards' +CREATE FUNCTION rounds_rewards_init() RETURNS void LANGUAGE PLPGSQL AS $$ + DECLARE + row record; + BEGIN + RAISE NOTICE 'Calculating rewards for rounds, please wait...'; + FOR row IN + SELECT + -- Round number + CEIL(height / 101::float)::int AS round + FROM blocks + -- Perform only for rounds that are completed and not present in 'rounds_rewards' + WHERE height % 101 = 0 AND height NOT IN (SELECT height FROM rounds_rewards) + -- Group by round + GROUP BY CEIL(height / 101::float)::int + -- Order by round + ORDER BY CEIL(height / 101::float)::int ASC + LOOP + WITH + -- Selecting all blocks of round, apply exception fees and rewards factor + round AS ( + SELECT + b.timestamp, b.height, b."generatorPublicKey" AS pk, b."totalFee" * COALESCE(e.fees_factor, 1) AS fees, + b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb + FROM blocks b + LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round + WHERE CEIL(b.height / 101::float)::int = row.round + ), + -- Calculating total fees of round, apply exception fees bonus + fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), + -- Get last delegate and timestamp of round's last block + last AS (SELECT pk, timestamp FROM round ORDER BY height DESC LIMIT 1) + INSERT INTO rounds_rewards + SELECT + -- Block height + round.height, + -- Timestamp of last round's block + last.timestamp, + -- Calculating real fee reward for delegate: + -- Rounded fee per delegate + remaining fees if block is last one of round + (fees.single + (CASE WHEN last.pk = round.pk AND last.timestamp = round.timestamp THEN (fees.total - fees.single * 101) ELSE 0 END)) AS fees, + -- Block reward + round.reward, + -- Delegate public key + round.pk + FROM last, fees, round + -- Sort fees by block height + ORDER BY round.height ASC; + END LOOP; + RETURN; +END $$; + +-- Execute 'rounds_rewards_init' function +SELECT rounds_rewards_init(); + +-- Create function for deleting round rewards when last block of round is deleted +CREATE FUNCTION round_rewards_delete() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + DELETE FROM rounds_rewards WHERE CEIL(height / 101::float)::int = (CEIL(OLD.height / 101::float)::int); + RETURN NULL; +END $$; + +-- Create trigger that will execute 'round_rewards_delete' after deletion of last block of round +CREATE TRIGGER rounds_rewards_delete + AFTER DELETE ON blocks + FOR EACH ROW + WHEN (OLD.height % 101 = 0) + EXECUTE PROCEDURE round_rewards_delete(); + +-- Create function for inserting round rewards when last block of round is inserted +CREATE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + WITH + round AS ( + -- Selecting all blocks of round, apply exception fees and rewards factor + SELECT + b.timestamp, b.height, b."generatorPublicKey" AS pk, b."totalFee" * COALESCE(e.fees_factor, 1) AS fees, + b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb + FROM blocks b + LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round + WHERE CEIL(height / 101::float)::int = CEIL(NEW.height / 101::float)::int + ), + -- Calculating total fees of round, apply exception fees bonus + fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), + -- Get last delegate and timestamp of round's last block + last AS (SELECT pk, timestamp FROM round ORDER BY height DESC LIMIT 1) + INSERT INTO rounds_rewards + SELECT + -- Block height + round.height, + -- Timestamp of last round's block + last.timestamp, + -- Calculating real fee reward for delegate: + -- Rounded fee per delegate + remaining fees if block is last one of round + (fees.single + (CASE WHEN last.pk = round.pk AND last.timestamp = round.timestamp THEN (fees.total - fees.single * 101) ELSE 0 END)) AS fees, + -- Block reward + round.reward, + -- Delegate public key + round.pk + FROM last, fees, round + -- Sort fees by block height + ORDER BY round.height ASC; + RETURN NULL; +END $$; + +-- Create trigger that will execute 'round_rewards_insert' after insertion of last block of round +CREATE TRIGGER rounds_rewards_insert + AFTER INSERT ON blocks + FOR EACH ROW + WHEN (NEW.height % 101 = 0) + EXECUTE PROCEDURE round_rewards_insert(); + +-- Create indexes on columns of 'rounds_rewards' + additional index for round +CREATE INDEX IF NOT EXISTS "rounds_rewards_timestamp" ON rounds_rewards (timestamp); +CREATE INDEX IF NOT EXISTS "rounds_rewards_height" ON rounds_rewards (height); +CREATE INDEX IF NOT EXISTS "rounds_rewards_round" ON rounds_rewards ((CEIL(height / 101::float)::int)); +CREATE INDEX IF NOT EXISTS "rounds_rewards_public_key" ON rounds_rewards (pk); + +COMMIT; From 3ae9b5955066f0d64f999f427c1ec407f20628fa Mon Sep 17 00:00:00 2001 From: 4miners Date: Mon, 8 May 2017 03:54:35 +0200 Subject: [PATCH 04/88] Create SQL function for validating memory balances, validate at node start --- modules/loader.js | 7 +++++- sql/loader.js | 4 ++- ...s.sql => 20170507001337_roundsRewards.sql} | 7 +++++- .../20170508001337_validateMemTables.sql | 25 +++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) rename sql/migrations/{29170507001337_roundsRewards.sql => 20170507001337_roundsRewards.sql} (97%) create mode 100644 sql/migrations/20170508001337_validateMemTables.sql diff --git a/modules/loader.js b/modules/loader.js index 0af74bfce51..da7bb733c30 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -281,7 +281,8 @@ __private.loadBlockChain = function () { t.one(sql.countBlocks), t.query(sql.getGenesisBlock), t.one(sql.countMemAccounts), - t.query(sql.getMemRounds) + t.query(sql.getMemRounds), + t.query(sql.validateMemBalances) ]; return t.batch(promises); @@ -354,6 +355,10 @@ __private.loadBlockChain = function () { return reload(count, 'Detected unapplied rounds in mem_round'); } + if (results[4].length) { + return reload(count, 'Memory balances doesn\'t match blockchain balances'); + } + function updateMemAccounts (t) { var promises = [ t.none(sql.updateMemAccounts), diff --git a/sql/loader.js b/sql/loader.js index dc39ceae39d..b79d4ecfe81 100644 --- a/sql/loader.js +++ b/sql/loader.js @@ -13,7 +13,9 @@ var LoaderSql = { getOrphanedMemAccounts: 'SELECT a."blockId", b."id" FROM mem_accounts a LEFT OUTER JOIN blocks b ON b."id" = a."blockId" WHERE a."blockId" IS NOT NULL AND a."blockId" != \'0\' AND b."id" IS NULL', - getDelegates: 'SELECT ENCODE("publicKey", \'hex\') FROM mem_accounts WHERE "isDelegate" = 1' + getDelegates: 'SELECT ENCODE("publicKey", \'hex\') FROM mem_accounts WHERE "isDelegate" = 1', + + validateMemBalances: 'SELECT * FROM validateMemBalances()' }; module.exports = LoaderSql; diff --git a/sql/migrations/29170507001337_roundsRewards.sql b/sql/migrations/20170507001337_roundsRewards.sql similarity index 97% rename from sql/migrations/29170507001337_roundsRewards.sql rename to sql/migrations/20170507001337_roundsRewards.sql index 07095fa5100..4a2015a3297 100644 --- a/sql/migrations/29170507001337_roundsRewards.sql +++ b/sql/migrations/20170507001337_roundsRewards.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS "rounds_rewards"( "timestamp" INT NOT NULL, "fees" BIGINT NOT NULL, "reward" BIGINT NOT NULL, + "round" INT NOT NULL, "pk" BYTEA NOT NULL ); @@ -75,6 +76,8 @@ CREATE FUNCTION rounds_rewards_init() RETURNS void LANGUAGE PLPGSQL AS $$ (fees.single + (CASE WHEN last.pk = round.pk AND last.timestamp = round.timestamp THEN (fees.total - fees.single * 101) ELSE 0 END)) AS fees, -- Block reward round.reward, + -- Round + CEIL(round.height / 101::float)::int, -- Delegate public key round.pk FROM last, fees, round @@ -129,6 +132,8 @@ CREATE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ (fees.single + (CASE WHEN last.pk = round.pk AND last.timestamp = round.timestamp THEN (fees.total - fees.single * 101) ELSE 0 END)) AS fees, -- Block reward round.reward, + -- Round + CEIL(round.height / 101::float)::int, -- Delegate public key round.pk FROM last, fees, round @@ -147,7 +152,7 @@ CREATE TRIGGER rounds_rewards_insert -- Create indexes on columns of 'rounds_rewards' + additional index for round CREATE INDEX IF NOT EXISTS "rounds_rewards_timestamp" ON rounds_rewards (timestamp); CREATE INDEX IF NOT EXISTS "rounds_rewards_height" ON rounds_rewards (height); -CREATE INDEX IF NOT EXISTS "rounds_rewards_round" ON rounds_rewards ((CEIL(height / 101::float)::int)); +CREATE INDEX IF NOT EXISTS "rounds_rewards_round" ON rounds_rewards (round); CREATE INDEX IF NOT EXISTS "rounds_rewards_public_key" ON rounds_rewards (pk); COMMIT; diff --git a/sql/migrations/20170508001337_validateMemTables.sql b/sql/migrations/20170508001337_validateMemTables.sql new file mode 100644 index 00000000000..6ea352bd8c2 --- /dev/null +++ b/sql/migrations/20170508001337_validateMemTables.sql @@ -0,0 +1,25 @@ +/* + * Create function 'validateMemBalances' for validation memory balances against blockchain + */ + +BEGIN; + +-- Create function that validates memory balances against blockchain +CREATE FUNCTION validateMemBalances() RETURNS TABLE(address VARCHAR(22), pk TEXT, username VARCHAR(20), blockchain BIGINT, memory BIGINT, diff BIGINT) LANGUAGE PLPGSQL AS $$ +BEGIN + RETURN QUERY + WITH balances AS ( + (SELECT UPPER("senderId") AS address, -SUM(amount+fee) AS amount FROM trs GROUP BY UPPER("senderId")) + UNION ALL + (SELECT UPPER("recipientId") AS address, SUM(amount) AS amount FROM trs WHERE "recipientId" IS NOT NULL GROUP BY UPPER("recipientId")) + UNION ALL + (SELECT a.address, r.amount FROM + (SELECT r.pk, SUM(r.fees)+SUM(r.reward) AS amount FROM rounds_rewards r GROUP BY r.pk) r LEFT JOIN mem_accounts a ON r.pk = a."publicKey" + ) + ), + accounts AS (SELECT b.address, SUM(b.amount) AS balance FROM balances b GROUP BY b.address) + SELECT m.address, ENCODE(m."publicKey", 'hex') AS pk, m.username, a.balance::BIGINT AS blockchain, m.balance::BIGINT AS memory, (m.balance-a.balance)::BIGINT AS diff + FROM accounts a LEFT JOIN mem_accounts m ON a.address = m.address WHERE a.balance <> m.balance; +END $$; + +COMMIT; From 897c5ad73db4ee39a13a3b05c29d69254ca764c2 Mon Sep 17 00:00:00 2001 From: 4miners Date: Mon, 8 May 2017 07:17:38 +0200 Subject: [PATCH 05/88] Improve SQL query formatting --- .../20170508001337_validateMemTables.sql | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sql/migrations/20170508001337_validateMemTables.sql b/sql/migrations/20170508001337_validateMemTables.sql index 6ea352bd8c2..07e0e6032d1 100644 --- a/sql/migrations/20170508001337_validateMemTables.sql +++ b/sql/migrations/20170508001337_validateMemTables.sql @@ -7,19 +7,19 @@ BEGIN; -- Create function that validates memory balances against blockchain CREATE FUNCTION validateMemBalances() RETURNS TABLE(address VARCHAR(22), pk TEXT, username VARCHAR(20), blockchain BIGINT, memory BIGINT, diff BIGINT) LANGUAGE PLPGSQL AS $$ BEGIN - RETURN QUERY - WITH balances AS ( - (SELECT UPPER("senderId") AS address, -SUM(amount+fee) AS amount FROM trs GROUP BY UPPER("senderId")) - UNION ALL - (SELECT UPPER("recipientId") AS address, SUM(amount) AS amount FROM trs WHERE "recipientId" IS NOT NULL GROUP BY UPPER("recipientId")) - UNION ALL - (SELECT a.address, r.amount FROM - (SELECT r.pk, SUM(r.fees)+SUM(r.reward) AS amount FROM rounds_rewards r GROUP BY r.pk) r LEFT JOIN mem_accounts a ON r.pk = a."publicKey" - ) - ), - accounts AS (SELECT b.address, SUM(b.amount) AS balance FROM balances b GROUP BY b.address) - SELECT m.address, ENCODE(m."publicKey", 'hex') AS pk, m.username, a.balance::BIGINT AS blockchain, m.balance::BIGINT AS memory, (m.balance-a.balance)::BIGINT AS diff - FROM accounts a LEFT JOIN mem_accounts m ON a.address = m.address WHERE a.balance <> m.balance; + RETURN QUERY + WITH balances AS ( + (SELECT UPPER("senderId") AS address, -SUM(amount+fee) AS amount FROM trs GROUP BY UPPER("senderId")) + UNION ALL + (SELECT UPPER("recipientId") AS address, SUM(amount) AS amount FROM trs WHERE "recipientId" IS NOT NULL GROUP BY UPPER("recipientId")) + UNION ALL + (SELECT a.address, r.amount FROM + (SELECT r.pk, SUM(r.fees) + SUM(r.reward) AS amount FROM rounds_rewards r GROUP BY r.pk) r LEFT JOIN mem_accounts a ON r.pk = a."publicKey" + ) + ), + accounts AS (SELECT b.address, SUM(b.amount) AS balance FROM balances b GROUP BY b.address) + SELECT m.address, ENCODE(m."publicKey", 'hex') AS pk, m.username, a.balance::BIGINT AS blockchain, m.balance::BIGINT AS memory, (m.balance-a.balance)::BIGINT AS diff + FROM accounts a LEFT JOIN mem_accounts m ON a.address = m.address WHERE a.balance <> m.balance; END $$; COMMIT; From ad0b94cbd9b37d28476cd6f73d61b2d08a9d75e6 Mon Sep 17 00:00:00 2001 From: 4miners Date: Mon, 8 May 2017 09:38:32 +0200 Subject: [PATCH 06/88] Create SQL function generateDelegatesList, added tests --- .../20170508101337_generateDelegatesList.sql | 57 +++ test/index.js | 1 + test/sql/delegatesList.js | 8 + test/unit/sql/delegatesList.js | 399 ++++++++++++++++++ 4 files changed, 465 insertions(+) create mode 100644 sql/migrations/20170508101337_generateDelegatesList.sql create mode 100644 test/sql/delegatesList.js create mode 100644 test/unit/sql/delegatesList.js diff --git a/sql/migrations/20170508101337_generateDelegatesList.sql b/sql/migrations/20170508101337_generateDelegatesList.sql new file mode 100644 index 00000000000..7caecdfd49e --- /dev/null +++ b/sql/migrations/20170508101337_generateDelegatesList.sql @@ -0,0 +1,57 @@ +/* + * Create function for generating delegates list + */ + +BEGIN; + +-- Enable crypto extension +CREATE EXTENSION pgcrypto; + +-- Create function that generate delegates list +-- @IMMUTABLE - always returns the same result +CREATE OR REPLACE FUNCTION generateDelegatesList(round int, delegates text[]) RETURNS text[] LANGUAGE PLPGSQL IMMUTABLE AS $$ +DECLARE + i int; + x int; + n int; + old text; + hash bytea; + len int; +BEGIN + -- Check if arguments are valid + IF round IS NULL OR round < 1 OR delegates IS NULL OR array_length(delegates, 1) IS NULL OR array_length(delegates, 1) < 1 THEN + RAISE invalid_parameter_value USING MESSAGE = 'Invalid parameters supplied'; + END IF; + + -- Create hash from round + hash := digest(round::text, 'sha256'); + len := array_length(delegates, 1); + + i := 0; + LOOP + EXIT WHEN i >= 101; + x := 0; + LOOP + EXIT WHEN x >= 4 OR i >= len; + -- Calculate index to swap at + n := get_byte(hash, x) % len; + -- Copy delegate before overwrite + old := delegates[n+1]; + -- Swap delegates at index n with delegate at index i + delegates[n+1] = delegates[i+1]; + delegates[i+1] = old; + -- Increment iterators + i := i + 1; + x := x + 1; + END LOOP; + -- Calculate next hash + hash := digest(hash, 'sha256'); + -- Increment iterator + i := i + 1; + END LOOP; + + -- Return generated delagets list + RETURN delegates; +END $$; + +COMMIT; diff --git a/test/index.js b/test/index.js index 2087c394fcd..8f697da5fea 100644 --- a/test/index.js +++ b/test/index.js @@ -1,6 +1,7 @@ require('./unit/helpers/request-limiter.js'); require('./unit/logic/blockReward.js'); require('./unit/sql/blockRewards.js'); +require('./unit/sql/delegatesList.js'); require('./unit/modules/peers.js'); require('./unit/modules/blocks.js'); diff --git a/test/sql/delegatesList.js b/test/sql/delegatesList.js new file mode 100644 index 00000000000..e09ae042d4e --- /dev/null +++ b/test/sql/delegatesList.js @@ -0,0 +1,8 @@ +'use strict'; + +var DelegatesList = { + generateDelegatesList: 'SELECT generateDelegatesList AS delegates FROM generateDelegatesList(${round}, ${delegates});', + generateDelegatesListCast: 'SELECT generateDelegatesList AS delegates FROM generateDelegatesList(${round}, ${delegates}::text[]);', +}; + +module.exports = DelegatesList; diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js new file mode 100644 index 00000000000..cf4aa1bd1d7 --- /dev/null +++ b/test/unit/sql/delegatesList.js @@ -0,0 +1,399 @@ +'use strict'; + +var chai = require('chai'); +var expect = require('chai').expect; + +var crypto = require('crypto'); +var sql = require('../../sql/delegatesList.js'); +var modulesLoader = require('../../common/initModule').modulesLoader; +var db; + +before(function (done) { + modulesLoader.getDbConnection(function (err, db_handle) { + if (err) { + return done(err); + } + db = db_handle; + done(); + }); +}); + +function generateDelegatesList (round, delegates) { + var i, x, n, old, len; + var list = []; + var hash = crypto.createHash('sha256').update(round, 'utf8').digest(); + + // Copy delegates array + i = delegates.length; + while (i--) { + list[i] = delegates[i]; + } + + // Generate new delegates list + for (i = 0, len = list.length; i < len; i++) { + for (x = 0; x < 4 && i < len; i++, x++) { + n = hash[x] % len; + old = list[n]; + list[n] = list[i]; + list[i] = old; + } + hash = crypto.createHash('sha256').update(hash).digest(); + } + + return list; +}; + +describe('DelegatesListSQL', function () { + + describe('checking SQL function generateDelegatesList()', function () { + it('SQL generateDelegatesList() results should be equal to generateDelegatesList() - fake 101 delegates', function (done) { + var round = '26381'; + var delegates = []; + for (var i = 1; i <= 101; i++) { + delegates.push(i.toString()); + } + var expectedDelegates = generateDelegatesList(round, delegates); + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + expect(rows).to.be.array; + expect(rows.length).to.equal(1); + expect(rows[0]).to.be.object; + expect(rows[0].delegates).to.be.an.array; + expect(rows[0].delegates.length).to.equal(delegates.length); + for (var i = rows[0].delegates.length - 1; i >= 0; i--) { + expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); + } + done(); + }).catch(function (err) { + done(err); + }); + }); + + it('SQL generateDelegatesList() results should be equal to generateDelegatesList() - real 101 delegates, exact order', function (done) { + var round = '26381'; + var delegates = [ + 'ec111c8ad482445cfe83d811a7edd1f1d2765079c99d7d958cca1354740b7614', + 'b002f58531c074c7190714523eec08c48db8c7cfc0c943097db1a2e82ed87f84', + '1a99630b0ca1642b232888a3119e68b000b6194eced51e7fe3231bbe476f7c10', + '677c79b243ed96a8439e8bd193d6ab966ce43c9aa18830d2b9eb8974455d79f8', + '25e961fa459d202816776c8736560d493a94fdd7381971f63fb9b70479487598', + '32f20bee855238630b0f791560c02cf93014977b4b25c19ef93cd92220390276', + '00de7d28ec3f55f42329667f08352d0d90faa3d2d4e62c883d86d1d7b083dd7c', + 'ad936990fb57f7e686763c293e9ca773d1d921888f5235189945a10029cd95b0', + '253e674789632f72c98d47a650f1ca5ece0dbb82f591080471129d57ed88fb8a', + 'd12a6aef4b165b0197adb82d0d544202897b95300ff1fff93c339cf866defb0d', + '76ceefed8f29dd48664b07d207f4bf202122f2ffed6dcefa802d7fe348203b88', + '326bff18531703385d4037e5585b001e732c4a68afb8f82efe2b46c27dcf05aa', + '2493d52fc34ecaaa4a7d0d76e6de9bda24f1b5e11e3363c30a13d59e9c345f82', + 'c4d96fbfe80102f01579945fe0c5fe2a1874a7ffeca6bacef39140f9358e3db6', + '393f73238941510379d930e674e21ca4c00ba30c0877cd3728b5bd5874588671', + 'eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7', + '9771b09041466268948626830cbfea5a043527f39174d70e11a80011f386bb57', + 'b3953cb16e2457b9be78ad8c8a2985435dedaed5f0dd63443bdfbccc92d09f2d', + 'f147c1cba67acad603309d5004f25d9ab41ae073b318f4c6f972f96106c9b527', + 'ac09bc40c889f688f9158cca1fcfcdf6320f501242e0f7088d52a5077084ccba', + '93ec60444c28e8b8c0f1a613ed41f518b637280c454188e5500ea4e54e1a2f12', + '247bf1854471c1a97ccc363c63e876bb6b9a7f06038486048a17196a8a5493dc', + '1b774b27f49f4fe43cc8218e230bc39d0b16d0ee68abe828585ef87d316493ac', + '1305f8955a240a464393f52867d17ba271454fa2a6f2249fb5901b86e7c7334e', + 'adbe299504da4e6cf9d7eb481bdf72f23e6a0332df8049b4a018b99604e394da', + '41bb70d08312d9c17ec89a4577d30da77d5b936594fc06ccb0646602bed6ad40', + '5c4a92f575822b2d2deaa4bc0985ec9a57a17719bd5427af634ec1b4bf9c045b', + '186ffbe710bc27690934ef6cd64aeda2afdd634cbbaf6d23310ca7a31ab96e60', + '2d59fbcce531fb9661cdfa8371c49b6898ce0895fe71da88ffec851c7ed60782', + '484beb54e2990e17c18119b6065d00c8a65954039ec2d40a9e4ac41862dc561e', + '1681920f9cb83ff2590a8e5c502a7015d4834f5365cf5ed17392c9c78147f94d', + '7e838ec9b59a50d2c3333f079b0489871f12c1726eff483c3a88a287dbe36713', + '130649e3d8d34eb59197c00bcf6f199bc4ec06ba0968f1d473b010384569e7f0', + '9172179a88f8cfeeb81518ad31da4397555273b8658eb3ea2d1eca7965d8e615', + 'aad413159fe85e4f4d1941166ddcc97850f5964ee2ef8bda95519d019af8d488', + '6a01c4b86f4519ec9fa5c3288ae20e2e7a58822ebe891fb81e839588b95b242a', + 'feac3a6303ac41ebb9561d52fe2f3b4271fe846d2d2ffae722f18b6f04fc4ce9', + 'c7a0f96797a9dc3085534463650a09e1f160fecb6c0ec6c21e74ef2a222b73a4', + 'e36f75a27598512c2f4ad06fffeeffa37d3aad81c3a4edb77d871ec0ee933471', + 'eaa5ccb65e635e9ad3ebd98c2b7402b3a7c048fcd300c2d8aed8864f621ee6b2', + '2cb967f6c73d9b6b8604d7b199271fed3183ff18ae0bd9cde6d6ef6072f83c05', + '76c321881c08b0c2f538abf753044603ab3081f5441fe069c125a6e2803015da', + 'f8fa9e01047c19102133d2af06aab6cc377d5665bede412f04f81bcdc368d00e', + 'f91766de68f3a8859a3634c3a0fdde38ebd82dd91fc37b67ac6cf010800a3e6e', + 'b70f1d97cd254e93e2dd7b24567b3dbe06a60b5cbabe3443463c61cb87879b47', + 'f88b86d0a104bda71b2ff4d8234fef4e184ee771a9c2d3a298280790c185231b', + 'b6de69ebd1ba0bfe2d37ea6733c64b7e3eb262bee6c9cee05034b0b4465e2678', + 'b73fa499a7794c111fcd011cdc7dcc426341a28c6c2d6a32b8d7d028dcb8493f', + '0a13f5d075186bc99b9ec5b7bd3fbaeee0ab68a9314ac8d12a1f562e82d5e1c5', + 'e0f1c6cca365cd61bbb01cfb454828a698fa4b7170e85a597dde510567f9dda5', + '5386c93dbc76fce1e3a5ae5436ba98bb39e6a0929d038ee2118af54afd45614a', + 'e5c785871ac07632b42bc3862e7035330ff44fb0314e2253d1d7c0a35f3866f9', + 'c88af4585b4fabba89e8015bcf180c38a8027a8057dcf575977875a361282d7b', + 'a0f768d6476a9cfec1a64a895064fe114b26bd3fb6aeda397ccce7ef7f3f98ef', + '6cb825715058d2e821aa4af75fbd0da52181910d9fda90fabe73cd533eeb6acb', + 'a2c3a994fdf110802d5856ff18f306e7a3731452ed7a0fed8aac48e58fd729aa', + 'fbac76743fad9448ed0b45fb4c97a62f81a358908aa14f6a2c76d2a8dc207141', + 'de918e28b554600a81cbf119abf5414648b58a8efafbc3b0481df0242684dc1b', + '90ad9bfed339af2d6b4b3b7f7cdf25d927b255f9f25dbbc892ee9ca57ef67807', + 'a40c3e1549a9bbea71606ef05b793629923bdb151390145e3730dfe2b28b9217', + 'e7ac617b33d0f019d9d030c2e34870767d8994680e7b10ebdaf2af0e59332524', + 'b851863cf6b4769df5cecca718463173485bb9fe21e20f7cfb0802f5ab5973c2', + '33e8874f91f2b1295a2218e8d9f83761827a8d326fbc23e26b52a527714e75f0', + 'faf9f863e704f9cf560bc7a5718a25d851666d38195cba3cacd360cd5fa96fd3', + 'a2fc2420262f081d0f6426364301ef40597756e163f6b1fd813eff9b03594125', + '8966b54a95b327651e3103d8adb69579ff50bf22a004a65731a41f7caca2859f', + 'db4b4db208667f9266e8a4d7fad9d8b2e711891175a21ee5f5f2cd088d1d8083', + '6971dc02efc00140fbfcb262dd6f84d2dee533b258427de7017528b2e10ac2b1', + '613e4178a65c1194192eaa29910f0ecca3737f92587dd05d58c6435da41220f6', + 'e4717693ad6a02a4e6615e1ad4070fdf24d6a628a9d19a8396e4c91018a11307', + 'a81d59b68ba8942d60c74d10bc6488adec2ae1fa9b564a22447289076fe7b1e4', + 'c119a622b3ea85727b236574c43e83350252973ae765bb2061623a13c4f3d431', + 'ca1285393e1848ee41ba0c5e47789e5e0c570a7b51d8e2f7f6db37417b892cf9', + 'b690204a2a4a39431b8aaa4bb9af4e53aead93d2d46c5042edada9f5d43d6cd3', + '9c99976107b5d98e5669452392d912edf33c968e5832b52f2eedcd044b5cc2f2', + '942972349c8f2afe93ad874b3f19de05de7e34c120b23803438c7eeb8e6113b7', + '7fba92f4a2a510ae7301dddddf136e1f8673b54fd0ff0d92ec63f59b68bf4a8f', + 'abe994f962c34d7997506a657beee403f6b807eb2b2605dc6c3b93bb67b839eb', + '0fec636f5866c66f63a0d3db9b00e3cd5ba1b3a0324712c7935ae845dbfcf58a', + '63db2063e7760b241b0fe69436834fa2b759746b8237e1aafd2e099a38fc64d6', + '72f0cd8486d8627b5bd4f10c2e592a4512ac58e572edb3e37c0448b3ac7dd405', + '3345cae6361e329bc931fda1245e263617c797c8b21b7abfb7914fcda1a7833b', + '77c59f444c8a49bcd354759cc912166fe6eaa603a5f9d4a9525405b30a52ac10', + '45ab8f54edff6b802335dc3ea5cd5bc5324e4031c0598a2cdcae79402e4941f8', + '465085ba003a03d1fed0cfd35b7f3c07927c9db41d32194d273f8fe2fa238faa', + 'b68f666f1ede5615bf382958a815988a42aea8e4e03fbf0470a57bceac7714db', + 'c58078a7d12d81190ef0c5deb7611f97fc923d064647b66b9b25512029a13daf', + 'd9299750eeb71720dda470bccb8fafa57cf13f4939749615642c75a191481dea', + '2f58f5b6b1e2e91a9634dfadd1d6726a5aed2875f33260b6753cb9ec7da72917', + '0c0c8f58e7feeaa687d7dc9a5146ea14afe1bc647f518990b197b9f55728effa', + '968ba2fa993ea9dc27ed740da0daf49eddd740dbd7cb1cb4fc5db3a20baf341b', + '47226b469031d48f215973a11876c3f03a6d74360b40a55192b2ba9e5a74ede5', + '88260051bbe6634431f8a2f3ac66680d1ee9ef1087222e6823d9b4d81170edc7', + '619a3113c6cb1d3db7ef9731e6e06b618296815b3cfe7ca8d23f3767198b00ea', + '7ac9d4b708fb19eaa200eb883be56601ddceed96290a3a033114750b7fda9d0b', + 'fd039dd8caa03d58c0ecbaa09069403e7faff864dccd5933da50a41973292fa1', + 'b7633636a88ba1ce8acd98aa58b4a9618650c8ab860c167be6f8d78404265bae', + 'f495866ce86de18d8d4e746fca6a3a130608e5882875b88908b6551104f28e6a', + '31e1174043091ab0feb8d1e2ada4041a0ff54d0ced1e809890940bd706ffc201', + 'e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f', + 'f54ce2a222ab3513c49e586464d89a2a7d9959ecce60729289ec0bb6106bd4ce' + ]; + var expectedDelegates = generateDelegatesList(round, delegates); + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + expect(rows).to.be.array; + expect(rows.length).to.equal(1); + expect(rows[0]).to.be.object; + expect(rows[0].delegates).to.be.an.array; + expect(rows[0].delegates.length).to.equal(delegates.length); + for (var i = rows[0].delegates.length - 1; i >= 0; i--) { + expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); + } + expect(rows[0].delegates[0]).to.equal('b7633636a88ba1ce8acd98aa58b4a9618650c8ab860c167be6f8d78404265bae'); + expect(rows[0].delegates[1]).to.equal('1681920f9cb83ff2590a8e5c502a7015d4834f5365cf5ed17392c9c78147f94d'); + expect(rows[0].delegates[2]).to.equal('186ffbe710bc27690934ef6cd64aeda2afdd634cbbaf6d23310ca7a31ab96e60'); + expect(rows[0].delegates[3]).to.equal('0a13f5d075186bc99b9ec5b7bd3fbaeee0ab68a9314ac8d12a1f562e82d5e1c5'); + expect(rows[0].delegates[4]).to.equal('25e961fa459d202816776c8736560d493a94fdd7381971f63fb9b70479487598'); + expect(rows[0].delegates[5]).to.equal('3345cae6361e329bc931fda1245e263617c797c8b21b7abfb7914fcda1a7833b'); + expect(rows[0].delegates[6]).to.equal('a2c3a994fdf110802d5856ff18f306e7a3731452ed7a0fed8aac48e58fd729aa'); + expect(rows[0].delegates[7]).to.equal('f495866ce86de18d8d4e746fca6a3a130608e5882875b88908b6551104f28e6a'); + expect(rows[0].delegates[8]).to.equal('e36f75a27598512c2f4ad06fffeeffa37d3aad81c3a4edb77d871ec0ee933471'); + expect(rows[0].delegates[9]).to.equal('d12a6aef4b165b0197adb82d0d544202897b95300ff1fff93c339cf866defb0d'); + expect(rows[0].delegates[10]).to.equal('d9299750eeb71720dda470bccb8fafa57cf13f4939749615642c75a191481dea'); + expect(rows[0].delegates[11]).to.equal('77c59f444c8a49bcd354759cc912166fe6eaa603a5f9d4a9525405b30a52ac10'); + expect(rows[0].delegates[12]).to.equal('b73fa499a7794c111fcd011cdc7dcc426341a28c6c2d6a32b8d7d028dcb8493f'); + expect(rows[0].delegates[13]).to.equal('47226b469031d48f215973a11876c3f03a6d74360b40a55192b2ba9e5a74ede5'); + expect(rows[0].delegates[14]).to.equal('393f73238941510379d930e674e21ca4c00ba30c0877cd3728b5bd5874588671'); + expect(rows[0].delegates[15]).to.equal('abe994f962c34d7997506a657beee403f6b807eb2b2605dc6c3b93bb67b839eb'); + expect(rows[0].delegates[16]).to.equal('e4717693ad6a02a4e6615e1ad4070fdf24d6a628a9d19a8396e4c91018a11307'); + expect(rows[0].delegates[17]).to.equal('f91766de68f3a8859a3634c3a0fdde38ebd82dd91fc37b67ac6cf010800a3e6e'); + expect(rows[0].delegates[18]).to.equal('7e838ec9b59a50d2c3333f079b0489871f12c1726eff483c3a88a287dbe36713'); + expect(rows[0].delegates[19]).to.equal('942972349c8f2afe93ad874b3f19de05de7e34c120b23803438c7eeb8e6113b7'); + expect(rows[0].delegates[20]).to.equal('33e8874f91f2b1295a2218e8d9f83761827a8d326fbc23e26b52a527714e75f0'); + expect(rows[0].delegates[21]).to.equal('7ac9d4b708fb19eaa200eb883be56601ddceed96290a3a033114750b7fda9d0b'); + expect(rows[0].delegates[22]).to.equal('90ad9bfed339af2d6b4b3b7f7cdf25d927b255f9f25dbbc892ee9ca57ef67807'); + expect(rows[0].delegates[23]).to.equal('968ba2fa993ea9dc27ed740da0daf49eddd740dbd7cb1cb4fc5db3a20baf341b'); + expect(rows[0].delegates[24]).to.equal('adbe299504da4e6cf9d7eb481bdf72f23e6a0332df8049b4a018b99604e394da'); + expect(rows[0].delegates[25]).to.equal('e7ac617b33d0f019d9d030c2e34870767d8994680e7b10ebdaf2af0e59332524'); + expect(rows[0].delegates[26]).to.equal('b002f58531c074c7190714523eec08c48db8c7cfc0c943097db1a2e82ed87f84'); + expect(rows[0].delegates[27]).to.equal('ec111c8ad482445cfe83d811a7edd1f1d2765079c99d7d958cca1354740b7614'); + expect(rows[0].delegates[28]).to.equal('c88af4585b4fabba89e8015bcf180c38a8027a8057dcf575977875a361282d7b'); + expect(rows[0].delegates[29]).to.equal('484beb54e2990e17c18119b6065d00c8a65954039ec2d40a9e4ac41862dc561e'); + expect(rows[0].delegates[30]).to.equal('c119a622b3ea85727b236574c43e83350252973ae765bb2061623a13c4f3d431'); + expect(rows[0].delegates[31]).to.equal('feac3a6303ac41ebb9561d52fe2f3b4271fe846d2d2ffae722f18b6f04fc4ce9'); + expect(rows[0].delegates[32]).to.equal('c7a0f96797a9dc3085534463650a09e1f160fecb6c0ec6c21e74ef2a222b73a4'); + expect(rows[0].delegates[33]).to.equal('e5c785871ac07632b42bc3862e7035330ff44fb0314e2253d1d7c0a35f3866f9'); + expect(rows[0].delegates[34]).to.equal('253e674789632f72c98d47a650f1ca5ece0dbb82f591080471129d57ed88fb8a'); + expect(rows[0].delegates[35]).to.equal('db4b4db208667f9266e8a4d7fad9d8b2e711891175a21ee5f5f2cd088d1d8083'); + expect(rows[0].delegates[36]).to.equal('b6de69ebd1ba0bfe2d37ea6733c64b7e3eb262bee6c9cee05034b0b4465e2678'); + expect(rows[0].delegates[37]).to.equal('6a01c4b86f4519ec9fa5c3288ae20e2e7a58822ebe891fb81e839588b95b242a'); + expect(rows[0].delegates[38]).to.equal('aad413159fe85e4f4d1941166ddcc97850f5964ee2ef8bda95519d019af8d488'); + expect(rows[0].delegates[39]).to.equal('b690204a2a4a39431b8aaa4bb9af4e53aead93d2d46c5042edada9f5d43d6cd3'); + expect(rows[0].delegates[40]).to.equal('eaa5ccb65e635e9ad3ebd98c2b7402b3a7c048fcd300c2d8aed8864f621ee6b2'); + expect(rows[0].delegates[41]).to.equal('1305f8955a240a464393f52867d17ba271454fa2a6f2249fb5901b86e7c7334e'); + expect(rows[0].delegates[42]).to.equal('1a99630b0ca1642b232888a3119e68b000b6194eced51e7fe3231bbe476f7c10'); + expect(rows[0].delegates[43]).to.equal('31e1174043091ab0feb8d1e2ada4041a0ff54d0ced1e809890940bd706ffc201'); + expect(rows[0].delegates[44]).to.equal('ad936990fb57f7e686763c293e9ca773d1d921888f5235189945a10029cd95b0'); + expect(rows[0].delegates[45]).to.equal('faf9f863e704f9cf560bc7a5718a25d851666d38195cba3cacd360cd5fa96fd3'); + expect(rows[0].delegates[46]).to.equal('a81d59b68ba8942d60c74d10bc6488adec2ae1fa9b564a22447289076fe7b1e4'); + expect(rows[0].delegates[47]).to.equal('1b774b27f49f4fe43cc8218e230bc39d0b16d0ee68abe828585ef87d316493ac'); + expect(rows[0].delegates[48]).to.equal('00de7d28ec3f55f42329667f08352d0d90faa3d2d4e62c883d86d1d7b083dd7c'); + expect(rows[0].delegates[49]).to.equal('e0f1c6cca365cd61bbb01cfb454828a698fa4b7170e85a597dde510567f9dda5'); + expect(rows[0].delegates[50]).to.equal('5386c93dbc76fce1e3a5ae5436ba98bb39e6a0929d038ee2118af54afd45614a'); + expect(rows[0].delegates[51]).to.equal('f54ce2a222ab3513c49e586464d89a2a7d9959ecce60729289ec0bb6106bd4ce'); + expect(rows[0].delegates[52]).to.equal('b70f1d97cd254e93e2dd7b24567b3dbe06a60b5cbabe3443463c61cb87879b47'); + expect(rows[0].delegates[53]).to.equal('613e4178a65c1194192eaa29910f0ecca3737f92587dd05d58c6435da41220f6'); + expect(rows[0].delegates[54]).to.equal('6cb825715058d2e821aa4af75fbd0da52181910d9fda90fabe73cd533eeb6acb'); + expect(rows[0].delegates[55]).to.equal('76c321881c08b0c2f538abf753044603ab3081f5441fe069c125a6e2803015da'); + expect(rows[0].delegates[56]).to.equal('9172179a88f8cfeeb81518ad31da4397555273b8658eb3ea2d1eca7965d8e615'); + expect(rows[0].delegates[57]).to.equal('41bb70d08312d9c17ec89a4577d30da77d5b936594fc06ccb0646602bed6ad40'); + expect(rows[0].delegates[58]).to.equal('fd039dd8caa03d58c0ecbaa09069403e7faff864dccd5933da50a41973292fa1'); + expect(rows[0].delegates[59]).to.equal('a40c3e1549a9bbea71606ef05b793629923bdb151390145e3730dfe2b28b9217'); + expect(rows[0].delegates[60]).to.equal('f88b86d0a104bda71b2ff4d8234fef4e184ee771a9c2d3a298280790c185231b'); + expect(rows[0].delegates[61]).to.equal('88260051bbe6634431f8a2f3ac66680d1ee9ef1087222e6823d9b4d81170edc7'); + expect(rows[0].delegates[62]).to.equal('0fec636f5866c66f63a0d3db9b00e3cd5ba1b3a0324712c7935ae845dbfcf58a'); + expect(rows[0].delegates[63]).to.equal('677c79b243ed96a8439e8bd193d6ab966ce43c9aa18830d2b9eb8974455d79f8'); + expect(rows[0].delegates[64]).to.equal('a2fc2420262f081d0f6426364301ef40597756e163f6b1fd813eff9b03594125'); + expect(rows[0].delegates[65]).to.equal('de918e28b554600a81cbf119abf5414648b58a8efafbc3b0481df0242684dc1b'); + expect(rows[0].delegates[66]).to.equal('c58078a7d12d81190ef0c5deb7611f97fc923d064647b66b9b25512029a13daf'); + expect(rows[0].delegates[67]).to.equal('619a3113c6cb1d3db7ef9731e6e06b618296815b3cfe7ca8d23f3767198b00ea'); + expect(rows[0].delegates[68]).to.equal('c4d96fbfe80102f01579945fe0c5fe2a1874a7ffeca6bacef39140f9358e3db6'); + expect(rows[0].delegates[69]).to.equal('b851863cf6b4769df5cecca718463173485bb9fe21e20f7cfb0802f5ab5973c2'); + expect(rows[0].delegates[70]).to.equal('a0f768d6476a9cfec1a64a895064fe114b26bd3fb6aeda397ccce7ef7f3f98ef'); + expect(rows[0].delegates[71]).to.equal('5c4a92f575822b2d2deaa4bc0985ec9a57a17719bd5427af634ec1b4bf9c045b'); + expect(rows[0].delegates[72]).to.equal('2493d52fc34ecaaa4a7d0d76e6de9bda24f1b5e11e3363c30a13d59e9c345f82'); + expect(rows[0].delegates[73]).to.equal('2cb967f6c73d9b6b8604d7b199271fed3183ff18ae0bd9cde6d6ef6072f83c05'); + expect(rows[0].delegates[74]).to.equal('9c99976107b5d98e5669452392d912edf33c968e5832b52f2eedcd044b5cc2f2'); + expect(rows[0].delegates[75]).to.equal('ac09bc40c889f688f9158cca1fcfcdf6320f501242e0f7088d52a5077084ccba'); + expect(rows[0].delegates[76]).to.equal('ca1285393e1848ee41ba0c5e47789e5e0c570a7b51d8e2f7f6db37417b892cf9'); + expect(rows[0].delegates[77]).to.equal('6971dc02efc00140fbfcb262dd6f84d2dee533b258427de7017528b2e10ac2b1'); + expect(rows[0].delegates[78]).to.equal('b3953cb16e2457b9be78ad8c8a2985435dedaed5f0dd63443bdfbccc92d09f2d'); + expect(rows[0].delegates[79]).to.equal('63db2063e7760b241b0fe69436834fa2b759746b8237e1aafd2e099a38fc64d6'); + expect(rows[0].delegates[80]).to.equal('f8fa9e01047c19102133d2af06aab6cc377d5665bede412f04f81bcdc368d00e'); + expect(rows[0].delegates[81]).to.equal('93ec60444c28e8b8c0f1a613ed41f518b637280c454188e5500ea4e54e1a2f12'); + expect(rows[0].delegates[82]).to.equal('130649e3d8d34eb59197c00bcf6f199bc4ec06ba0968f1d473b010384569e7f0'); + expect(rows[0].delegates[83]).to.equal('0c0c8f58e7feeaa687d7dc9a5146ea14afe1bc647f518990b197b9f55728effa'); + expect(rows[0].delegates[84]).to.equal('b68f666f1ede5615bf382958a815988a42aea8e4e03fbf0470a57bceac7714db'); + expect(rows[0].delegates[85]).to.equal('32f20bee855238630b0f791560c02cf93014977b4b25c19ef93cd92220390276'); + expect(rows[0].delegates[86]).to.equal('45ab8f54edff6b802335dc3ea5cd5bc5324e4031c0598a2cdcae79402e4941f8'); + expect(rows[0].delegates[87]).to.equal('f147c1cba67acad603309d5004f25d9ab41ae073b318f4c6f972f96106c9b527'); + expect(rows[0].delegates[88]).to.equal('72f0cd8486d8627b5bd4f10c2e592a4512ac58e572edb3e37c0448b3ac7dd405'); + expect(rows[0].delegates[89]).to.equal('326bff18531703385d4037e5585b001e732c4a68afb8f82efe2b46c27dcf05aa'); + expect(rows[0].delegates[90]).to.equal('fbac76743fad9448ed0b45fb4c97a62f81a358908aa14f6a2c76d2a8dc207141'); + expect(rows[0].delegates[91]).to.equal('2f58f5b6b1e2e91a9634dfadd1d6726a5aed2875f33260b6753cb9ec7da72917'); + expect(rows[0].delegates[92]).to.equal('9771b09041466268948626830cbfea5a043527f39174d70e11a80011f386bb57'); + expect(rows[0].delegates[93]).to.equal('76ceefed8f29dd48664b07d207f4bf202122f2ffed6dcefa802d7fe348203b88'); + expect(rows[0].delegates[94]).to.equal('247bf1854471c1a97ccc363c63e876bb6b9a7f06038486048a17196a8a5493dc'); + expect(rows[0].delegates[95]).to.equal('8966b54a95b327651e3103d8adb69579ff50bf22a004a65731a41f7caca2859f'); + expect(rows[0].delegates[96]).to.equal('7fba92f4a2a510ae7301dddddf136e1f8673b54fd0ff0d92ec63f59b68bf4a8f'); + expect(rows[0].delegates[97]).to.equal('2d59fbcce531fb9661cdfa8371c49b6898ce0895fe71da88ffec851c7ed60782'); + expect(rows[0].delegates[98]).to.equal('465085ba003a03d1fed0cfd35b7f3c07927c9db41d32194d273f8fe2fa238faa'); + expect(rows[0].delegates[99]).to.equal('e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f'); + expect(rows[0].delegates[100]).to.equal('eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7'); + done(); + }).catch(function (err) { + done(err); + }); + }); + + it('SQL generateDelegatesList() should raise exception for round 0', function (done) { + var round = '0'; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + + it('SQL generateDelegatesList() should raise exception for undefined round', function (done) { + var round = undefined; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + + it('SQL generateDelegatesList() should raise exception for null round', function (done) { + var round = null; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + + it('SQL generateDelegatesList() should raise exception for negative round', function (done) { + var round = -1; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + + it('SQL generateDelegatesList() should raise exception for empty delegates', function (done) { + var round = 1; + var delegates = []; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('cannot determine type of empty array'); + done(); + }); + }); + + it('SQL generateDelegatesListCast() should raise exception for empty delegates', function (done) { + var round = 1; + var delegates = []; + + db.query(sql.generateDelegatesListCast, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + + it('SQL generateDelegatesList() should raise exception for null delegates', function (done) { + var round = 1; + var delegates = null; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + + it('SQL generateDelegatesList() should raise exception for null delegates', function (done) { + var round = 1; + var delegates = undefined; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); + }); +}); From dd7c4505537ee122c43d793351ff4ad2123a723d Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 8 Jun 2017 03:55:29 +0200 Subject: [PATCH 07/88] Port exceptions for rounds to db layer on node start --- modules/loader.js | 26 ++++++++++++++++++++++++++ sql/loader.js | 4 +++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/modules/loader.js b/modules/loader.js index 77c3bb9234e..2289f329147 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -2,6 +2,7 @@ var async = require('async'); var constants = require('../helpers/constants.js'); +var exceptions = require('../helpers/exceptions'); var ip = require('ip'); var sandboxHelper = require('../helpers/sandbox.js'); var schema = require('../schema/loader.js'); @@ -412,6 +413,31 @@ __private.loadBlockChain = function () { } } + // Port rounds exceptions to database layer + library.db.tx(function (t) { + var queries = []; + + Object.keys(exceptions.rounds).forEach(function (round) { + var ex = exceptions.rounds[round]; + queries.push( + t.none(sql.insertRoundException, { + round: round, + rewards_factor: ex.rewards_factor, + fees_factor: ex.fees_factor, + fees_bonus: ex.fees_bonus + }) + ); + } + + return t.batch(queries); + }).then(function (data) { + library.logger.info('Exceptions for round ported to database layer'); + return setImmediate(cb); + }).catch(function (err) { + library.logger.error('Port exceptions to database layer failed', {error: err.message || err}); + return process.emit('exit'); + }); + library.db.task(checkMemTables).then(function (results) { var count = results[0].count; diff --git a/sql/loader.js b/sql/loader.js index b79d4ecfe81..95f4258372b 100644 --- a/sql/loader.js +++ b/sql/loader.js @@ -15,7 +15,9 @@ var LoaderSql = { getDelegates: 'SELECT ENCODE("publicKey", \'hex\') FROM mem_accounts WHERE "isDelegate" = 1', - validateMemBalances: 'SELECT * FROM validateMemBalances()' + validateMemBalances: 'SELECT * FROM validateMemBalances()', + + insertRoundException: 'INSERT INTO rounds_exceptions (round, rewards_factor, fees_factor, fees_bonus) VALUES (${round}, ${rewards_factor}, ${fees_factor}, ${fees_bonus}) ON CONFLICT (round) DO NOTHING;' }; module.exports = LoaderSql; From 26b01be2ddf47c03e51e38caaf365836d95f55f7 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 8 Jun 2017 04:57:52 +0200 Subject: [PATCH 08/88] Check if rounds exceptions match database instead of force-update --- modules/loader.js | 62 +++++++++++++++++++---------------------------- sql/loader.js | 2 +- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/modules/loader.js b/modules/loader.js index 2289f329147..fccbe625066 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -289,7 +289,7 @@ __private.loadTransactions = function (cb) { * @implements {modules.blocks.deleteAfterBlock} * @implements {modules.blocks.loadLastBlock} * @emits exit - * @throws {string} When fails to match genesis block with database + * @throws {string} When fails to match genesis block with database or rounds exceptions doesn't match database */ __private.loadBlockChain = function () { var offset = 0, limit = Number(library.config.loading.loadPerIteration) || 1000; @@ -373,7 +373,8 @@ __private.loadBlockChain = function () { t.query(sql.getGenesisBlock), t.one(sql.countMemAccounts), t.query(sql.getMemRounds), - t.query(sql.validateMemBalances) + t.query(sql.validateMemBalances), + t.query(sql.getRoundsExceptions) ]; return t.batch(promises); @@ -413,34 +414,8 @@ __private.loadBlockChain = function () { } } - // Port rounds exceptions to database layer - library.db.tx(function (t) { - var queries = []; - - Object.keys(exceptions.rounds).forEach(function (round) { - var ex = exceptions.rounds[round]; - queries.push( - t.none(sql.insertRoundException, { - round: round, - rewards_factor: ex.rewards_factor, - fees_factor: ex.fees_factor, - fees_bonus: ex.fees_bonus - }) - ); - } - - return t.batch(queries); - }).then(function (data) { - library.logger.info('Exceptions for round ported to database layer'); - return setImmediate(cb); - }).catch(function (err) { - library.logger.error('Port exceptions to database layer failed', {error: err.message || err}); - return process.emit('exit'); - }); - - library.db.task(checkMemTables).then(function (results) { - var count = results[0].count; - + library.db.task(checkMemTables).then(function ([countBlocks, getGenesisBlock, countMemAccounts, getMemRounds, validateMemBalances, dbRoundsExceptions]) { + var count = countBlocks.count; library.logger.info('Blocks ' + count); var round = modules.rounds.calc(count); @@ -449,7 +424,7 @@ __private.loadBlockChain = function () { return reload(count); } - matchGenesisBlock(results[1][0]); + matchGenesisBlock(getGenesisBlock[0]); verify = verifySnapshot(count, round); @@ -457,13 +432,13 @@ __private.loadBlockChain = function () { return reload(count, 'Blocks verification enabled'); } - var missed = !(results[2].count); + var missed = !(countMemAccounts.count); if (missed) { return reload(count, 'Detected missed blocks in mem_accounts'); } - var unapplied = results[3].filter(function (row) { + var unapplied = getMemRounds.filter(function (row) { return (row.round !== String(round)); }); @@ -471,10 +446,23 @@ __private.loadBlockChain = function () { return reload(count, 'Detected unapplied rounds in mem_round'); } - if (results[4].length) { + if (validateMemBalances.length) { return reload(count, 'Memory balances doesn\'t match blockchain balances'); } + // Compare rounds exceptions with database layer + var roundsExceptions = Object.keys(exceptions.rounds); + if (roundsExceptions.length !== dbRoundsExceptions.length) { + throw 'Rounds exceptions count doesn\'t match database layer'; + } else { + dbRoundsExceptions.forEach(function (row) { + var ex = exceptions.rounds[row.round]; + if (!ex || ex.rewards_factor !== row.rewards_factor || ex.fees_factor !== row.fees_factor || ex.fees_bonus !== Number(row.fees_bonus)) { + throw 'Rounds exceptions values doesn\'t match database layer'; + } + }); + } + function updateMemAccounts (t) { var promises = [ t.none(sql.updateMemAccounts), @@ -485,12 +473,12 @@ __private.loadBlockChain = function () { return t.batch(promises); } - library.db.task(updateMemAccounts).then(function (results) { - if (results[1].length > 0) { + library.db.task(updateMemAccounts).then(function ([updateMemAccounts, getOrphanedMemAccounts, getDelegates]) { + if (getOrphanedMemAccounts.length > 0) { return reload(count, 'Detected orphaned blocks in mem_accounts'); } - if (results[2].length === 0) { + if (getDelegates.length === 0) { return reload(count, 'No delegates found'); } diff --git a/sql/loader.js b/sql/loader.js index 95f4258372b..1aaf513dcb0 100644 --- a/sql/loader.js +++ b/sql/loader.js @@ -17,7 +17,7 @@ var LoaderSql = { validateMemBalances: 'SELECT * FROM validateMemBalances()', - insertRoundException: 'INSERT INTO rounds_exceptions (round, rewards_factor, fees_factor, fees_bonus) VALUES (${round}, ${rewards_factor}, ${fees_factor}, ${fees_bonus}) ON CONFLICT (round) DO NOTHING;' + getRoundsExceptions: 'SELECT * FROM rounds_exceptions;' }; module.exports = LoaderSql; From a7a8789ed460e5636d587cf7f323a204e1af248e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 8 Jun 2017 05:34:47 +0200 Subject: [PATCH 09/88] Initial implementation of rounds management in postgres --- logic/account.js | 64 +--- logic/delegate.js | 12 +- modules/blocks/chain.js | 36 +- modules/blocks/process.js | 2 +- modules/delegates.js | 15 +- modules/loader.js | 12 +- sql/delegates.js | 2 + sql/loader.js | 2 - sql/memoryTables.sql | 11 - .../20170521001337_roundsRewrite.sql | 340 ++++++++++++++++++ 10 files changed, 367 insertions(+), 129 deletions(-) create mode 100644 sql/migrations/20170521001337_roundsRewrite.sql diff --git a/logic/account.js b/logic/account.js index 5a349e1e17f..e50635c006a 100644 --- a/logic/account.js +++ b/logic/account.js @@ -404,7 +404,6 @@ function Account (scope, cb) { /** * Creates memory tables related to accounts: * - mem_accounts - * - mem_round * - mem_accounts2delegates * - mem_accounts2u_delegates * - mem_accounts2multisignatures @@ -425,7 +424,6 @@ Account.prototype.createTables = function (cb) { /** * Deletes the contents of these tables: - * - mem_round * - mem_accounts2delegates * - mem_accounts2u_delegates * - mem_accounts2multisignatures @@ -437,7 +435,6 @@ Account.prototype.removeTables = function (cb) { var sqles = [], sql; [this.table, - 'mem_round', 'mem_accounts2delegates', 'mem_accounts2u_delegates', 'mem_accounts2multisignatures', @@ -642,15 +639,13 @@ Account.prototype.set = function (address, fields, cb) { /** * Updates account from mem_account with diff data belonging to an editable field. - * Inserts into mem_round "address", "amount", "delegate", "blockId", "round" - * based on field balance or delegates. * @param {address} address * @param {Object} diff - Must contains only mem_account editable fields. * @param {function} cb - Callback function. * @returns {setImmediateCallback|cb|done} Multiple returns: done() or error. */ Account.prototype.merge = function (address, diff, cb) { - var update = {}, remove = {}, insert = {}, insert_object = {}, remove_object = {}, round = []; + var update = {}, remove = {}, insert = {}, insert_object = {}, remove_object = {}; // Verify public key this.verifyPublicKey(diff.publicKey); @@ -674,17 +669,6 @@ Account.prototype.merge = function (address, diff, cb) { } else if (Math.abs(trueValue) === trueValue && trueValue !== 0) { update.$inc = update.$inc || {}; update.$inc[value] = Math.floor(trueValue); - if (value === 'balance') { - round.push({ - query: 'INSERT INTO mem_round ("address", "amount", "delegate", "blockId", "round") SELECT ${address}, (${amount})::bigint, "dependentId", ${blockId}, ${round} FROM mem_accounts2delegates WHERE "accountId" = ${address};', - values: { - address: address, - amount: trueValue, - blockId: diff.blockId, - round: diff.round - } - }); - } } else if (trueValue < 0) { update.$dec = update.$dec || {}; update.$dec[value] = Math.floor(Math.abs(trueValue)); @@ -693,17 +677,6 @@ Account.prototype.merge = function (address, diff, cb) { // Remove virginity and ensure marked columns become immutable update.virgin = 0; } - if (value === 'balance') { - round.push({ - query: 'INSERT INTO mem_round ("address", "amount", "delegate", "blockId", "round") SELECT ${address}, (${amount})::bigint, "dependentId", ${blockId}, ${round} FROM mem_accounts2delegates WHERE "accountId" = ${address};', - values: { - address: address, - amount: trueValue, - blockId: diff.blockId, - round: diff.round - } - }); - } } break; case Array: @@ -732,47 +705,14 @@ Account.prototype.merge = function (address, diff, cb) { val = trueValue[i].slice(1); remove[value] = remove[value] || []; remove[value].push(val); - if (value === 'delegates') { - round.push({ - query: 'INSERT INTO mem_round ("address", "amount", "delegate", "blockId", "round") SELECT ${address}, (-balance)::bigint, ${delegate}, ${blockId}, ${round} FROM mem_accounts WHERE address = ${address};', - values: { - address: address, - delegate: val, - blockId: diff.blockId, - round: diff.round - } - }); - } } else if (math === '+') { val = trueValue[i].slice(1); insert[value] = insert[value] || []; insert[value].push(val); - if (value === 'delegates') { - round.push({ - query: 'INSERT INTO mem_round ("address", "amount", "delegate", "blockId", "round") SELECT ${address}, (balance)::bigint, ${delegate}, ${blockId}, ${round} FROM mem_accounts WHERE address = ${address};', - values: { - address: address, - delegate: val, - blockId: diff.blockId, - round: diff.round - } - }); - } } else { val = trueValue[i]; insert[value] = insert[value] || []; insert[value].push(val); - if (value === 'delegates') { - round.push({ - query: 'INSERT INTO mem_round ("address", "amount", "delegate", "blockId", "round") SELECT ${address}, (balance)::bigint, ${delegate}, ${blockId}, ${round} FROM mem_accounts WHERE address = ${address};', - values: { - address: address, - delegate: val, - blockId: diff.blockId, - round: diff.round - } - }); - } } } } @@ -862,7 +802,7 @@ Account.prototype.merge = function (address, diff, cb) { } } - var queries = sqles.concat(round).map(function (sql) { + var queries = sqles.map(function (sql) { return pgp.as.format(sql.query, sql.values); }).join(''); diff --git a/logic/delegate.js b/logic/delegate.js index 777b19ff9db..bc756a4e696 100644 --- a/logic/delegate.js +++ b/logic/delegate.js @@ -302,8 +302,10 @@ Delegate.prototype.dbRead = function (raw) { Delegate.prototype.dbTable = 'delegates'; Delegate.prototype.dbFields = [ - 'username', - 'transactionId' + 'tx_id', + 'name', + 'pk', + 'address' ]; /** @@ -316,8 +318,10 @@ Delegate.prototype.dbSave = function (trs) { table: this.dbTable, fields: this.dbFields, values: { - username: trs.asset.delegate.username, - transactionId: trs.id + tx_id: trs.id, + name: trs.asset.delegate.username, + pk: Buffer.from(trs.senderPublicKey, 'hex'), + address: trs.senderId } }; }; diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index 14d274a509b..151ab273917 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -255,9 +255,7 @@ Chain.prototype.applyGenesisBlock = function (block, cb) { } else { // Set genesis block as last block modules.blocks.lastBlock.set(block); - // Tick round - // WARNING: DB_WRITE - modules.rounds.tick(block, cb); + return cb(); } }); }; @@ -456,14 +454,11 @@ Chain.prototype.applyBlock = function (block, broadcast, cb, saveBlock) { library.logger.debug('Block applied correctly with ' + block.transactions.length + ' transactions'); library.bus.message('newBlock', block, broadcast); - // DATABASE write. Update delegates accounts - modules.rounds.tick(block, seriesCb); + return seriesCb(); }); } else { library.bus.message('newBlock', block, broadcast); - - // DATABASE write. Update delegates accounts - modules.rounds.tick(block, seriesCb); + return seriesCb(); } }, // Push back unconfirmed transactions list (minus the one that were on the block if applied correctly). @@ -549,12 +544,12 @@ __private.popLastBlock = function (oldLastBlock, cb) { return process.exitCode = 0; } - // Perform backward tick on rounds - // WARNING: DB_WRITE - modules.rounds.backwardTick(oldLastBlock, previousBlock, function (err) { + // Delete last block from blockchain + // WARNING: Db_WRITE + self.deleteBlock(oldLastBlock.id, function (err) { if (err) { // Fatal error, memory tables will be inconsistent - library.logger.error('Failed to perform backwards tick', err); + library.logger.error('Failed to delete block', err); /** * Exits process gracefully with code 0 @@ -563,22 +558,7 @@ __private.popLastBlock = function (oldLastBlock, cb) { return process.exitCode = 0; } - // Delete last block from blockchain - // WARNING: Db_WRITE - self.deleteBlock(oldLastBlock.id, function (err) { - if (err) { - // Fatal error, memory tables will be inconsistent - library.logger.error('Failed to delete block', err); - - /** - * Exits process gracefully with code 0 - * @see {@link https://nodejs.org/api/process.html#process_process_exit_code} - */ - return process.exitCode = 0; - } - - return setImmediate(cb, null, previousBlock); - }); + return setImmediate(cb, null, previousBlock); }); }); }); diff --git a/modules/blocks/process.js b/modules/blocks/process.js index e118213b4ed..1e55507d457 100644 --- a/modules/blocks/process.js +++ b/modules/blocks/process.js @@ -339,7 +339,7 @@ Process.prototype.onReceiveBlock = function (block) { library.sequence.add(function (cb) { // When client is not loaded, is syncing or round is ticking // Do not receive new blocks as client is not ready - if (!__private.loaded || modules.loader.syncing() || modules.rounds.ticking()) { + if (!__private.loaded || modules.loader.syncing()) { library.logger.debug('Client not ready to receive block', block.id); return; } diff --git a/modules/delegates.js b/modules/delegates.js index d6435820191..b7039a03f70 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -55,17 +55,12 @@ function Delegates (cb, scope) { * @returns {setImmediateCallback} */ __private.getKeysSortByVote = function (cb) { - modules.accounts.getAccounts({ - isDelegate: 1, - sort: {'vote': -1, 'publicKey': 1}, - limit: slots.delegates - }, ['publicKey'], function (err, rows) { - if (err) { - return setImmediate(cb, err); - } + library.db.query(sql.delegateList).then(function (rows) { return setImmediate(cb, null, rows.map(function (el) { - return el.publicKey; + return el.pk; })); + }).catch(function (err) { + return setImmediate(cb, err); }); }; @@ -114,7 +109,7 @@ __private.forge = function (cb) { // When client is not loaded, is syncing or round is ticking // Do not try to forge new blocks as client is not ready - if (!__private.loaded || modules.loader.syncing() || !modules.rounds.loaded() || modules.rounds.ticking()) { + if (!__private.loaded || modules.loader.syncing()) { library.logger.debug('Client not ready to forge'); return setImmediate(cb); } diff --git a/modules/loader.js b/modules/loader.js index fccbe625066..10cc2907b40 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -270,7 +270,6 @@ __private.loadTransactions = function (cb) { * - count blocks from `blocks` table * - get genesis block from `blocks` table * - count accounts from `mem_accounts` table by block id - * - get rounds from `mem_round` * Matchs genesis block with database. * Verifies Snapshot mode. * Recreates memory tables when neccesary: @@ -372,7 +371,6 @@ __private.loadBlockChain = function () { t.one(sql.countBlocks), t.query(sql.getGenesisBlock), t.one(sql.countMemAccounts), - t.query(sql.getMemRounds), t.query(sql.validateMemBalances), t.query(sql.getRoundsExceptions) ]; @@ -414,7 +412,7 @@ __private.loadBlockChain = function () { } } - library.db.task(checkMemTables).then(function ([countBlocks, getGenesisBlock, countMemAccounts, getMemRounds, validateMemBalances, dbRoundsExceptions]) { + library.db.task(checkMemTables).then(function ([countBlocks, getGenesisBlock, countMemAccounts, validateMemBalances, dbRoundsExceptions]) { var count = countBlocks.count; library.logger.info('Blocks ' + count); @@ -438,14 +436,6 @@ __private.loadBlockChain = function () { return reload(count, 'Detected missed blocks in mem_accounts'); } - var unapplied = getMemRounds.filter(function (row) { - return (row.round !== String(round)); - }); - - if (unapplied.length > 0) { - return reload(count, 'Detected unapplied rounds in mem_round'); - } - if (validateMemBalances.length) { return reload(count, 'Memory balances doesn\'t match blockchain balances'); } diff --git a/sql/delegates.js b/sql/delegates.js index 9d43f816b0b..e37222d9da7 100644 --- a/sql/delegates.js +++ b/sql/delegates.js @@ -18,6 +18,8 @@ var DelegatesSql = { count: 'SELECT COUNT(*)::int FROM delegates', + delegateList: 'SELECT ENCODE(pk, \'hex\') AS pk FROM delegates ORDER BY rank ASC LIMIT 101', + search: function (params) { var sql = [ 'WITH', diff --git a/sql/loader.js b/sql/loader.js index 1aaf513dcb0..6d208490200 100644 --- a/sql/loader.js +++ b/sql/loader.js @@ -7,8 +7,6 @@ var LoaderSql = { countMemAccounts: 'SELECT COUNT(*)::int FROM mem_accounts WHERE "blockId" = (SELECT "id" FROM "blocks" ORDER BY "height" DESC LIMIT 1)', - getMemRounds: 'SELECT "round" FROM mem_round GROUP BY "round"', - updateMemAccounts: 'UPDATE mem_accounts SET "u_isDelegate" = "isDelegate", "u_secondSignature" = "secondSignature", "u_username" = "username", "u_balance" = "balance", "u_delegates" = "delegates", "u_multisignatures" = "multisignatures", "u_multimin" = "multimin", "u_multilifetime" = "multilifetime" WHERE "u_isDelegate" <> "isDelegate" OR "u_secondSignature" <> "secondSignature" OR "u_username" <> "username" OR "u_balance" <> "balance" OR "u_delegates" <> "delegates" OR "u_multisignatures" <> "multisignatures" OR "u_multimin" <> "multimin" OR "u_multilifetime" <> "multilifetime";', getOrphanedMemAccounts: 'SELECT a."blockId", b."id" FROM mem_accounts a LEFT OUTER JOIN blocks b ON b."id" = a."blockId" WHERE a."blockId" IS NOT NULL AND a."blockId" != \'0\' AND b."id" IS NULL', diff --git a/sql/memoryTables.sql b/sql/memoryTables.sql index 0da9d1421aa..b1b7fc206d9 100644 --- a/sql/memoryTables.sql +++ b/sql/memoryTables.sql @@ -38,17 +38,6 @@ CREATE TABLE IF NOT EXISTS "mem_accounts"( CREATE INDEX IF NOT EXISTS "mem_accounts_balance" ON "mem_accounts"("balance"); -CREATE TABLE IF NOT EXISTS "mem_round"( - "address" VARCHAR(22), - "amount" BIGINT, - "delegate" VARCHAR(64), - "blockId" VARCHAR(20), - "round" BIGINT -); - -CREATE INDEX IF NOT EXISTS "mem_round_address" ON "mem_round"("address"); -CREATE INDEX IF NOT EXISTS "mem_round_round" ON "mem_round"("round"); - CREATE TABLE IF NOT EXISTS "mem_accounts2delegates"( "accountId" VARCHAR(22) NOT NULL, "dependentId" VARCHAR(64) NOT NULL, diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql new file mode 100644 index 00000000000..3621d8967e1 --- /dev/null +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -0,0 +1,340 @@ +BEGIN; + +-- Consistency checks - 'mem_accounts' against blockchain +DO $$ +DECLARE + diff int; +BEGIN + RAISE NOTICE 'Rounds rewrite migration, please wait...'; + SELECT COUNT(1) FROM validateMemBalances() INTO diff; + + IF diff > 0 THEN + RAISE check_violation USING MESSAGE = 'Migration failed, mem_accounts are inconsistent'; + END IF; +END $$; + +-- Rename table 'delegates' to 'delegates_old' +ALTER TABLE delegates RENAME TO delegates_old; + +-- Create table 'delegates' +CREATE TABLE IF NOT EXISTS "delegates" ( + "tx_id" VARCHAR(20) REFERENCES trs(id) ON DELETE CASCADE, + "name" VARCHAR(20) NOT NULL UNIQUE, + "pk" BYTEA NOT NULL UNIQUE, + "address" VARCHAR(22) NOT NULL UNIQUE, + "rank" INT DEFAULT NULL, + "fees" BIGINT NOT NULL DEFAULT 0, + "rewards" BIGINT NOT NULL DEFAULT 0, + "voters_balance" BIGINT NOT NULL DEFAULT 0, + "voters_cnt" BIGINT NOT NULL DEFAULT 0, + "blocks_forged_cnt" INT NOT NULL DEFAULT 0, + "blocks_missed_cnt" INT NOT NULL DEFAULT 0 +); + +-- Populate 'delegates' table from blockchain ('delegates_old', 'trs') +INSERT INTO delegates (tx_id, name, pk, address) (SELECT t.id, d.username, t."senderPublicKey", t."senderId" FROM delegates_old d LEFT JOIN trs t ON t.id = d."transactionId"); + +-- Set rewards and fees from blockchain ('rounds_rewards') +WITH new AS (SELECT pk, SUM(reward) AS rewards, SUM(fees) AS fees FROM rounds_rewards GROUP BY pk) +UPDATE delegates SET rewards = new.rewards, fees = new.fees FROM new WHERE delegates.pk = new.pk; + +-- Set blocks_forged_cnt from blockchain ('blocks') +WITH new AS (SELECT "generatorPublicKey" AS pk, COUNT(1) AS cnt FROM blocks GROUP BY "generatorPublicKey") +UPDATE delegates SET blocks_forged_cnt = new.cnt FROM new WHERE delegates.pk = new.pk; + +-- Set blocks_missed_cnt from blockchain ('mem_accounts') +WITH new AS (SELECT "publicKey" AS pk, missedblocks FROM mem_accounts) +UPDATE delegates SET blocks_missed_cnt = new.missedblocks FROM new WHERE delegates.pk = new.pk; + +-- Create table 'votes_details' +CREATE TABLE IF NOT EXISTS "votes_details"( + "tx_id" VARCHAR(20) REFERENCES trs(id) ON DELETE CASCADE, + "voter_address" VARCHAR(22) NOT NULL, + "type" VARCHAR(3) NOT NULL, + "timestamp" INT NOT NULL, + "round" INT NOT NULL, + "delegate_pk" BYTEA REFERENCES delegates(pk) ON DELETE CASCADE +); + +-- Populate 'votes_details' table from blockchain ('votes', 'trs', 'blocks') +INSERT INTO votes_details +SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'add' ELSE 'rem' END) AS type, r.timestamp, r.round, DECODE(substring(vote, 2), 'hex') AS delegate_pk FROM ( + SELECT v."transactionId" AS tx_id, t."senderId" AS voter_address, b.timestamp AS timestamp, CEIL(b.height / 101::float)::int AS round, regexp_split_to_table(v.votes, ',') AS vote + FROM votes v, trs t, blocks b WHERE v."transactionId" = t.id AND b.id = t."blockId" +) AS r ORDER BY r.timestamp ASC; + +-- Create function for inserting votes details +CREATE FUNCTION vote_insert() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + INSERT INTO votes_details + SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'add' ELSE 'rem' END) AS type, r.timestamp, r.round, DECODE(substring(vote, 2), 'hex') AS delegate_pk FROM ( + SELECT v."transactionId" AS tx_id, t."senderId" AS voter_address, b.timestamp AS timestamp, CEIL(b.height / 101::float)::int AS round, regexp_split_to_table(v.votes, ',') AS vote + FROM votes v, trs t, blocks b WHERE v."transactionId" = NEW."transactionId" AND v."transactionId" = t.id AND b.id = t."blockId" + ) AS r ORDER BY r.timestamp ASC; + RETURN NULL; +END $$; + +-- Create trigger that will execute 'vote_insert' after insertion of new vote +CREATE TRIGGER vote_insert + AFTER INSERT ON votes + FOR EACH ROW + EXECUTE PROCEDURE vote_insert(); + +-- Create indexes on 'votes_details' +CREATE INDEX votes_details_voter_address ON votes_details(voter_address); +CREATE INDEX votes_details_type ON votes_details(type); +CREATE INDEX votes_details_round ON votes_details(round); +CREATE INDEX votes_details_sort ON votes_details(voter_address ASC, timestamp DESC); +CREATE INDEX votes_details_dpk ON votes_details(delegate_pk); + +-- Create function 'delegates_voters_cnt_update' for updating voters_cnt from blockchain ('votes_details', 'blocks') +CREATE FUNCTION delegates_voters_cnt_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ + BEGIN + RETURN QUERY + WITH + last_round AS (SELECT CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 ORDER BY height DESC LIMIT 1), + updated AS (UPDATE delegates SET voters_cnt = cnt FROM + (SELECT + d.pk, + (SELECT COUNT(1) AS cnt FROM + (SELECT DISTINCT ON (voter_address) voter_address, delegate_pk, type FROM votes_details + WHERE delegate_pk = d.pk AND round <= (SELECT round FROM last_round) + ORDER BY voter_address, timestamp DESC + ) v WHERE type = 'add' + ) FROM delegates d + ) dd WHERE delegates.pk = dd.pk RETURNING 1) + SELECT COUNT(1)::INT FROM updated; +END $$; + +-- Execute 'delegates_voters_cnt_update' +SELECT delegates_voters_cnt_update(); + +-- Create function 'delegates_voters_balance_update' for updating voters_balance from blockchain ('votes_details', 'trs', 'blocks') +CREATE FUNCTION delegates_voters_balance_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ + BEGIN + RETURN QUERY + WITH last_round AS (SELECT height, CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 ORDER BY height DESC LIMIT 1), + current_round_txs AS (SELECT t.id FROM trs t LEFT JOIN blocks b ON b.id = t."blockId" WHERE b.height > (SELECT height FROM last_round)), + voters AS (SELECT DISTINCT ON (voter_address) voter_address FROM votes_details), + balances AS ( + (SELECT UPPER("senderId") AS address, -SUM(amount+fee) AS amount FROM trs GROUP BY UPPER("senderId")) + UNION ALL + (SELECT UPPER("senderId") AS address, SUM(amount+fee) AS amount FROM trs WHERE id IN (SELECT * FROM current_round_txs) GROUP BY UPPER("senderId")) + UNION ALL + (SELECT UPPER("recipientId") AS address, SUM(amount) AS amount FROM trs WHERE "recipientId" IS NOT NULL GROUP BY UPPER("recipientId")) + UNION ALL + (SELECT UPPER("recipientId") AS address, -SUM(amount) AS amount FROM trs WHERE id IN (SELECT * FROM current_round_txs) AND "recipientId" IS NOT NULL GROUP BY UPPER("recipientId")) + UNION ALL + (SELECT d.address, d.fees+d.rewards AS amount FROM delegates d) + ), + filtered AS (SELECT * FROM balances WHERE address IN (SELECT * FROM voters)), + accounts AS (SELECT b.address, SUM(b.amount) AS balance FROM filtered b GROUP BY b.address), + updated AS (UPDATE delegates SET voters_balance = balance FROM + (SELECT d.pk, ( + (SELECT COALESCE(SUM(balance), 0) AS balance FROM accounts WHERE address IN + (SELECT v.voter_address FROM + (SELECT DISTINCT ON (voter_address) voter_address, type FROM votes_details + WHERE delegate_pk = d.pk AND round <= (SELECT round FROM last_round) + ORDER BY voter_address, timestamp DESC + ) v + WHERE v.type = 'add' + ) + ) + ) FROM delegates d) dd WHERE delegates.pk = dd.pk RETURNING 1) + SELECT COUNT(1)::INT FROM updated; +END $$; + +-- Execute 'delegates_voters_balance_update' +SELECT delegates_voters_balance_update(); + +-- Update delegates rank +CREATE FUNCTION delegates_rank_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ + BEGIN + RETURN QUERY + WITH new AS (SELECT row_number() OVER (ORDER BY voters_balance DESC, pk ASC) AS rank, tx_id FROM delegates), + updated AS (UPDATE delegates SET rank = new.rank FROM new WHERE delegates.tx_id = new.tx_id RETURNING 1) + SELECT COUNT(1)::INT FROM updated; +END $$; + +SELECT delegates_rank_update(); + +-- Consistency checks - new 'delegates' table against 'mem_accounts' +DO $$ +DECLARE + diff int; +BEGIN + SELECT COUNT(1) FROM delegates d LEFT JOIN mem_accounts m ON d.pk = m."publicKey" + WHERE m.rewards <> d.rewards OR m.fees <> d.fees OR m.vote <> d.voters_balance OR m.producedblocks <> d.blocks_forged_cnt OR m.missedblocks <> d.blocks_missed_cnt + INTO diff; + + IF diff > 0 THEN + RAISE check_violation USING MESSAGE = 'Migration failed, delegates not match mem_accounts'; + END IF; +END $$; + +-- Drop 'full_blocks_list' view +DROP VIEW full_blocks_list; + +-- Recreate 'full_blocks_list' view +CREATE VIEW full_blocks_list AS SELECT + b."id" AS "b_id", + b."version" AS "b_version", + b."timestamp" AS "b_timestamp", + b."height" AS "b_height", + b."previousBlock" AS "b_previousBlock", + b."numberOfTransactions" AS "b_numberOfTransactions", + (b."totalAmount")::bigint AS "b_totalAmount", + (b."totalFee")::bigint AS "b_totalFee", + (b."reward")::bigint AS "b_reward", + b."payloadLength" AS "b_payloadLength", + ENCODE(b."payloadHash", 'hex') AS "b_payloadHash", + ENCODE(b."generatorPublicKey", 'hex') AS "b_generatorPublicKey", + ENCODE(b."blockSignature", 'hex') AS "b_blockSignature", + t."id" AS "t_id", + t."rowId" AS "t_rowId", + t."type" AS "t_type", + t."timestamp" AS "t_timestamp", + ENCODE(t."senderPublicKey", 'hex') AS "t_senderPublicKey", + t."senderId" AS "t_senderId", + t."recipientId" AS "t_recipientId", + (t."amount")::bigint AS "t_amount", + (t."fee")::bigint AS "t_fee", + ENCODE(t."signature", 'hex') AS "t_signature", + ENCODE(t."signSignature", 'hex') AS "t_signSignature", + ENCODE(s."publicKey", 'hex') AS "s_publicKey", + d."name" AS "d_username", + v."votes" AS "v_votes", + m."min" AS "m_min", + m."lifetime" AS "m_lifetime", + m."keysgroup" AS "m_keysgroup", + dapp."name" AS "dapp_name", + dapp."description" AS "dapp_description", + dapp."tags" AS "dapp_tags", + dapp."type" AS "dapp_type", + dapp."link" AS "dapp_link", + dapp."category" AS "dapp_category", + dapp."icon" AS "dapp_icon", + it."dappId" AS "in_dappId", + ot."dappId" AS "ot_dappId", + ot."outTransactionId" AS "ot_outTransactionId", + ENCODE(t."requesterPublicKey", 'hex') AS "t_requesterPublicKey", + t."signatures" AS "t_signatures" +FROM blocks b +LEFT OUTER JOIN trs AS t ON t."blockId" = b."id" +LEFT OUTER JOIN delegates AS d ON d."tx_id" = t."id" +LEFT OUTER JOIN votes AS v ON v."transactionId" = t."id" +LEFT OUTER JOIN signatures AS s ON s."transactionId" = t."id" +LEFT OUTER JOIN multisignatures AS m ON m."transactionId" = t."id" +LEFT OUTER JOIN dapps AS dapp ON dapp."transactionId" = t."id" +LEFT OUTER JOIN intransfer AS it ON it."transactionId" = t."id" +LEFT OUTER JOIN outtransfer AS ot ON ot."transactionId" = t."id"; + +-- Drop table 'delegates_old' +DROP TABLE delegates_old; + +-- Create function for update 'delegates'.'blocks_forged_cnt' +CREATE FUNCTION delegates_forged_blocks_cnt_update() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + IF (TG_OP = 'INSERT') THEN + UPDATE delegates SET blocks_forged_cnt = blocks_forged_cnt+1 WHERE pk = NEW."generatorPublicKey"; + ELSIF (TG_OP = 'DELETE') THEN + UPDATE delegates SET blocks_forged_cnt = blocks_forged_cnt-1 WHERE pk = OLD."generatorPublicKey"; + END IF; + RETURN NULL; +END $$; + +-- Create trigger that will execute 'delegates_forged_blocks_cnt_update' after insertion or deletion of block +CREATE TRIGGER vote_insert_delete + AFTER INSERT OR DELETE ON blocks + FOR EACH ROW + EXECUTE PROCEDURE delegates_forged_blocks_cnt_update(); + +-- Create function 'delegates_update_on_block' for updating 'delegates' table data +CREATE FUNCTION delegates_update_on_block() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + PERFORM delegates_voters_cnt_update(); + PERFORM delegates_voters_balance_update(); + PERFORM delegates_rank_update(); + RETURN NULL; +END $$; + +-- Create trigger that will execute 'delegates_update_on_block' after insertion of last block of round +-- Trigger is deferred - will be executed after transaction in which block is inserted - block's transactions are already inserted here +CREATE CONSTRAINT TRIGGER block_insert + AFTER INSERT ON blocks + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW + WHEN (NEW.height % 101 = 0) + EXECUTE PROCEDURE delegates_update_on_block(); + +-- Create trigger that will execute 'delegates_update_on_block' after deletion of last block of round +-- Trigger is deferred - will be executed after transaction in which block is inserted - block's transactions are already deleted here +CREATE CONSTRAINT TRIGGER block_delete + AFTER DELETE ON blocks + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW + WHEN (OLD.height % 101 = 0) + EXECUTE PROCEDURE delegates_update_on_block(); + +-- Replace function for deleting round rewards when last block of round is deleted +CREATE OR REPLACE FUNCTION round_rewards_delete() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int) GROUP BY pk) + UPDATE delegates SET rewards = delegates.rewards-r.rewards, fees = delegates.fees-r.fees FROM r WHERE delegates.pk = r.pk; + + WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int) GROUP BY pk) + UPDATE mem_accounts SET balance = mem_accounts.balance-r.rewards-r.fees, u_balance = mem_accounts.u_balance-r.rewards-r.fees FROM r WHERE mem_accounts."publicKey" = r.pk; + + DELETE FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int); + RETURN NULL; +END $$; + +-- Replace function for inserting round rewards when last block of round is inserted +CREATE OR REPLACE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + WITH + round AS ( + -- Selecting all blocks of round, apply exception fees and rewards factor + SELECT + b.timestamp, b.height, b."generatorPublicKey" AS pk, b."totalFee" * COALESCE(e.fees_factor, 1) AS fees, + b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb + FROM blocks b + LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round + WHERE CEIL(height / 101::float)::int = CEIL(NEW.height / 101::float)::int + ), + -- Calculating total fees of round, apply exception fees bonus + fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), + -- Get last delegate and timestamp of round's last block + last AS (SELECT pk, timestamp FROM round ORDER BY height DESC LIMIT 1) + INSERT INTO rounds_rewards + SELECT + -- Block height + round.height, + -- Timestamp of last round's block + last.timestamp, + -- Calculating real fee reward for delegate: + -- Rounded fee per delegate + remaining fees if block is last one of round + (fees.single + (CASE WHEN last.pk = round.pk AND last.timestamp = round.timestamp THEN (fees.total - fees.single * 101) ELSE 0 END)) AS fees, + -- Block reward + round.reward, + -- Round + CEIL(round.height / 101::float)::int, + -- Delegate public key + round.pk + FROM last, fees, round + -- Sort fees by block height + ORDER BY round.height ASC; + + WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(NEW.height / 101::float)::int) GROUP BY pk) + UPDATE delegates SET rewards = delegates.rewards+r.rewards, fees = delegates.fees+r.fees FROM r WHERE delegates.pk = r.pk; + + WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(NEW.height / 101::float)::int) GROUP BY pk) + UPDATE mem_accounts SET balance = mem_accounts.balance+r.rewards+r.fees, u_balance = mem_accounts.u_balance+r.rewards+r.fees FROM r WHERE mem_accounts."publicKey" = r.pk; + + RETURN NULL; +END $$; + +-- Drop mem_round table +DROP TABLE IF EXISTS mem_round; + +COMMIT; From d50f55945541b3ce60a97be9a07121d460ce9b66 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 8 Jun 2017 06:12:30 +0200 Subject: [PATCH 10/88] Improve comments for SQL stuff --- sql/migrations/20170521001337_roundsRewrite.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 3621d8967e1..c2a41feff45 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -147,7 +147,7 @@ END $$; -- Execute 'delegates_voters_balance_update' SELECT delegates_voters_balance_update(); --- Update delegates rank +-- Create function 'delegates_rank_update' for updating delegates ranks CREATE FUNCTION delegates_rank_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ BEGIN RETURN QUERY @@ -156,6 +156,7 @@ CREATE FUNCTION delegates_rank_update() RETURNS TABLE(updated INT) LANGUAGE PLPG SELECT COUNT(1)::INT FROM updated; END $$; +-- Execute 'delegates_rank_update' SELECT delegates_rank_update(); -- Consistency checks - new 'delegates' table against 'mem_accounts' @@ -268,7 +269,7 @@ CREATE CONSTRAINT TRIGGER block_insert EXECUTE PROCEDURE delegates_update_on_block(); -- Create trigger that will execute 'delegates_update_on_block' after deletion of last block of round --- Trigger is deferred - will be executed after transaction in which block is inserted - block's transactions are already deleted here +-- Trigger is deferred - will be executed after transaction in which block is deleted - block's transactions are already deleted here CREATE CONSTRAINT TRIGGER block_delete AFTER DELETE ON blocks DEFERRABLE INITIALLY DEFERRED @@ -279,12 +280,15 @@ CREATE CONSTRAINT TRIGGER block_delete -- Replace function for deleting round rewards when last block of round is deleted CREATE OR REPLACE FUNCTION round_rewards_delete() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN + -- Update 'delagate' table with round rewards WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int) GROUP BY pk) UPDATE delegates SET rewards = delegates.rewards-r.rewards, fees = delegates.fees-r.fees FROM r WHERE delegates.pk = r.pk; + -- Update 'mem_accounts' table with round rewards WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int) GROUP BY pk) UPDATE mem_accounts SET balance = mem_accounts.balance-r.rewards-r.fees, u_balance = mem_accounts.u_balance-r.rewards-r.fees FROM r WHERE mem_accounts."publicKey" = r.pk; + -- Delete round from 'rounds_rewards' DELETE FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int); RETURN NULL; END $$; @@ -325,9 +329,11 @@ CREATE OR REPLACE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGS -- Sort fees by block height ORDER BY round.height ASC; + -- Update 'delagate' table with round rewards WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(NEW.height / 101::float)::int) GROUP BY pk) UPDATE delegates SET rewards = delegates.rewards+r.rewards, fees = delegates.fees+r.fees FROM r WHERE delegates.pk = r.pk; + -- Update 'mem_accounts' table with round rewards WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(NEW.height / 101::float)::int) GROUP BY pk) UPDATE mem_accounts SET balance = mem_accounts.balance+r.rewards+r.fees, u_balance = mem_accounts.u_balance+r.rewards+r.fees FROM r WHERE mem_accounts."publicKey" = r.pk; From 638ccb9f30b071d67196d0b0803ec6490a3f59f3 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 8 Jun 2017 07:15:51 +0200 Subject: [PATCH 11/88] Adjust delegates endpoint to use new table, add rank update trigger --- sql/delegates.js | 28 +++++++++---------- .../20170521001337_roundsRewrite.sql | 15 +++++++++- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/sql/delegates.js b/sql/delegates.js index e37222d9da7..423feae0050 100644 --- a/sql/delegates.js +++ b/sql/delegates.js @@ -24,24 +24,22 @@ var DelegatesSql = { var sql = [ 'WITH', 'supply AS (SELECT calcSupply((SELECT height FROM blocks ORDER BY height DESC LIMIT 1))::numeric),', - 'delegates AS (SELECT row_number() OVER (ORDER BY vote DESC, m."publicKey" ASC)::int AS rank,', - 'm.username,', - 'm.address,', - 'ENCODE(m."publicKey", \'hex\') AS "publicKey",', - 'm.vote,', - 'm.producedblocks,', - 'm.missedblocks,', - 'ROUND(vote / (SELECT * FROM supply) * 100, 2)::float AS approval,', - '(CASE WHEN producedblocks + missedblocks = 0 THEN 0.00 ELSE', - 'ROUND(100 - (missedblocks::numeric / (producedblocks + missedblocks) * 100), 2)', + 'delegates AS (SELECT ', + 'd.rank,', + 'd.name AS username,', + 'd.address,', + 'ENCODE(d."pk", \'hex\') AS "publicKey",', + 'd.voters_balance AS vote,', + 'd.blocks_forged_cnt AS producedblocks,', + 'd.blocks_missed_cnt AS missedblocks,', + 'ROUND(d.voters_balance / (SELECT * FROM supply) * 100, 2)::float AS approval,', + '(CASE WHEN d.blocks_forged_cnt + d.blocks_missed_cnt = 0 THEN 0.00 ELSE', + 'ROUND(100 - (d.blocks_missed_cnt::numeric / (d.blocks_forged_cnt + d.blocks_missed_cnt) * 100), 2)', 'END)::float AS productivity,', - 'COALESCE(v.voters_cnt, 0) AS voters_cnt,', + 'd.voters_cnt,', 't.timestamp AS register_timestamp', 'FROM delegates d', - 'LEFT JOIN mem_accounts m ON d.username = m.username', - 'LEFT JOIN trs t ON d."transactionId" = t.id', - 'LEFT JOIN (SELECT "dependentId", COUNT(1)::int AS voters_cnt from mem_accounts2delegates GROUP BY "dependentId") v ON v."dependentId" = ENCODE(m."publicKey", \'hex\')', - 'WHERE m."isDelegate" = 1', + 'LEFT JOIN trs t ON d.tx_id = t.id', 'ORDER BY ' + [params.sortField, params.sortMethod].join(' ') + ')', 'SELECT * FROM delegates WHERE username LIKE ${q} LIMIT ${limit}' ].join(' '); diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index c2a41feff45..08368257a68 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS "delegates" ( "fees" BIGINT NOT NULL DEFAULT 0, "rewards" BIGINT NOT NULL DEFAULT 0, "voters_balance" BIGINT NOT NULL DEFAULT 0, - "voters_cnt" BIGINT NOT NULL DEFAULT 0, + "voters_cnt" INT NOT NULL DEFAULT 0, "blocks_forged_cnt" INT NOT NULL DEFAULT 0, "blocks_missed_cnt" INT NOT NULL DEFAULT 0 ); @@ -159,6 +159,19 @@ END $$; -- Execute 'delegates_rank_update' SELECT delegates_rank_update(); +-- Create function 'delegate_change_ranks_update' for updating delegates ranks +CREATE FUNCTION delegate_change_ranks_update() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + BEGIN + PERFORM delegates_rank_update(); + RETURN NULL; +END $$; + +-- Create function 'delegate_insert_delete' for updating delegates ranks when delegate is inserted or deleted +CREATE TRIGGER delegate_insert_delete + AFTER INSERT OR DELETE ON delegates + FOR EACH ROW + EXECUTE PROCEDURE delegate_change_ranks_update(); + -- Consistency checks - new 'delegates' table against 'mem_accounts' DO $$ DECLARE From 1614640e7f1cb2d0c6b165e9893fcdf0770cedae Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 6 Jul 2017 11:31:17 +0200 Subject: [PATCH 12/88] Fix error from merge --- modules/loader.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/loader.js b/modules/loader.js index fe21d842722..d846645c63f 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -438,7 +438,7 @@ __private.loadBlockChain = function () { } } - library.db.task(checkMemTables).then(function ([countBlocks, getGenesisBlock, countMemAccounts, validateMemBalances, dbRoundsExceptions]) { + library.db.task(checkMemTables).then(function ([countBlocks, getGenesisBlock, countMemAccounts, validateMemBalances, dbRoundsExceptions, dbDuplicatedDelegates]) { var count = countBlocks.count; library.logger.info('Blocks ' + count); @@ -479,7 +479,7 @@ __private.loadBlockChain = function () { }); } - var duplicatedDelegates = +results[4][0].count; + var duplicatedDelegates = dbDuplicatedDelegates[0].count; if (duplicatedDelegates > 0) { library.logger.error('Delegates table corrupted with duplicated entries'); From 9fcf8e07e2239fba0f1aedb4716c19642240a021 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 6 Jul 2017 12:46:33 +0200 Subject: [PATCH 13/88] Not use destructuring assignment --- modules/loader.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/loader.js b/modules/loader.js index d846645c63f..02b296ae79a 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -438,7 +438,14 @@ __private.loadBlockChain = function () { } } - library.db.task(checkMemTables).then(function ([countBlocks, getGenesisBlock, countMemAccounts, validateMemBalances, dbRoundsExceptions, dbDuplicatedDelegates]) { + library.db.task(checkMemTables).then(function (res) { + var countBlocks = res[0], + getGenesisBlock = res[1], + countMemAccounts = res[2], + validateMemBalances = res[3], + dbRoundsExceptions = res[4], + dbDuplicatedDelegates = res[5]; + var count = countBlocks.count; library.logger.info('Blocks ' + count); @@ -496,7 +503,11 @@ __private.loadBlockChain = function () { return t.batch(promises); } - library.db.task(updateMemAccounts).then(function ([updateMemAccounts, getOrphanedMemAccounts, getDelegates]) { + library.db.task(updateMemAccounts).then(function (res) { + var updateMemAccounts = res[0], + getOrphanedMemAccounts = res[1], + getDelegates = res[2]; + if (getOrphanedMemAccounts.length > 0) { return reload(count, 'Detected orphaned blocks in mem_accounts'); } From ea5479de7502ee3b814a1c168be727187125fc7b Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 6 Jul 2017 13:45:39 +0200 Subject: [PATCH 14/88] Fix intend, add 'grunt release' test --- Jenkinsfile | 827 ++++++++++++++++++++++++++-------------------------- 1 file changed, 417 insertions(+), 410 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index af9e6222bf4..e4922099cc4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,431 +1,438 @@ def initBuild() { - sh '''#!/bin/bash - pkill -f app.js -9 || true - sudo service postgresql restart - dropdb lisk_test || true - createdb lisk_test - ''' - deleteDir() - checkout scm + sh '''#!/bin/bash + pkill -f app.js -9 || true + sudo service postgresql restart + dropdb lisk_test || true + createdb lisk_test + ''' + deleteDir() + checkout scm } def buildDependency() { - try { - sh '''#!/bin/bash + try { + sh '''#!/bin/bash - # Install Deps - npm install + # Install Deps + npm install - # Install Nodejs - tar -zxf ~/lisk-node-Linux-x86_64.tar.gz + # Install Nodejs + tar -zxf ~/lisk-node-Linux-x86_64.tar.gz - ''' - } catch (err) { - currentBuild.result = 'FAILURE' - error('Stopping build, installation failed') - } + ''' + } catch (err) { + currentBuild.result = 'FAILURE' + error('Stopping build, installation failed') + } } def startLisk() { - try { - sh '''#!/bin/bash - cp test/config.json test/genesisBlock.json . - export NODE_ENV=test - BUILD_ID=dontKillMe ~/start_lisk.sh - ''' - } catch (err) { - currentBuild.result = 'FAILURE' - error('Stopping build, Lisk failed') - } + try { + sh '''#!/bin/bash + cp test/config.json test/genesisBlock.json . + export NODE_ENV=test + BUILD_ID=dontKillMe ~/start_lisk.sh + ''' + } catch (err) { + currentBuild.result = 'FAILURE' + error('Stopping build, Lisk failed') + } } lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { - stage ('Prepare Workspace') { - parallel( - "Build Node-01" : { - node('node-01'){ - initBuild() - } - }, - "Build Node-02" : { - node('node-02'){ - initBuild() - } - }, - "Build Node-03" : { - node('node-03'){ - initBuild() - } - }, - "Initialize Master Workspace" : { - node('master-01'){ - sh ''' - cd /var/lib/jenkins/coverage/ - rm -rf node-0* - rm -rf *.zip - rm -rf coverage-unit/* - rm -rf lisk/* - rm -f merged-lcov.info - ''' - deleteDir() - checkout scm - } - } - ) - } + stage ('Prepare Workspace') { + parallel( + "Build Node-01" : { + node('node-01'){ + initBuild() + } + }, + "Build Node-02" : { + node('node-02'){ + initBuild() + } + }, + "Build Node-03" : { + node('node-03'){ + initBuild() + } + }, + "Initialize Master Workspace" : { + node('master-01'){ + sh ''' + cd /var/lib/jenkins/coverage/ + rm -rf node-0* + rm -rf *.zip + rm -rf coverage-unit/* + rm -rf lisk/* + rm -f merged-lcov.info + ''' + deleteDir() + checkout scm + } + } + ) + } - stage ('Build Dependencies') { - parallel( - "Build Dependencies Node-01" : { - node('node-01'){ - buildDependency() - } - }, - "Build Dependencies Node-02" : { - node('node-02'){ - buildDependency() - } - }, - "Build Dependencies Node-03" : { - node('node-03'){ - buildDependency() - } - } - ) - } + stage ('Build Dependencies') { + parallel( + "Build Dependencies Node-01" : { + node('node-01'){ + buildDependency() + } + }, + "Build Dependencies Node-02" : { + node('node-02'){ + buildDependency() + } + }, + "Build Dependencies Node-03" : { + node('node-03'){ + buildDependency() + } + } + ) + } - stage ('Start Lisk') { - parallel( - "Start Lisk Node-01" : { - node('node-01'){ - startLisk() - } - }, - "Start Lisk Node-02" : { - node('node-02'){ - startLisk() - } - }, - "Start Lisk Node-03" : { - node('node-03'){ - startLisk() - } - } - ) - } + stage ('Start Lisk') { + parallel( + "Start Lisk Node-01" : { + node('node-01'){ + startLisk() + } + }, + "Start Lisk Node-02" : { + node('node-02'){ + startLisk() + } + }, + "Start Lisk Node-03" : { + node('node-03'){ + startLisk() + } + } + ) + } - stage ('Parallel Tests') { - parallel( - "ESLint" : { - node('node-01'){ - sh ''' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run eslint - ''' - } - }, - "Functional Accounts" : { - node('node-01'){ - sh ''' - export TEST=test/api/accounts.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Blocks" : { - node('node-01'){ - sh ''' - export TEST=test/api/blocks.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Delegates" : { - node('node-01'){ - sh ''' - export TEST=test/api/delegates.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Dapps" : { - node('node-01'){ - sh ''' - export TEST=test/api/dapps.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Loader" : { - node('node-01'){ - sh ''' - export TEST=test/api/loader.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Multisignatures" : { - node('node-01'){ - sh ''' - export TEST=test/api/multisignatures.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Signatures" : { - node('node-01'){ - sh ''' - export TEST=test/api/signatures.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Transactions" : { - node('node-01'){ - sh ''' - export TEST=test/api/transactions.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, //End node-01 tests - "Functional Peer - Peer" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Dapp" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.dapp.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Blocks" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.blocks.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Signatures" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.signatures.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Transactions Collision" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.transactions.collision.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Transactions Delegates" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.transactions.delegates.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Transactions Main" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.transactions.main.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Transaction Signatures" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.transactions.signatures.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Peers" : { - node('node-02'){ - sh ''' - export TEST=test/api/peers.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Peer - Votes" : { - node('node-02'){ - sh ''' - export TEST=test/api/peer.transactions.votes.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, // End Node-02 Tests - "Unit - Helpers" : { - node('node-03'){ - sh ''' - export TEST=test/unit/helpers TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Unit - Modules" : { - node('node-03'){ - sh ''' - export TEST=test/unit/modules TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Unit - SQL" : { - node('node-03'){ - sh ''' - export TEST=test/unit/sql TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Unit - Logic" : { - node('node-03'){ - sh ''' - export TEST=test/unit/logic TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, - "Functional Stress - Transactions" : { - node('node-03'){ - sh ''' - export TEST=test/api/peer.transactions.stress.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - } - ) // End Parallel - } + stage ('Parallel Tests') { + parallel( + "ESLint" : { + node('node-01'){ + sh ''' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run eslint + ''' + }, + "Release" : { + node('node-01'){ + sh ''' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + grunt release + ''' + } + }, + "Functional Accounts" : { + node('node-01'){ + sh ''' + export TEST=test/api/accounts.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Blocks" : { + node('node-01'){ + sh ''' + export TEST=test/api/blocks.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Delegates" : { + node('node-01'){ + sh ''' + export TEST=test/api/delegates.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Dapps" : { + node('node-01'){ + sh ''' + export TEST=test/api/dapps.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Loader" : { + node('node-01'){ + sh ''' + export TEST=test/api/loader.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Multisignatures" : { + node('node-01'){ + sh ''' + export TEST=test/api/multisignatures.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Signatures" : { + node('node-01'){ + sh ''' + export TEST=test/api/signatures.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Transactions" : { + node('node-01'){ + sh ''' + export TEST=test/api/transactions.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, //End node-01 tests + "Functional Peer - Peer" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Dapp" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.dapp.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Blocks" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.blocks.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Signatures" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.signatures.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Transactions Collision" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.transactions.collision.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Transactions Delegates" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.transactions.delegates.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Transactions Main" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.transactions.main.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Transaction Signatures" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.transactions.signatures.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Peers" : { + node('node-02'){ + sh ''' + export TEST=test/api/peers.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Peer - Votes" : { + node('node-02'){ + sh ''' + export TEST=test/api/peer.transactions.votes.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, // End Node-02 Tests + "Unit - Helpers" : { + node('node-03'){ + sh ''' + export TEST=test/unit/helpers TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Unit - Modules" : { + node('node-03'){ + sh ''' + export TEST=test/unit/modules TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Unit - SQL" : { + node('node-03'){ + sh ''' + export TEST=test/unit/sql TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Unit - Logic" : { + node('node-03'){ + sh ''' + export TEST=test/unit/logic TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, + "Functional Stress - Transactions" : { + node('node-03'){ + sh ''' + export TEST=test/api/peer.transactions.stress.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + } + ) // End Parallel + } - stage ('Gather Coverage') { - parallel( - "Gather Coverage Node-01" : { - node('node-01'){ - sh '''#!/bin/bash - export HOST=127.0.0.1:4000 - npm run fetchCoverage - # Submit coverage reports to Master - scp test/.coverage-func.zip jenkins@master-01:/var/lib/jenkins/coverage/coverage-func-node-01.zip - ''' - } - }, - "Gather Coverage Node-02" : { - node('node-02'){ - sh '''#!/bin/bash - export HOST=127.0.0.1:4000 - npm run fetchCoverage - # Submit coverage reports to Master - scp test/.coverage-func.zip jenkins@master-01:/var/lib/jenkins/coverage/coverage-func-node-02.zip - ''' - } - }, - "Gather Coverage Node-03" : { - node('node-03'){ - sh '''#!/bin/bash - export HOST=127.0.0.1:4000 - npm run fetchCoverage - # Submit coverage reports to Master - scp test/.coverage-unit/* jenkins@master-01:/var/lib/jenkins/coverage/coverage-unit/ - scp test/.coverage-func.zip jenkins@master-01:/var/lib/jenkins/coverage/coverage-func-node-03.zip - ''' - } - } - ) - } + stage ('Gather Coverage') { + parallel( + "Gather Coverage Node-01" : { + node('node-01'){ + sh '''#!/bin/bash + export HOST=127.0.0.1:4000 + npm run fetchCoverage + # Submit coverage reports to Master + scp test/.coverage-func.zip jenkins@master-01:/var/lib/jenkins/coverage/coverage-func-node-01.zip + ''' + } + }, + "Gather Coverage Node-02" : { + node('node-02'){ + sh '''#!/bin/bash + export HOST=127.0.0.1:4000 + npm run fetchCoverage + # Submit coverage reports to Master + scp test/.coverage-func.zip jenkins@master-01:/var/lib/jenkins/coverage/coverage-func-node-02.zip + ''' + } + }, + "Gather Coverage Node-03" : { + node('node-03'){ + sh '''#!/bin/bash + export HOST=127.0.0.1:4000 + npm run fetchCoverage + # Submit coverage reports to Master + scp test/.coverage-unit/* jenkins@master-01:/var/lib/jenkins/coverage/coverage-unit/ + scp test/.coverage-func.zip jenkins@master-01:/var/lib/jenkins/coverage/coverage-func-node-03.zip + ''' + } + } + ) + } - stage ('Submit Coverage') { - node('master-01'){ - sh ''' - cd /var/lib/jenkins/coverage/ - unzip coverage-func-node-01.zip -d node-01 - unzip coverage-func-node-02.zip -d node-02 - unzip coverage-func-node-03.zip -d node-03 - bash merge_lcov.sh . merged-lcov.info - cp merged-lcov.info $WORKSPACE/merged-lcov.info - cp .coveralls.yml $WORKSPACE/.coveralls.yml - cd $WORKSPACE - cat merged-lcov.info | coveralls -v - ''' - } - } + stage ('Submit Coverage') { + node('master-01'){ + sh ''' + cd /var/lib/jenkins/coverage/ + unzip coverage-func-node-01.zip -d node-01 + unzip coverage-func-node-02.zip -d node-02 + unzip coverage-func-node-03.zip -d node-03 + bash merge_lcov.sh . merged-lcov.info + cp merged-lcov.info $WORKSPACE/merged-lcov.info + cp .coveralls.yml $WORKSPACE/.coveralls.yml + cd $WORKSPACE + cat merged-lcov.info | coveralls -v + ''' + } + } - stage ('Cleanup') { - parallel( - "Cleanup Node-01" : { - node('node-01'){ - sh ''' - pkill -f app.js -9 - ''' - } - }, - "Cleanup Node-02" : { - node('node-02'){ - sh ''' - pkill -f app.js -9 - ''' - } - }, - "Cleanup Node-03" : { - node('node-03'){ - sh ''' - pkill -f app.js -9 - ''' - } - }, - "Cleanup Master" : { - node('master-01'){ - sh ''' - cd /var/lib/jenkins/coverage/ - rm -rf node-0* - rm -rf *.zip - rm -rf coverage-unit/* - rm -f merged-lcov.info - rm -rf lisk/* - ''' - } - } - ) - } + stage ('Cleanup') { + parallel( + "Cleanup Node-01" : { + node('node-01'){ + sh ''' + pkill -f app.js -9 + ''' + } + }, + "Cleanup Node-02" : { + node('node-02'){ + sh ''' + pkill -f app.js -9 + ''' + } + }, + "Cleanup Node-03" : { + node('node-03'){ + sh ''' + pkill -f app.js -9 + ''' + } + }, + "Cleanup Master" : { + node('master-01'){ + sh ''' + cd /var/lib/jenkins/coverage/ + rm -rf node-0* + rm -rf *.zip + rm -rf coverage-unit/* + rm -f merged-lcov.info + rm -rf lisk/* + ''' + } + } + ) + } - stage ('Set milestone') { - milestone 1 - currentBuild.result = 'SUCCESS' - } + stage ('Set milestone') { + milestone 1 + currentBuild.result = 'SUCCESS' + } } From 652730bfd4e55f8b787454ab482c242cb1b5cbe5 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 6 Jul 2017 13:51:51 +0200 Subject: [PATCH 15/88] Delete deprecated migration --- .../20170614155841_uniqueDelegatesConstraint.sql | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 sql/migrations/20170614155841_uniqueDelegatesConstraint.sql diff --git a/sql/migrations/20170614155841_uniqueDelegatesConstraint.sql b/sql/migrations/20170614155841_uniqueDelegatesConstraint.sql deleted file mode 100644 index 31475d1a941..00000000000 --- a/sql/migrations/20170614155841_uniqueDelegatesConstraint.sql +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Setting unique constraints on delegates table - */ - -BEGIN; - -ALTER TABLE delegates ADD CONSTRAINT delegates_unique UNIQUE ("username", "transactionId"); - -COMMIT; From 768b1dba96b184b8bb57b9f1c9dd3a2adf5f7567 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 6 Jul 2017 14:09:36 +0200 Subject: [PATCH 16/88] Remove test for duplicated delegates --- modules/loader.js | 13 ++----------- sql/loader.js | 4 +--- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/modules/loader.js b/modules/loader.js index 02b296ae79a..123915fe07c 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -395,8 +395,7 @@ __private.loadBlockChain = function () { t.query(sql.getGenesisBlock), t.one(sql.countMemAccounts), t.query(sql.validateMemBalances), - t.query(sql.getRoundsExceptions), - t.query(sql.countDuplicatedDelegates) + t.query(sql.getRoundsExceptions) ]; return t.batch(promises); @@ -443,8 +442,7 @@ __private.loadBlockChain = function () { getGenesisBlock = res[1], countMemAccounts = res[2], validateMemBalances = res[3], - dbRoundsExceptions = res[4], - dbDuplicatedDelegates = res[5]; + dbRoundsExceptions = res[4]; var count = countBlocks.count; library.logger.info('Blocks ' + count); @@ -486,13 +484,6 @@ __private.loadBlockChain = function () { }); } - var duplicatedDelegates = dbDuplicatedDelegates[0].count; - - if (duplicatedDelegates > 0) { - library.logger.error('Delegates table corrupted with duplicated entries'); - return process.emit('exit'); - } - function updateMemAccounts (t) { var promises = [ t.none(sql.updateMemAccounts), diff --git a/sql/loader.js b/sql/loader.js index 8edf96e4a8d..6d208490200 100644 --- a/sql/loader.js +++ b/sql/loader.js @@ -15,9 +15,7 @@ var LoaderSql = { validateMemBalances: 'SELECT * FROM validateMemBalances()', - getRoundsExceptions: 'SELECT * FROM rounds_exceptions;', - - countDuplicatedDelegates: 'WITH duplicates AS (SELECT COUNT(1) FROM delegates GROUP BY "transactionId" HAVING COUNT(1) > 1) SELECT count(1) FROM duplicates' + getRoundsExceptions: 'SELECT * FROM rounds_exceptions;' }; module.exports = LoaderSql; From a564b1ca03daaaa39099911abb472ea978bfb98c Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 7 Jul 2017 10:00:03 +0200 Subject: [PATCH 17/88] Initial implementation for postgres LISTEN/NOTIFY feature --- app.js | 4 ++ helpers/pg-notify.js | 92 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 97 insertions(+) create mode 100644 helpers/pg-notify.js diff --git a/app.js b/app.js index 5df230699f3..90dfc3adcac 100644 --- a/app.js +++ b/app.js @@ -432,6 +432,10 @@ d.run(function () { var db = require('./helpers/database.js'); db.connect(config.db, logger, cb); }, + pg_notify: ['db', 'bus', 'logger', function (scope, cb) { + var pg_notify = require('./helpers/pg-notify.js'); + pg_notify.init(scope.db, scope.bus, scope.logger, cb); + }], /** * It tries to connect with redis server based on config. provided in config.json file * @param {function} cb diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js new file mode 100644 index 00000000000..411a04ff6f4 --- /dev/null +++ b/helpers/pg-notify.js @@ -0,0 +1,92 @@ +'use strict'; + +var Promise = require('bluebird'); + +module.exports.init = function (db, bus, logger, cb) { + // Global connection for permanent event listeners + var connection; + // Map channels to bus.message events + var channels = { + round: 'finishRound' + }; + + function onNotification (data) { + // Broadcast notify via events if channel is supported + if (channels[data.channel]) { + bus.message(channels[data.channel], data.payload); + } else { + logger.error('pg-notify: Invalid channel:', data.channel); + } + } + + function setListeners (client) { + client.on('notification', onNotification); + connection.none('LISTEN $1~', 'round') + .catch(function (err) { + logger.error(err); + }); + } + + function removeListeners (client) { + if (connection) { + connection.none('UNLISTEN $1~', 'my-channel') + .catch(function (err) { + logger.error(err); + }); + } + client.removeListener('notification', onNotification); + } + + function onConnectionLost (err, e) { + logger.error('pg-notify: Connection lost', err); + // Prevent use of the connection + connection = null; + removeListeners(e.client); + // Try to re-establish connection 10 times, every 5 seconds + reconnect(5000, 10) + .then(function (obj) { + logger.info('pg-notify: Reconnected successfully'); + }) + .catch(function () { + // Failed after 10 attempts + logger.error('pg-notify: Failed to reconnect - connection lost'); + process.exit(); + }); + } + + function reconnect (delay, maxAttempts) { + delay = delay > 0 ? delay : 0; + maxAttempts = maxAttempts > 0 ? maxAttempts : 1; + return new Promise(function (resolve, reject) { + setTimeout(function () { + db.connect({direct: true, onLost: onConnectionLost}) + .then(function (obj) { + // Global connection is now available + connection = obj; + setListeners(obj.client); + resolve(obj); + }) + .catch(function (err) { + logger.error('pg-notify: Error connecting', err); + if (--maxAttempts) { + reconnect(delay, maxAttempts) + .then(resolve) + .catch(reject); + } else { + reject(err); + } + }); + }, delay); + }); + } + + reconnect () + .then(function (obj) { + logger.info('pg-notify: Initial connection estabilished'); + return setImmediate(cb); + }) + .catch(function (err) { + logger.info('pg-notify: Initial connection failed', err); + return setImmediate(cb, err); + }); +}; diff --git a/package.json b/package.json index eb61372efd5..04249a03177 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "async": "=2.4.1", "bignumber.js": "=4.0.2", + "bluebird": "=3.5.0", "body-parser": "=1.17.2", "bytebuffer": "=5.0.1", "change-case": "=3.0.1", From b94582fcb2bbba85d35093646fc5751796d79e6e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 7 Jul 2017 10:28:27 +0200 Subject: [PATCH 18/88] Fix intend and missing bracket in Jenkinsfile --- Jenkinsfile | 61 +++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e4922099cc4..be3f4909474 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -118,10 +118,11 @@ lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { parallel( "ESLint" : { node('node-01'){ - sh ''' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run eslint - ''' + sh ''' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run eslint + ''' + } }, "Release" : { node('node-01'){ @@ -295,48 +296,48 @@ lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { }, // End Node-02 Tests "Unit - Helpers" : { node('node-03'){ - sh ''' - export TEST=test/unit/helpers TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' + sh ''' + export TEST=test/unit/helpers TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' } }, "Unit - Modules" : { node('node-03'){ - sh ''' - export TEST=test/unit/modules TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' + sh ''' + export TEST=test/unit/modules TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' } }, "Unit - SQL" : { node('node-03'){ - sh ''' - export TEST=test/unit/sql TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' + sh ''' + export TEST=test/unit/sql TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' } }, "Unit - Logic" : { node('node-03'){ - sh ''' - export TEST=test/unit/logic TEST_TYPE='UNIT' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' + sh ''' + export TEST=test/unit/logic TEST_TYPE='UNIT' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' } }, "Functional Stress - Transactions" : { node('node-03'){ - sh ''' - export TEST=test/api/peer.transactions.stress.js TEST_TYPE='FUNC' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } + sh ''' + export TEST=test/api/peer.transactions.stress.js TEST_TYPE='FUNC' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } } ) // End Parallel } From 6a7208126a2c41777d1e453d0bbd9e4161d4cd9c Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 7 Jul 2017 11:24:44 +0200 Subject: [PATCH 19/88] Adjust DelegatesListSQL tests to new deps versions --- test/unit/sql/delegatesList.js | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index cf4aa1bd1d7..fcb0a3d6d63 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -55,11 +55,9 @@ describe('DelegatesListSQL', function () { var expectedDelegates = generateDelegatesList(round, delegates); db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - expect(rows).to.be.array; - expect(rows.length).to.equal(1); - expect(rows[0]).to.be.object; - expect(rows[0].delegates).to.be.an.array; - expect(rows[0].delegates.length).to.equal(delegates.length); + expect(rows).to.be.an('array').and.lengthOf(1); + expect(rows[0]).to.be.an('object'); + expect(rows[0].delegates).to.be.an('array').and.lengthOf(delegates.length); for (var i = rows[0].delegates.length - 1; i >= 0; i--) { expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); } @@ -177,11 +175,9 @@ describe('DelegatesListSQL', function () { var expectedDelegates = generateDelegatesList(round, delegates); db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - expect(rows).to.be.array; - expect(rows.length).to.equal(1); - expect(rows[0]).to.be.object; - expect(rows[0].delegates).to.be.an.array; - expect(rows[0].delegates.length).to.equal(delegates.length); + expect(rows).to.be.an('array').and.lengthOf(1); + expect(rows[0]).to.be.an('object'); + expect(rows[0].delegates).to.be.an('array').and.lengthOf(delegates.length); for (var i = rows[0].delegates.length - 1; i >= 0; i--) { expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); } From d1db3612fde339dde18c26d1c205f2f2e1f31aea Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 7 Jul 2017 11:57:31 +0200 Subject: [PATCH 20/88] Notify from postgres when round changes --- helpers/pg-notify.js | 4 +++- sql/migrations/20170521001337_roundsRewrite.sql | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index 411a04ff6f4..f26e8d8b824 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -13,7 +13,9 @@ module.exports.init = function (db, bus, logger, cb) { function onNotification (data) { // Broadcast notify via events if channel is supported if (channels[data.channel]) { - bus.message(channels[data.channel], data.payload); + var round = parseInt(data.payload); + logger.debug('pg-notify: Round changes:', round); + bus.message(channels[data.channel], round); } else { logger.error('pg-notify: Invalid channel:', data.channel); } diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 08368257a68..2ad05317487 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -265,10 +265,18 @@ CREATE TRIGGER vote_insert_delete -- Create function 'delegates_update_on_block' for updating 'delegates' table data CREATE FUNCTION delegates_update_on_block() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ + DECLARE + height int; BEGIN PERFORM delegates_voters_cnt_update(); PERFORM delegates_voters_balance_update(); PERFORM delegates_rank_update(); + IF (TG_OP = 'INSERT') THEN + height := NEW.height + 1; + ELSIF (TG_OP = 'DELETE') THEN + height := OLD.height; + END IF; + PERFORM pg_notify('round', CEIL(height / 101::float)::text); RETURN NULL; END $$; From ab6482058f4c3853d004716188dba9350f82716d Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 11:42:04 +0200 Subject: [PATCH 21/88] Replaced 'round' notification with 'round-closed' and 'round-reopened' --- helpers/pg-notify.js | 22 +++++++++++++++---- .../20170521001337_roundsRewrite.sql | 10 ++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index f26e8d8b824..dad7da7942e 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -7,16 +7,30 @@ module.exports.init = function (db, bus, logger, cb) { var connection; // Map channels to bus.message events var channels = { - round: 'finishRound' + 'round-closed': 'finishRound', + 'round-reopened': 'finishRound' }; function onNotification (data) { + logger.debug('pg-notify: Notification received:', {channel: data.channel, data: data.payload}); + // Broadcast notify via events if channel is supported if (channels[data.channel]) { - var round = parseInt(data.payload); - logger.debug('pg-notify: Round changes:', round); - bus.message(channels[data.channel], round); + // Process round-releated things + if (data.channel === 'round-closed') { + data.payload = parseInt(data.payload); + logger.info('pg-notify: Round closed:', data.payload); + // Set new round + data.payload += 1; + } else if (data.channel === 'round-reopened') { + data.payload = parseInt(data.payload); + logger.warn('pg-notify: Round reopened:', data.payload); + } + + // Propagate notification + bus.message(channels[data.channel], data.payload); } else { + // Channel is not supported logger.error('pg-notify: Invalid channel:', data.channel); } } diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 2ad05317487..1430d8b7f1b 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -265,18 +265,18 @@ CREATE TRIGGER vote_insert_delete -- Create function 'delegates_update_on_block' for updating 'delegates' table data CREATE FUNCTION delegates_update_on_block() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ - DECLARE - height int; BEGIN PERFORM delegates_voters_cnt_update(); PERFORM delegates_voters_balance_update(); PERFORM delegates_rank_update(); + -- Perform notification to backend that round changed IF (TG_OP = 'INSERT') THEN - height := NEW.height + 1; + -- Last block of round inserted - round is closed here and processing is done + PERFORM pg_notify('round-closed', CEIL(NEW.height / 101::float)::text); ELSIF (TG_OP = 'DELETE') THEN - height := OLD.height; + -- Last block of round deleted - round reopened, processing is done here + PERFORM pg_notify('round-reopened', CEIL(OLD.height / 101::float)::text); END IF; - PERFORM pg_notify('round', CEIL(height / 101::float)::text); RETURN NULL; END $$; From 036107b2504b078924cc719501f366c79051fb73 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 13:01:31 +0200 Subject: [PATCH 22/88] Listen/unlisten to all supported channels properly --- helpers/pg-notify.js | 59 +++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index dad7da7942e..ce4356b8392 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -12,32 +12,42 @@ module.exports.init = function (db, bus, logger, cb) { }; function onNotification (data) { - logger.debug('pg-notify: Notification received:', {channel: data.channel, data: data.payload}); + logger.debug('pg-notify: Notification received', {channel: data.channel, data: data.payload}); - // Broadcast notify via events if channel is supported - if (channels[data.channel]) { - // Process round-releated things - if (data.channel === 'round-closed') { - data.payload = parseInt(data.payload); - logger.info('pg-notify: Round closed:', data.payload); - // Set new round - data.payload += 1; - } else if (data.channel === 'round-reopened') { - data.payload = parseInt(data.payload); - logger.warn('pg-notify: Round reopened:', data.payload); - } + if (!channels[data.channel]) { + // Channel is not supported - should never happen + logger.error('pg-notify: Invalid channel', data.channel); + return; + } - // Propagate notification - bus.message(channels[data.channel], data.payload); - } else { - // Channel is not supported - logger.error('pg-notify: Invalid channel:', data.channel); + // Process round-releated things + if (data.channel === 'round-closed') { + data.payload = parseInt(data.payload); + logger.info('pg-notify: Round closed', data.payload); + // Set new round + data.payload += 1; + } else if (data.channel === 'round-reopened') { + data.payload = parseInt(data.payload); + logger.warn('pg-notify: Round reopened', data.payload); } + + // Broadcast notify via events + bus.message(channels[data.channel], data.payload); } function setListeners (client) { client.on('notification', onNotification); - connection.none('LISTEN $1~', 'round') + + // Generate list of queries for listen to every supported channels + function listenQueries (t) { + var queries = []; + Object.keys(channels).forEach(function (channel) { + queries.push(t.none('LISTEN $1~', channel)); + }); + return t.batch(queries); + } + + connection.task(listenQueries) .catch(function (err) { logger.error(err); }); @@ -45,7 +55,16 @@ module.exports.init = function (db, bus, logger, cb) { function removeListeners (client) { if (connection) { - connection.none('UNLISTEN $1~', 'my-channel') + // Generate list of queries for unlisten to every supported channels + function unlistenQueries (t) { + var queries = []; + Object.keys(channels).forEach(function (channel) { + queries.push(t.none('UNLISTEN $1~', channel)); + }); + return t.batch(queries); + } + + connection.task(unlistenQueries) .catch(function (err) { logger.error(err); }); From 4ecff425681741958f14590b7513f44244d2e4c2 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 13:13:51 +0200 Subject: [PATCH 23/88] Update 'pg-promise' dependency to latest varsion --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 04249a03177..f9e5ff84be4 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "npm": "=5.0.3", "pg-monitor": "=0.8.2", "pg-native": "=1.10.1", - "pg-promise": "=5.9.3", + "pg-promise": "=6.3.4", "popsicle": "=9.1.0", "randomstring": "=1.1.5", "redis": "=2.7.1", From 5eba226b10d0bb8a9eb8bd313a29cb14c9f703a1 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 15:55:33 +0200 Subject: [PATCH 24/88] Change log level for logging fail of initial connection --- helpers/pg-notify.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index ce4356b8392..94804f5348d 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -121,7 +121,8 @@ module.exports.init = function (db, bus, logger, cb) { return setImmediate(cb); }) .catch(function (err) { - logger.info('pg-notify: Initial connection failed', err); + logger.error('pg-notify: Initial connection failed', err); + // Error is passed to callback here, so node will not start in that case return setImmediate(cb, err); }); }; From 8aafed54c51ba1ff2aa78590617abd7bd6100188 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 15:57:07 +0200 Subject: [PATCH 25/88] Add 'init' tests for pg-notify helper --- test/index.js | 1 + test/unit/helpers/pg-notify.js | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 test/unit/helpers/pg-notify.js diff --git a/test/index.js b/test/index.js index 8f697da5fea..6ab5d519637 100644 --- a/test/index.js +++ b/test/index.js @@ -1,4 +1,5 @@ require('./unit/helpers/request-limiter.js'); +require('./unit/helpers/pg-notify.js'); require('./unit/logic/blockReward.js'); require('./unit/sql/blockRewards.js'); require('./unit/sql/delegatesList.js'); diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js new file mode 100644 index 00000000000..397af9a7274 --- /dev/null +++ b/test/unit/helpers/pg-notify.js @@ -0,0 +1,74 @@ +'use strict'; + +// Init tests dependencies +var chai = require('chai'); +var expect = require('chai').expect; +var sinon = require('sinon'); + +// Load config file - global (not one from test directory) +var config = require('../../../config.json'); + +// Init tests subject +var pg_notify = require('../../../helpers/pg-notify.js'); + +// Init global variables +var db, invalid_db, logger, bus; + +describe('helpers/pg-notify', function () { + + before(function (done) { + // Init dummy connection with database - valid, used for tests here + // We don't use pg-native here on purpose - it lacks some properties on notifications objects, and we need those to perform detailed tests + var pgp = require('pg-promise')(); + config.db.user = config.db.user || process.env.USER; + db = pgp(config.db); + + // Init dummy connection with database - invalid one + invalid_db = pgp({user: 'invalidUser'}); + + // Set spies for logger + logger = { + debug: sinon.spy(), + info: sinon.spy(), + error: sinon.spy() + }; + + // Set spy for bus + bus = { + message: sinon.spy() + }; + + done(); + }); + + beforeEach(function () { + // Reset state of spies + logger.debug.reset(); + logger.info.reset(); + logger.error.reset(); + bus.message.reset(); + }); + + describe('init', function () { + it('try to estabilish initial connection with valid params should succeed', function (done) { + pg_notify.init(db, bus, logger, function () { + expect(logger.info.args[0][0]).equal('pg-notify: Initial connection estabilished'); + done(); + }); + }); + + it('try to estabilish initial connection with invalid params should fail after 1 retry', function (done) { + pg_notify.init(invalid_db, bus, logger, function () { + // First try + expect(logger.error.args[0][0]).equal('pg-notify: Error connecting'); + expect(logger.error.args[0][1]).to.be.an('error'); + expect(logger.error.args[0][1].message).equal('password authentication failed for user "invalidUser"'); + // Retry + expect(logger.error.args[1][0]).equal('pg-notify: Initial connection failed'); + expect(logger.error.args[1][1]).to.be.an('error'); + expect(logger.error.args[1][1].message).equal('password authentication failed for user "invalidUser"'); + done(); + }); + }); + }); +}); From a6ea67115abb0eb8e2b6f93094b3bf4ae500a190 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 20:46:56 +0200 Subject: [PATCH 26/88] Refactor visibility, don't call process.exit on test env --- helpers/pg-notify.js | 195 +++++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 91 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index 94804f5348d..66793dc55dd 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -2,118 +2,131 @@ var Promise = require('bluebird'); -module.exports.init = function (db, bus, logger, cb) { - // Global connection for permanent event listeners - var connection; - // Map channels to bus.message events - var channels = { - 'round-closed': 'finishRound', - 'round-reopened': 'finishRound' - }; - - function onNotification (data) { - logger.debug('pg-notify: Notification received', {channel: data.channel, data: data.payload}); - - if (!channels[data.channel]) { - // Channel is not supported - should never happen - logger.error('pg-notify: Invalid channel', data.channel); - return; - } +// Init global module variables +var db, bus, logger; +// Global connection for permanent event listeners +var connection; +// Map channels to bus.message events +var channels = { + 'round-closed': 'finishRound', + 'round-reopened': 'finishRound' +}; - // Process round-releated things - if (data.channel === 'round-closed') { - data.payload = parseInt(data.payload); - logger.info('pg-notify: Round closed', data.payload); - // Set new round - data.payload += 1; - } else if (data.channel === 'round-reopened') { - data.payload = parseInt(data.payload); - logger.warn('pg-notify: Round reopened', data.payload); - } +function isTestEnv () { + return true ? process.env['NODE_ENV'] === 'TEST' : false; +} + +function onNotification (data) { + logger.debug('pg-notify: Notification received', {channel: data.channel, data: data.payload}); - // Broadcast notify via events - bus.message(channels[data.channel], data.payload); + if (!channels[data.channel]) { + // Channel is not supported - should never happen + logger.error('pg-notify: Invalid channel', data.channel); + return; } - function setListeners (client) { - client.on('notification', onNotification); + // Process round-releated things + if (data.channel === 'round-closed') { + data.payload = parseInt(data.payload); + logger.info('pg-notify: Round closed', data.payload); + // Set new round + data.payload += 1; + } else if (data.channel === 'round-reopened') { + data.payload = parseInt(data.payload); + logger.warn('pg-notify: Round reopened', data.payload); + } + + // Broadcast notify via events + bus.message(channels[data.channel], data.payload); +} + +function setListeners (client) { + client.on('notification', onNotification); + + // Generate list of queries for listen to every supported channels + function listenQueries (t) { + var queries = []; + Object.keys(channels).forEach(function (channel) { + queries.push(t.none('LISTEN $1~', channel)); + }); + return t.batch(queries); + } + + connection.task(listenQueries) + .catch(function (err) { + logger.error(err); + }); +} - // Generate list of queries for listen to every supported channels - function listenQueries (t) { +function removeListeners (client) { + if (connection) { + // Generate list of queries for unlisten to every supported channels + function unlistenQueries (t) { var queries = []; Object.keys(channels).forEach(function (channel) { - queries.push(t.none('LISTEN $1~', channel)); + queries.push(t.none('UNLISTEN $1~', channel)); }); return t.batch(queries); } - connection.task(listenQueries) + connection.task(unlistenQueries) .catch(function (err) { logger.error(err); }); } + client.removeListener('notification', onNotification); +} - function removeListeners (client) { - if (connection) { - // Generate list of queries for unlisten to every supported channels - function unlistenQueries (t) { - var queries = []; - Object.keys(channels).forEach(function (channel) { - queries.push(t.none('UNLISTEN $1~', channel)); - }); - return t.batch(queries); +function onConnectionLost (err, e) { + logger.error('pg-notify: Connection lost', err); + // Prevent use of the connection + connection = null; + removeListeners(e.client); + // Try to re-establish connection 10 times, every 5 seconds + reconnect(5000, 10) + .then(function (obj) { + logger.info('pg-notify: Reconnected successfully'); + }) + .catch(function () { + // Failed after 10 attempts + logger.error('pg-notify: Failed to reconnect - connection lost'); + // Kill node if we are not in test environment + if (!isTestEnv) { + process.exit(); } + }); +} - connection.task(unlistenQueries) +function reconnect (delay, maxAttempts) { + delay = delay > 0 ? delay : 0; + maxAttempts = maxAttempts > 0 ? maxAttempts : 1; + return new Promise(function (resolve, reject) { + setTimeout(function () { + db.connect({direct: true, onLost: onConnectionLost}) + .then(function (obj) { + // Global connection is now available + connection = obj; + setListeners(obj.client); + resolve(obj); + }) .catch(function (err) { - logger.error(err); + logger.error('pg-notify: Error connecting', err); + if (--maxAttempts) { + reconnect(delay, maxAttempts) + .then(resolve) + .catch(reject); + } else { + reject(err); + } }); - } - client.removeListener('notification', onNotification); - } + }, delay); + }); +} - function onConnectionLost (err, e) { - logger.error('pg-notify: Connection lost', err); - // Prevent use of the connection - connection = null; - removeListeners(e.client); - // Try to re-establish connection 10 times, every 5 seconds - reconnect(5000, 10) - .then(function (obj) { - logger.info('pg-notify: Reconnected successfully'); - }) - .catch(function () { - // Failed after 10 attempts - logger.error('pg-notify: Failed to reconnect - connection lost'); - process.exit(); - }); - } - - function reconnect (delay, maxAttempts) { - delay = delay > 0 ? delay : 0; - maxAttempts = maxAttempts > 0 ? maxAttempts : 1; - return new Promise(function (resolve, reject) { - setTimeout(function () { - db.connect({direct: true, onLost: onConnectionLost}) - .then(function (obj) { - // Global connection is now available - connection = obj; - setListeners(obj.client); - resolve(obj); - }) - .catch(function (err) { - logger.error('pg-notify: Error connecting', err); - if (--maxAttempts) { - reconnect(delay, maxAttempts) - .then(resolve) - .catch(reject); - } else { - reject(err); - } - }); - }, delay); - }); - } +module.exports.init = function (_db, _bus, _logger, cb) { + db = _db; + bus = _bus; + logger = _logger; reconnect () .then(function (obj) { From 8bbf0a2a9bcf67c5e804b21ad40e9173a6c36411 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 20:48:22 +0200 Subject: [PATCH 27/88] Added more tests for pg-notify helper --- package.json | 1 + test/sql/pgNotify.js | 7 ++ test/unit/helpers/pg-notify.js | 146 +++++++++++++++++++++++++++++++-- 3 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 test/sql/pgNotify.js diff --git a/package.json b/package.json index f9e5ff84be4..9ac0a9f1cdf 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "lisk-js": "=0.4.3", "mocha": "=3.4.2", "moment": "=2.18.1", + "rewire": "=2.5.2", "sinon": "=2.3.4", "supertest": "=3.0.0" } diff --git a/test/sql/pgNotify.js b/test/sql/pgNotify.js new file mode 100644 index 00000000000..618e277ae13 --- /dev/null +++ b/test/sql/pgNotify.js @@ -0,0 +1,7 @@ +'use strict'; + +var pgNotify = { + interruptConnection: 'SELECT pg_terminate_backend(${pid});' +}; + +module.exports = pgNotify; diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 397af9a7274..52db0e5205b 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -4,12 +4,14 @@ var chai = require('chai'); var expect = require('chai').expect; var sinon = require('sinon'); +var rewire = require('rewire'); // Load config file - global (not one from test directory) var config = require('../../../config.json'); +var sql = require('../../sql/pgNotify.js'); // Init tests subject -var pg_notify = require('../../../helpers/pg-notify.js'); +var pg_notify = rewire('../../../helpers/pg-notify.js'); // Init global variables var db, invalid_db, logger, bus; @@ -41,34 +43,160 @@ describe('helpers/pg-notify', function () { done(); }); - beforeEach(function () { + function resetSpiesState () { // Reset state of spies logger.debug.reset(); logger.info.reset(); logger.error.reset(); bus.message.reset(); + } + + beforeEach(function () { + resetSpiesState(); }); describe('init', function () { it('try to estabilish initial connection with valid params should succeed', function (done) { - pg_notify.init(db, bus, logger, function () { - expect(logger.info.args[0][0]).equal('pg-notify: Initial connection estabilished'); + pg_notify.init(db, bus, logger, function (err) { + // Should be no error + expect(err).to.be.an('undefined'); + expect(logger.info.args[0][0]).to.equal('pg-notify: Initial connection estabilished'); done(); }); }); it('try to estabilish initial connection with invalid params should fail after 1 retry', function (done) { - pg_notify.init(invalid_db, bus, logger, function () { + pg_notify.init(invalid_db, bus, logger, function (err) { + var err_msg = 'password authentication failed for user "invalidUser"'; + // Error should propagate + expect(err).to.be.an('error'); + expect(err.message).to.equal(err_msg); // First try - expect(logger.error.args[0][0]).equal('pg-notify: Error connecting'); + expect(logger.error.args[0][0]).to.equal('pg-notify: Error connecting'); expect(logger.error.args[0][1]).to.be.an('error'); - expect(logger.error.args[0][1].message).equal('password authentication failed for user "invalidUser"'); + expect(logger.error.args[0][1].message).to.equal(err_msg); // Retry - expect(logger.error.args[1][0]).equal('pg-notify: Initial connection failed'); + expect(logger.error.args[1][0]).to.equal('pg-notify: Initial connection failed'); expect(logger.error.args[1][1]).to.be.an('error'); - expect(logger.error.args[1][1].message).equal('password authentication failed for user "invalidUser"'); + expect(logger.error.args[1][1].message).to.equal(err_msg); done(); }); }); }); + + describe('setListeners', function () { + it('listeners should be set correctly after successfull connection', function (done) { + var setListeners = pg_notify.__get__('setListeners'); + var connection = pg_notify.__get__('connection'); + var onNotification = pg_notify.__get__('onNotification'); + + expect(setListeners).to.be.an('function'); + expect(connection).to.be.an('object').and.have.property('client'); + expect(connection.client._events.notification).to.be.an('function'); + expect(connection.client._events.notification).equal(onNotification); + done(); + }); + }); + + describe('isTestEnv', function () { + it('should return true if NODE_ENV is TEST', function (done) { + var node_env = process.env['NODE_ENV']; + + process.env['NODE_ENV'] = 'TEST'; + var isTestEnv = pg_notify.__get__('isTestEnv'); + expect(isTestEnv()).to.be.ok; + process.env['NODE_ENV'] = node_env; + done(); + }); + + it('should return false if NODE_ENV is not TEST', function (done) { + var node_env = process.env['NODE_ENV']; + + process.env['NODE_ENV'] = 'PRODUCTION'; + var isTestEnv = pg_notify.__get__('isTestEnv'); + expect(isTestEnv()).to.be.not.ok; + process.env['NODE_ENV'] = node_env; + done(); + }); + }); + + describe('onConnectionLost', function () { + it('should fail after 10 retries if cannot reconnect', function (done) { + // Spy private functions + var setListeners = pg_notify.__get__('setListeners'); + var connection = pg_notify.__get__('connection'); + + // Execute query that terminate existing connection + db.query(sql.interruptConnection, {database: config.db.database, pid: connection.client.processID}).then(setTimeout(function () { + // 12 errors should be collected + expect(logger.error.args).to.be.an('array').and.lengthOf(12); + + // First error is caused by our test SQL query + expect(logger.error.args[0][0]).to.equal('pg-notify: Connection lost'); + expect(logger.error.args[0][1]).to.be.an('error'); + expect(logger.error.args[0][1].message).to.equal('terminating connection due to administrator command'); + + var errors = logger.error.args.slice(1, 11); + // Iterating over errors (failed retires) + for (var i = errors.length - 1; i >= 0; i--) { + expect(errors[0][0]).to.equal('pg-notify: Error connecting'); + expect(errors[0][1]).to.be.an('error'); + expect(errors[0][1].message).to.equal('password authentication failed for user "invalidUser"'); + } + + // Last error - function should fail to reconnect + expect(logger.error.args[11][0]).to.equal('pg-notify: Failed to reconnect - connection lost'); + + //Connection should be cleared + connection = pg_notify.__get__('connection'); + expect(connection).to.be.an('null'); + + done(); + }, 60000)).catch(function (err) { + done(err); + }); + }); + + it('should reconnect successfully if it\'s possible', function (done) { + // Re-init connection + pg_notify.init(db, bus, logger, function (err) { + // Should be no error + expect(err).to.be.an('undefined'); + expect(logger.info.args[0][0]).to.equal('pg-notify: Initial connection estabilished'); + + // Spy private functions + var setListeners = pg_notify.__get__('setListeners'); + var connection = pg_notify.__get__('connection'); + + resetSpiesState(); + + // Execute query that terminate existing connection + db.query(sql.interruptConnection, {pid: connection.client.processID}).then(setTimeout(function () { + expect(logger.error.args[0][0]).to.equal('pg-notify: Connection lost'); + expect(logger.error.args[0][1]).to.be.an('error'); + expect(logger.error.args[0][1].message).to.equal('terminating connection due to administrator command'); + + expect(logger.info.args[0][0]).to.equal('pg-notify: Reconnected successfully'); + done(); + }, 10000)).catch(function (err) { + done(err); + }); //db.query + }); // pg_notify.init + }); // it + }); + + describe('removeListeners', function () { + it('listeners should be removed correctly', function (done) { + var removeListeners = pg_notify.__get__('removeListeners'); + var connection = pg_notify.__get__('connection'); + removeListeners(connection.client); + + expect(removeListeners).to.be.an('function'); + expect(connection).to.be.an('object').and.have.property('client'); + expect(connection.client._events.notification).to.be.an('undefined'); + done(); + }); + }); + + }); From 2c4d00177ab05a1fc9433d618fc47767cffbf3f1 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 23:11:16 +0200 Subject: [PATCH 28/88] Better error handling for pg-notify helper --- helpers/pg-notify.js | 71 +++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index 66793dc55dd..b342f99b936 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -40,48 +40,60 @@ function onNotification (data) { bus.message(channels[data.channel], data.payload); } -function setListeners (client) { - client.on('notification', onNotification); +// Generate list of queries for listen to every supported channels +function listenQueries (t) { + var queries = []; + Object.keys(channels).forEach(function (channel) { + queries.push(t.none('LISTEN $1~', channel)); + }); + return t.batch(queries); +} - // Generate list of queries for listen to every supported channels - function listenQueries (t) { - var queries = []; - Object.keys(channels).forEach(function (channel) { - queries.push(t.none('LISTEN $1~', channel)); - }); - return t.batch(queries); - } +function setListeners (client, cb) { + client.on('notification', onNotification); connection.task(listenQueries) + .then(function () { + return setImmediate(cb); + }) .catch(function (err) { - logger.error(err); - }); + logger.error('pg-notify: Failed to execute LISTEN queries', err); + return setImmediate(cb, err) + }) } -function removeListeners (client) { - if (connection) { - // Generate list of queries for unlisten to every supported channels - function unlistenQueries (t) { - var queries = []; - Object.keys(channels).forEach(function (channel) { - queries.push(t.none('UNLISTEN $1~', channel)); - }); - return t.batch(queries); - } +// Generate list of queries for unlisten to every supported channels +function unlistenQueries (t) { + var queries = []; + Object.keys(channels).forEach(function (channel) { + queries.push(t.none('UNLISTEN $1~', channel)); + }); + return t.batch(queries); +} +function removeListeners (client, cb) { + client.removeListener('notification', onNotification); + + if (connection) { connection.task(unlistenQueries) + .then(function () { + return setImmediate(cb); + }) .catch(function (err) { - logger.error(err); + logger.error('pg-notify: Failed to execute UNLISTEN queries', err); + return setImmediate(cb, err); }); + } else { + return setImmediate(cb); } - client.removeListener('notification', onNotification); } function onConnectionLost (err, e) { logger.error('pg-notify: Connection lost', err); // Prevent use of the connection connection = null; - removeListeners(e.client); + // We don't care about error here, so passing empty function as callback + removeListeners(e.client, function () {}); // Try to re-establish connection 10 times, every 5 seconds reconnect(5000, 10) .then(function (obj) { @@ -106,8 +118,13 @@ function reconnect (delay, maxAttempts) { .then(function (obj) { // Global connection is now available connection = obj; - setListeners(obj.client); - resolve(obj); + setListeners(obj.client, function (err) { + if (err) { + reject(err); + } else { + resolve(obj); + } + }); }) .catch(function (err) { logger.error('pg-notify: Error connecting', err); From d2576646e04a5b0304216d2328dacb0a63d6272b Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 23:12:22 +0200 Subject: [PATCH 29/88] Added/refactored tests for pg-notify helper --- test/unit/helpers/pg-notify.js | 193 ++++++++++++++++++++++++--------- 1 file changed, 139 insertions(+), 54 deletions(-) diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 52db0e5205b..fcb04b401fd 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -51,8 +51,24 @@ describe('helpers/pg-notify', function () { bus.message.reset(); } - beforeEach(function () { + function failQueryBatch (t) { + var queries = []; + queries.push(t.none('SELECT invalid_sql_query')); + return t.batch(queries); + } + + function reconnect (done) { + pg_notify.init(db, bus, logger, function (err) { + // Should be no error + expect(err).to.be.an('undefined'); + expect(logger.info.args[0][0]).to.equal('pg-notify: Initial connection estabilished'); + done(); + }); + } + + beforeEach(function (done) { resetSpiesState(); + reconnect(done); }); describe('init', function () { @@ -82,10 +98,33 @@ describe('helpers/pg-notify', function () { done(); }); }); + + it('try to estabilish initial connection with valid params but error during LISTEN queries should fail after 1 retry', function (done) { + // Spy private functions + var setListeners = pg_notify.__get__('setListeners'); + var connection = pg_notify.__get__('connection'); + // Overwrite listenQueries function with one that always fail + var restore = pg_notify.__set__('listenQueries', failQueryBatch); + + pg_notify.init(db, bus, logger, function (err) { + var err_msg = 'column "invalid_sql_query" does not exist'; + // Error should propagate + expect(err).to.deep.include({name: 'BatchError', message: err_msg}); + // First try + expect(logger.error.args[0][0]).to.equal('pg-notify: Failed to execute LISTEN queries'); + expect(logger.error.args[0][1]).to.deep.include({name: 'BatchError', message: err_msg}); + // Retry + expect(logger.error.args[1][0]).to.equal('pg-notify: Initial connection failed'); + expect(logger.error.args[1][1]).to.deep.include({name: 'BatchError', message: err_msg}); + restore(); + done(); + }); + }); }); describe('setListeners', function () { it('listeners should be set correctly after successfull connection', function (done) { + // Spy private functions var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); var onNotification = pg_notify.__get__('onNotification'); @@ -96,6 +135,21 @@ describe('helpers/pg-notify', function () { expect(connection.client._events.notification).equal(onNotification); done(); }); + + it('should fail if error occurred during LISTEN queries', function (done) { + // Spy private functions + var setListeners = pg_notify.__get__('setListeners'); + var connection = pg_notify.__get__('connection'); + // Overwrite listenQueries function with one that always fail + var restore = pg_notify.__set__('listenQueries', failQueryBatch); + + setListeners(connection.client, function (err) { + expect(logger.error.args[0][0]).to.equal('pg-notify: Failed to execute LISTEN queries'); + expect(err).to.deep.include({name: 'BatchError', message: 'column "invalid_sql_query" does not exist'}); + restore(); + return done(); + }); + }); }); describe('isTestEnv', function () { @@ -122,79 +176,110 @@ describe('helpers/pg-notify', function () { describe('onConnectionLost', function () { it('should fail after 10 retries if cannot reconnect', function (done) { - // Spy private functions - var setListeners = pg_notify.__get__('setListeners'); - var connection = pg_notify.__get__('connection'); - - // Execute query that terminate existing connection - db.query(sql.interruptConnection, {database: config.db.database, pid: connection.client.processID}).then(setTimeout(function () { - // 12 errors should be collected - expect(logger.error.args).to.be.an('array').and.lengthOf(12); - - // First error is caused by our test SQL query - expect(logger.error.args[0][0]).to.equal('pg-notify: Connection lost'); - expect(logger.error.args[0][1]).to.be.an('error'); - expect(logger.error.args[0][1].message).to.equal('terminating connection due to administrator command'); - - var errors = logger.error.args.slice(1, 11); - // Iterating over errors (failed retires) - for (var i = errors.length - 1; i >= 0; i--) { - expect(errors[0][0]).to.equal('pg-notify: Error connecting'); - expect(errors[0][1]).to.be.an('error'); - expect(errors[0][1].message).to.equal('password authentication failed for user "invalidUser"'); - } - - // Last error - function should fail to reconnect - expect(logger.error.args[11][0]).to.equal('pg-notify: Failed to reconnect - connection lost'); - - //Connection should be cleared - connection = pg_notify.__get__('connection'); - expect(connection).to.be.an('null'); - - done(); - }, 60000)).catch(function (err) { - done(err); - }); - }); - - it('should reconnect successfully if it\'s possible', function (done) { // Re-init connection - pg_notify.init(db, bus, logger, function (err) { - // Should be no error - expect(err).to.be.an('undefined'); - expect(logger.info.args[0][0]).to.equal('pg-notify: Initial connection estabilished'); + pg_notify.init(invalid_db, bus, logger, function (err) { + resetSpiesState(); // Spy private functions var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); - resetSpiesState(); - // Execute query that terminate existing connection - db.query(sql.interruptConnection, {pid: connection.client.processID}).then(setTimeout(function () { + db.query(sql.interruptConnection, {database: config.db.database, pid: connection.client.processID}).then(setTimeout(function () { + // 12 errors should be collected + expect(logger.error.args).to.be.an('array').and.lengthOf(12); + + // First error is caused by our test SQL query expect(logger.error.args[0][0]).to.equal('pg-notify: Connection lost'); expect(logger.error.args[0][1]).to.be.an('error'); expect(logger.error.args[0][1].message).to.equal('terminating connection due to administrator command'); - expect(logger.info.args[0][0]).to.equal('pg-notify: Reconnected successfully'); + var errors = logger.error.args.slice(1, 11); + // Iterating over errors (failed retires) + for (var i = errors.length - 1; i >= 0; i--) { + expect(errors[0][0]).to.equal('pg-notify: Error connecting'); + expect(errors[0][1]).to.be.an('error'); + expect(errors[0][1].message).to.equal('password authentication failed for user "invalidUser"'); + } + + // Last error - function should fail to reconnect + expect(logger.error.args[11][0]).to.equal('pg-notify: Failed to reconnect - connection lost'); + + //Connection should be cleared + connection = pg_notify.__get__('connection'); + expect(connection).to.be.an('null'); + done(); - }, 10000)).catch(function (err) { + }, 60000)).catch(function (err) { done(err); - }); //db.query - }); // pg_notify.init - }); // it + }); + }); + }); + + it('should reconnect successfully if it\'s possible', function (done) { + // Spy private functions + var setListeners = pg_notify.__get__('setListeners'); + var connection = pg_notify.__get__('connection'); + + resetSpiesState(); + + // Execute query that terminate existing connection + db.query(sql.interruptConnection, {pid: connection.client.processID}).then(setTimeout(function () { + expect(logger.error.args[0][0]).to.equal('pg-notify: Connection lost'); + expect(logger.error.args[0][1]).to.be.an('error'); + expect(logger.error.args[0][1].message).to.equal('terminating connection due to administrator command'); + + expect(logger.info.args[0][0]).to.equal('pg-notify: Reconnected successfully'); + done(); + }, 10000)).catch(function (err) { + done(err); + }); + }); }); describe('removeListeners', function () { it('listeners should be removed correctly', function (done) { + // Spy private functions var removeListeners = pg_notify.__get__('removeListeners'); var connection = pg_notify.__get__('connection'); - removeListeners(connection.client); - expect(removeListeners).to.be.an('function'); - expect(connection).to.be.an('object').and.have.property('client'); - expect(connection.client._events.notification).to.be.an('undefined'); - done(); + removeListeners(connection.client, function (err) { + expect(removeListeners).to.be.an('function'); + expect(connection).to.be.an('object').and.have.property('client'); + expect(connection.client._events.notification).to.be.an('undefined'); + done(); + }); + }); + + it('listeners should be removed correctly even if error occurred during UNLISTEN queries', function (done) { + // Spy private functions + var removeListeners = pg_notify.__get__('removeListeners'); + var connection = pg_notify.__get__('connection'); + // Overwrite listenQueries function with one that always fail + var restore = pg_notify.__set__('unlistenQueries', failQueryBatch); + + removeListeners(connection.client, function (err) { + expect(logger.error.args[0][0]).to.equal('pg-notify: Failed to execute UNLISTEN queries'); + expect(err).to.deep.include({name: 'BatchError', message: 'column "invalid_sql_query" does not exist'}); + restore(); + done(); + }); + }); + + it('listeners should be removed correctly even if connection is null', function (done) { + // Spy private functions + var removeListeners = pg_notify.__get__('removeListeners'); + var connection = pg_notify.__get__('connection'); + // Overwrite connection object with null + var restore = pg_notify.__set__('connection', null); + + removeListeners(connection.client, function (err) { + expect(removeListeners).to.be.an('function'); + expect(connection).to.be.an('object').and.have.property('client'); + expect(connection.client._events.notification).to.be.an('undefined'); + restore(); + done(); + }); }); }); From e1f45b0316ca689dc46a493388c142559ca7403a Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 10 Jul 2017 23:15:52 +0200 Subject: [PATCH 30/88] Fix eslint errors --- helpers/pg-notify.js | 8 ++++---- test/unit/helpers/pg-notify.js | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index b342f99b936..7424ff93a80 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -44,7 +44,7 @@ function onNotification (data) { function listenQueries (t) { var queries = []; Object.keys(channels).forEach(function (channel) { - queries.push(t.none('LISTEN $1~', channel)); + queries.push(t.none('LISTEN $1~', channel)); }); return t.batch(queries); } @@ -58,15 +58,15 @@ function setListeners (client, cb) { }) .catch(function (err) { logger.error('pg-notify: Failed to execute LISTEN queries', err); - return setImmediate(cb, err) - }) + return setImmediate(cb, err); + }); } // Generate list of queries for unlisten to every supported channels function unlistenQueries (t) { var queries = []; Object.keys(channels).forEach(function (channel) { - queries.push(t.none('UNLISTEN $1~', channel)); + queries.push(t.none('UNLISTEN $1~', channel)); }); return t.batch(queries); } diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index fcb04b401fd..09200c5d579 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -53,7 +53,7 @@ describe('helpers/pg-notify', function () { function failQueryBatch (t) { var queries = []; - queries.push(t.none('SELECT invalid_sql_query')); + queries.push(t.none('SELECT invalid_sql_query')); return t.batch(queries); } @@ -282,6 +282,4 @@ describe('helpers/pg-notify', function () { }); }); }); - - }); From 2770d9306d5d9feba09ecbcda08c147c4ca3b6c3 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 11 Jul 2017 00:44:23 +0200 Subject: [PATCH 31/88] Improved error handling, remove isTestEnv function --- helpers/pg-notify.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index 7424ff93a80..8f953d9ed4f 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -12,15 +12,11 @@ var channels = { 'round-reopened': 'finishRound' }; -function isTestEnv () { - return true ? process.env['NODE_ENV'] === 'TEST' : false; -} - function onNotification (data) { logger.debug('pg-notify: Notification received', {channel: data.channel, data: data.payload}); if (!channels[data.channel]) { - // Channel is not supported - should never happen + // Channel is invalid - should never happen logger.error('pg-notify: Invalid channel', data.channel); return; } @@ -34,6 +30,10 @@ function onNotification (data) { } else if (data.channel === 'round-reopened') { data.payload = parseInt(data.payload); logger.warn('pg-notify: Round reopened', data.payload); + } else { + // Channel is not supported - should never happen + logger.error('pg-notify: Channel not supported', data.channel); + return; } // Broadcast notify via events @@ -102,10 +102,7 @@ function onConnectionLost (err, e) { .catch(function () { // Failed after 10 attempts logger.error('pg-notify: Failed to reconnect - connection lost'); - // Kill node if we are not in test environment - if (!isTestEnv) { - process.exit(); - } + process.exit(); }); } From df30d35b5b7e9aa4c5385fb0f31c64ea4949f720 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 11 Jul 2017 00:46:18 +0200 Subject: [PATCH 32/88] Add triggerNotify SQL query for pg-notify tests --- test/sql/pgNotify.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sql/pgNotify.js b/test/sql/pgNotify.js index 618e277ae13..bf974f4b7ec 100644 --- a/test/sql/pgNotify.js +++ b/test/sql/pgNotify.js @@ -1,7 +1,8 @@ 'use strict'; var pgNotify = { - interruptConnection: 'SELECT pg_terminate_backend(${pid});' + interruptConnection: 'SELECT pg_terminate_backend(${pid});', + triggerNotify: 'SELECT pg_notify(${channel}, ${message});' }; module.exports = pgNotify; From 33be545a24633ab41b242225a089ace878872e3a Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 11 Jul 2017 00:48:43 +0200 Subject: [PATCH 33/88] Add tests for 'onNotification', removed tests for 'isTestEnv' --- test/unit/helpers/pg-notify.js | 135 ++++++++++++++++++++++++++------- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 09200c5d579..a60b1ed1997 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -32,6 +32,7 @@ describe('helpers/pg-notify', function () { logger = { debug: sinon.spy(), info: sinon.spy(), + warn: sinon.spy(), error: sinon.spy() }; @@ -47,6 +48,7 @@ describe('helpers/pg-notify', function () { // Reset state of spies logger.debug.reset(); logger.info.reset(); + logger.warn.reset(); logger.error.reset(); bus.message.reset(); } @@ -60,8 +62,9 @@ describe('helpers/pg-notify', function () { function reconnect (done) { pg_notify.init(db, bus, logger, function (err) { // Should be no error - expect(err).to.be.an('undefined'); + expect(err).to.be.undefined; expect(logger.info.args[0][0]).to.equal('pg-notify: Initial connection estabilished'); + resetSpiesState(); done(); }); } @@ -75,7 +78,7 @@ describe('helpers/pg-notify', function () { it('try to estabilish initial connection with valid params should succeed', function (done) { pg_notify.init(db, bus, logger, function (err) { // Should be no error - expect(err).to.be.an('undefined'); + expect(err).to.be.undefined; expect(logger.info.args[0][0]).to.equal('pg-notify: Initial connection estabilished'); done(); }); @@ -152,28 +155,6 @@ describe('helpers/pg-notify', function () { }); }); - describe('isTestEnv', function () { - it('should return true if NODE_ENV is TEST', function (done) { - var node_env = process.env['NODE_ENV']; - - process.env['NODE_ENV'] = 'TEST'; - var isTestEnv = pg_notify.__get__('isTestEnv'); - expect(isTestEnv()).to.be.ok; - process.env['NODE_ENV'] = node_env; - done(); - }); - - it('should return false if NODE_ENV is not TEST', function (done) { - var node_env = process.env['NODE_ENV']; - - process.env['NODE_ENV'] = 'PRODUCTION'; - var isTestEnv = pg_notify.__get__('isTestEnv'); - expect(isTestEnv()).to.be.not.ok; - process.env['NODE_ENV'] = node_env; - done(); - }); - }); - describe('onConnectionLost', function () { it('should fail after 10 retries if cannot reconnect', function (done) { // Re-init connection @@ -184,6 +165,8 @@ describe('helpers/pg-notify', function () { var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); + var exit = sinon.stub(process, 'exit'); + // Execute query that terminate existing connection db.query(sql.interruptConnection, {database: config.db.database, pid: connection.client.processID}).then(setTimeout(function () { // 12 errors should be collected @@ -209,6 +192,9 @@ describe('helpers/pg-notify', function () { connection = pg_notify.__get__('connection'); expect(connection).to.be.an('null'); + expect(exit.calledOnce).to.be.ok; + exit.restore(); + done(); }, 60000)).catch(function (err) { done(err); @@ -246,7 +232,7 @@ describe('helpers/pg-notify', function () { removeListeners(connection.client, function (err) { expect(removeListeners).to.be.an('function'); expect(connection).to.be.an('object').and.have.property('client'); - expect(connection.client._events.notification).to.be.an('undefined'); + expect(connection.client._events.notification).to.be.undefined; done(); }); }); @@ -255,7 +241,7 @@ describe('helpers/pg-notify', function () { // Spy private functions var removeListeners = pg_notify.__get__('removeListeners'); var connection = pg_notify.__get__('connection'); - // Overwrite listenQueries function with one that always fail + // Overwrite unlistenQueries function with one that always fail var restore = pg_notify.__set__('unlistenQueries', failQueryBatch); removeListeners(connection.client, function (err) { @@ -276,10 +262,105 @@ describe('helpers/pg-notify', function () { removeListeners(connection.client, function (err) { expect(removeListeners).to.be.an('function'); expect(connection).to.be.an('object').and.have.property('client'); - expect(connection.client._events.notification).to.be.an('undefined'); + expect(connection.client._events.notification).to.be.undefined; + restore(); + done(); + }); + }); + }); + + describe('onNotification', function () { + it('should notify about round-closed event', function (done) { + var channel = 'round-closed'; + var message = '123'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: '123'}]); + expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed', 123]); + expect(bus.message.args[0]).to.deep.equal(['finishRound', 124]); + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should notify about round-reopened event', function (done) { + var channel = 'round-reopened'; + var message = '123'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: '123'}]); + expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened', 123]); + expect(bus.message.args[0]).to.deep.equal(['finishRound', 123]); + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should not notify about unknown event', function (done) { + var channel = 'unknown'; + var message = 'unknown'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.be.undefined; + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.be.undefined; + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should not notify about event on invalid channel, but log it', function (done) { + var channel = 'round-reopened'; + var message = '123'; + + // Overwrite channels object with custom ones + var restore = pg_notify.__set__('channels', {}); + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: '123'}]); + expect(logger.error.args[0]).to.deep.equal(['pg-notify: Invalid channel', 'round-reopened']); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.be.undefined; restore(); done(); + }, 20)).catch(function (err) { + done(err); }); }); + + it('should not notify about event on not supported channel, but log it', function (done) { + var channel = 'test'; + var message = '123'; + + // Overwrite channels object with custom ones + var restore = pg_notify.__set__('channels', {test: 'test'}); + + pg_notify.init(db, bus, logger, function (err) { + resetSpiesState(); + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'test', data: '123'}]); + expect(logger.error.args[0]).to.deep.equal(['pg-notify: Channel not supported', 'test']); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.be.undefined; + restore(); + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + }); + }); }); From a153365f5b6a9065f36d4645e5a07d6604ea8390 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 11 Jul 2017 02:48:46 +0200 Subject: [PATCH 34/88] Upgrade 'pg-promise' dependency to 6.3.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9ac0a9f1cdf..fe0bdb5776a 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "npm": "=5.0.3", "pg-monitor": "=0.8.2", "pg-native": "=1.10.1", - "pg-promise": "=6.3.4", + "pg-promise": "=6.3.5", "popsicle": "=9.1.0", "randomstring": "=1.1.5", "redis": "=2.7.1", From 9c80dcd6c16f0fc70c223b60823eeaf8217572b2 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 11 Jul 2017 02:49:57 +0200 Subject: [PATCH 35/88] Fix indexes bug in pg-notify tests --- test/unit/helpers/pg-notify.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index a60b1ed1997..e029afa1ab0 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -180,9 +180,9 @@ describe('helpers/pg-notify', function () { var errors = logger.error.args.slice(1, 11); // Iterating over errors (failed retires) for (var i = errors.length - 1; i >= 0; i--) { - expect(errors[0][0]).to.equal('pg-notify: Error connecting'); - expect(errors[0][1]).to.be.an('error'); - expect(errors[0][1].message).to.equal('password authentication failed for user "invalidUser"'); + expect(errors[i][0]).to.equal('pg-notify: Error connecting'); + expect(errors[i][1]).to.be.an('error'); + expect(errors[i][1].message).to.equal('password authentication failed for user "invalidUser"'); } // Last error - function should fail to reconnect From b548cc8b4d3c46a601e2fe5ce1a3a206c2b1d3da Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 11 Jul 2017 14:54:28 +0200 Subject: [PATCH 36/88] Removed unused property, improve comment --- test/unit/helpers/pg-notify.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index e029afa1ab0..41395c832c4 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -20,7 +20,7 @@ describe('helpers/pg-notify', function () { before(function (done) { // Init dummy connection with database - valid, used for tests here - // We don't use pg-native here on purpose - it lacks some properties on notifications objects, and we need those to perform detailed tests + // We don't use pg-native here on purpose - it lacks connection.client.processID, and we need it to perform reconnect tests var pgp = require('pg-promise')(); config.db.user = config.db.user || process.env.USER; db = pgp(config.db); @@ -168,7 +168,7 @@ describe('helpers/pg-notify', function () { var exit = sinon.stub(process, 'exit'); // Execute query that terminate existing connection - db.query(sql.interruptConnection, {database: config.db.database, pid: connection.client.processID}).then(setTimeout(function () { + db.query(sql.interruptConnection, {pid: connection.client.processID}).then(setTimeout(function () { // 12 errors should be collected expect(logger.error.args).to.be.an('array').and.lengthOf(12); From e3983910e63c8220b2658be19c944b9051479540 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 13 Jul 2017 17:32:31 +0200 Subject: [PATCH 37/88] Fix bug in jobsQueue, added unit tests for it --- helpers/jobsQueue.js | 12 ++- test/index.js | 1 + test/unit/helpers/jobs-queue.js | 135 ++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 test/unit/helpers/jobs-queue.js diff --git a/helpers/jobsQueue.js b/helpers/jobsQueue.js index fe4efcc4fc6..d2926b6cf70 100644 --- a/helpers/jobsQueue.js +++ b/helpers/jobsQueue.js @@ -5,18 +5,24 @@ var jobsQueue = { jobs: {}, register: function (name, job, time) { - if (this.jobs[name]) { + // Check if job is already registered - we check only if property exists, because value can be undefined + if (hasOwnProperty.call(this.jobs, name)) { throw new Error('Synchronous job ' + name + ' already registered'); } + // Check if job is function, name is string and time is integer + if (!job || Object.prototype.toString.call(job) !== '[object Function]' || typeof name !== 'string' || !Number.isInteger(time)) { + throw new Error('Syntax error - invalid parameters supplied'); + } + var nextJob = function () { return job(function () { jobsQueue.jobs[name] = setTimeout(nextJob, time); }); }; - nextJob(); - return this.jobs[name]; + jobsQueue.jobs[name] = setTimeout(nextJob, time); + return jobsQueue.jobs[name]; } }; diff --git a/test/index.js b/test/index.js index 6ab5d519637..d332c86f9ad 100644 --- a/test/index.js +++ b/test/index.js @@ -1,5 +1,6 @@ require('./unit/helpers/request-limiter.js'); require('./unit/helpers/pg-notify.js'); +require('./unit/helpers/jobs-queue.js'); require('./unit/logic/blockReward.js'); require('./unit/sql/blockRewards.js'); require('./unit/sql/delegatesList.js'); diff --git a/test/unit/helpers/jobs-queue.js b/test/unit/helpers/jobs-queue.js new file mode 100644 index 00000000000..6db9b1154bb --- /dev/null +++ b/test/unit/helpers/jobs-queue.js @@ -0,0 +1,135 @@ +'use strict'; + +// Init tests dependencies +var chai = require('chai'); +var expect = require('chai').expect; +var sinon = require('sinon'); +var rewire = require('rewire'); + +// Init tests subject +var jobsQueue = require('../../../helpers/jobsQueue.js'); +var peers = rewire('../../../modules/peers'); + +// Global variables +var clock; +var recallInterval = 1000; +var execTimeInterval = 1; + +function dummyFunction (cb) { + setTimeout(cb, execTimeInterval); +} + +function testExecution (job, name, spy, done) { + expect(jobsQueue.jobs).to.be.an('object'); + // Job returned from 'register' should be equal to one in 'jobsQueue' + expect(job).to.equal(jobsQueue.jobs[name]); + // Shouldn't be called before recallInterval + expect(spy.callCount).to.equal(0); + clock.tick(recallInterval/2); + expect(spy.callCount).to.equal(0); + clock.tick(recallInterval/2); + // Should be called once after recallInterval + expect(spy.callCount).to.equal(1); + // Should be called once before execTimeInterval + clock.tick(execTimeInterval/2); + expect(spy.callCount).to.equal(1); + clock.tick(execTimeInterval/2); + expect(spy.callCount).to.equal(1); + // Should be called twice after another execTimeInterval+recallInterval + clock.tick(recallInterval); + expect(spy.callCount).to.equal(2); + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' + expect(job).to.not.equal(jobsQueue.jobs[name]); + // Should be called thrice after another execTimeInterval+recallInterval + clock.tick(execTimeInterval+recallInterval); + expect(spy.callCount).to.equal(3); + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' + expect(job).to.not.equal(jobsQueue.jobs[name]); +} + +beforeEach(function () { + clock = sinon.useFakeTimers(); +}); + +afterEach(function () { + clock.restore(); +}); + +describe('helpers/jobsQueue', function () { + + describe('register', function () { + it('should register first new job correctly and call properly (job exec: instant, job recall: 1s)', function () { + var name = 'job1'; + var spy = sinon.spy(dummyFunction); + var job = jobsQueue.register(name, spy, recallInterval); + expect(Object.keys(jobsQueue.jobs)).to.be.an('array').and.lengthOf(1); + testExecution(job, name, spy); + }); + + it('should register second new job correctly and call properly (job exec: 10s, job recall: 1s)', function () { + execTimeInterval = 10000; + + var name = 'job2'; + var spy = sinon.spy(dummyFunction); + var job = jobsQueue.register(name, spy, recallInterval); + expect(Object.keys(jobsQueue.jobs)).to.be.an('array').and.lengthOf(2); + testExecution(job, name, spy); + }); + + it('should register third new job correctly call properly (job exec: 2s, job recall: 10s)', function () { + recallInterval = 10000; + execTimeInterval = 2000; + + var name = 'job3'; + var spy = sinon.spy(dummyFunction); + var job = jobsQueue.register(name, spy, recallInterval); + expect(Object.keys(jobsQueue.jobs)).to.be.an('array').and.lengthOf(3); + testExecution(job, name, spy); + }); + + it('should throw an error imediatelly when try to register same job twice', function () { + var name = 'job4'; + var spy = sinon.spy(dummyFunction); + var job = jobsQueue.register(name, spy, recallInterval); + expect(Object.keys(jobsQueue.jobs)).to.be.an('array').and.lengthOf(4); + testExecution(job, name, spy); + + expect(function () { + jobsQueue.register('job4', dummyFunction, recallInterval); + }).to.throw('Synchronous job job4 already registered'); + }); + + it('should use same instance when required in different module (because of modules cache)', function () { + var jobsQueuePeers = peers.__get__('jobsQueue'); + // Instances should be the same + expect(jobsQueuePeers).to.equal(jobsQueue); + + // Register new job in peers module + var name = 'job5'; + var spy = sinon.spy(dummyFunction); + var job = jobsQueuePeers.register(name, spy, recallInterval); + expect(Object.keys(jobsQueuePeers.jobs)).to.be.an('array').and.lengthOf(5); + testExecution(job, name, spy); + // Instances still should be the same + expect(jobsQueuePeers).to.equal(jobsQueue); + }); + + it('should throw an error when try to pass job that is not a function', function () { + expect(function () { + jobsQueue.register('test_job', 'test', recallInterval); + }).to.throw('Syntax error - invalid parameters supplied'); + }); + + it('should throw an error when try to pass name that is not a string', function () { + expect(function () { + jobsQueue.register(123, dummyFunction, recallInterval); + }).to.throw('Syntax error - invalid parameters supplied'); + }); + + it('should throw an error when try to pass time that is not integer', function () { + expect(function () { + jobsQueue.register('test_job', dummyFunction, 0.22); + }).to.throw('Syntax error - invalid parameters supplied'); + }); + }); +}); From 19cf5e52053b47f7682b0faab7864e71372a3b8e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 13 Jul 2017 19:39:19 +0200 Subject: [PATCH 38/88] Move variables and functions to proper scope for jobsQueue test --- test/unit/helpers/jobs-queue.js | 89 ++++++++++++++++----------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/test/unit/helpers/jobs-queue.js b/test/unit/helpers/jobs-queue.js index 6db9b1154bb..dbfb6d62ccf 100644 --- a/test/unit/helpers/jobs-queue.js +++ b/test/unit/helpers/jobs-queue.js @@ -10,52 +10,51 @@ var rewire = require('rewire'); var jobsQueue = require('../../../helpers/jobsQueue.js'); var peers = rewire('../../../modules/peers'); -// Global variables -var clock; -var recallInterval = 1000; -var execTimeInterval = 1; - -function dummyFunction (cb) { - setTimeout(cb, execTimeInterval); -} - -function testExecution (job, name, spy, done) { - expect(jobsQueue.jobs).to.be.an('object'); - // Job returned from 'register' should be equal to one in 'jobsQueue' - expect(job).to.equal(jobsQueue.jobs[name]); - // Shouldn't be called before recallInterval - expect(spy.callCount).to.equal(0); - clock.tick(recallInterval/2); - expect(spy.callCount).to.equal(0); - clock.tick(recallInterval/2); - // Should be called once after recallInterval - expect(spy.callCount).to.equal(1); - // Should be called once before execTimeInterval - clock.tick(execTimeInterval/2); - expect(spy.callCount).to.equal(1); - clock.tick(execTimeInterval/2); - expect(spy.callCount).to.equal(1); - // Should be called twice after another execTimeInterval+recallInterval - clock.tick(recallInterval); - expect(spy.callCount).to.equal(2); - // Job returned from 'register' should no longer be equal to one in 'jobsQueue' - expect(job).to.not.equal(jobsQueue.jobs[name]); - // Should be called thrice after another execTimeInterval+recallInterval - clock.tick(execTimeInterval+recallInterval); - expect(spy.callCount).to.equal(3); - // Job returned from 'register' should no longer be equal to one in 'jobsQueue' - expect(job).to.not.equal(jobsQueue.jobs[name]); -} - -beforeEach(function () { - clock = sinon.useFakeTimers(); -}); - -afterEach(function () { - clock.restore(); -}); - describe('helpers/jobsQueue', function () { + // Test global variables + var clock; + var recallInterval = 1000; + var execTimeInterval = 1; + + function dummyFunction (cb) { + setTimeout(cb, execTimeInterval); + } + + function testExecution (job, name, spy, done) { + expect(jobsQueue.jobs).to.be.an('object'); + // Job returned from 'register' should be equal to one in 'jobsQueue' + expect(job).to.equal(jobsQueue.jobs[name]); + // Shouldn't be called before recallInterval + expect(spy.callCount).to.equal(0); + clock.tick(recallInterval/2); + expect(spy.callCount).to.equal(0); + clock.tick(recallInterval/2); + // Should be called once after recallInterval + expect(spy.callCount).to.equal(1); + // Should be called once before execTimeInterval + clock.tick(execTimeInterval/2); + expect(spy.callCount).to.equal(1); + clock.tick(execTimeInterval/2); + expect(spy.callCount).to.equal(1); + // Should be called twice after another execTimeInterval+recallInterval + clock.tick(recallInterval); + expect(spy.callCount).to.equal(2); + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' + expect(job).to.not.equal(jobsQueue.jobs[name]); + // Should be called thrice after another execTimeInterval+recallInterval + clock.tick(execTimeInterval+recallInterval); + expect(spy.callCount).to.equal(3); + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' + expect(job).to.not.equal(jobsQueue.jobs[name]); + } + + before(function () { + clock = sinon.useFakeTimers(); + }); + + after(function () { + clock.restore(); + }); describe('register', function () { it('should register first new job correctly and call properly (job exec: instant, job recall: 1s)', function () { From c929941cd32528ad27e46961af005d22182f2c4c Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 13 Jul 2017 20:45:28 +0200 Subject: [PATCH 39/88] Reverted jobsQueue to exec functions immediatelly when added to queue --- helpers/jobsQueue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/jobsQueue.js b/helpers/jobsQueue.js index d2926b6cf70..51f1d492bf8 100644 --- a/helpers/jobsQueue.js +++ b/helpers/jobsQueue.js @@ -21,7 +21,7 @@ var jobsQueue = { }); }; - jobsQueue.jobs[name] = setTimeout(nextJob, time); + jobsQueue.jobs[name] = nextJob(); return jobsQueue.jobs[name]; } From 471a579c42f352f3dd05e624755e0ac4ab73cf37 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Thu, 13 Jul 2017 20:46:02 +0200 Subject: [PATCH 40/88] Chenged expectations for jobsQueue tests --- test/unit/helpers/jobs-queue.js | 41 ++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/test/unit/helpers/jobs-queue.js b/test/unit/helpers/jobs-queue.js index dbfb6d62ccf..3f67c44fec6 100644 --- a/test/unit/helpers/jobs-queue.js +++ b/test/unit/helpers/jobs-queue.js @@ -24,26 +24,39 @@ describe('helpers/jobsQueue', function () { expect(jobsQueue.jobs).to.be.an('object'); // Job returned from 'register' should be equal to one in 'jobsQueue' expect(job).to.equal(jobsQueue.jobs[name]); - // Shouldn't be called before recallInterval - expect(spy.callCount).to.equal(0); - clock.tick(recallInterval/2); - expect(spy.callCount).to.equal(0); - clock.tick(recallInterval/2); - // Should be called once after recallInterval - expect(spy.callCount).to.equal(1); - // Should be called once before execTimeInterval - clock.tick(execTimeInterval/2); + + // First execution should happen immediatelly expect(spy.callCount).to.equal(1); - clock.tick(execTimeInterval/2); + + // Every next execution should happen after execTimeInterval+recallInterval and not before + var interval = execTimeInterval+recallInterval; + + clock.tick(interval-10); expect(spy.callCount).to.equal(1); - // Should be called twice after another execTimeInterval+recallInterval - clock.tick(recallInterval); + + clock.tick(11); + expect(spy.callCount).to.equal(2); + + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' + expect(job).to.not.equal(jobsQueue.jobs[name]); + + // Next execution should happen after recallInterval+execTimeInterval + clock.tick(interval-10); expect(spy.callCount).to.equal(2); + + clock.tick(11); + expect(spy.callCount).to.equal(3); + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' expect(job).to.not.equal(jobsQueue.jobs[name]); - // Should be called thrice after another execTimeInterval+recallInterval - clock.tick(execTimeInterval+recallInterval); + + // Next execution should happen after recallInterval+execTimeInterval + clock.tick(interval-10); expect(spy.callCount).to.equal(3); + + clock.tick(11); + expect(spy.callCount).to.equal(4); + // Job returned from 'register' should no longer be equal to one in 'jobsQueue' expect(job).to.not.equal(jobsQueue.jobs[name]); } From 2ba556e6c27824c2811e661ca97a55ba2a2a47a2 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 02:13:12 +0200 Subject: [PATCH 41/88] Create SQL function getDelegatesList --- .../20170508101337_generateDelegatesList.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sql/migrations/20170508101337_generateDelegatesList.sql b/sql/migrations/20170508101337_generateDelegatesList.sql index 7caecdfd49e..41584d902f4 100644 --- a/sql/migrations/20170508101337_generateDelegatesList.sql +++ b/sql/migrations/20170508101337_generateDelegatesList.sql @@ -54,4 +54,20 @@ BEGIN RETURN delegates; END $$; +-- Create function that returns generated delegates list for current round +CREATE OR REPLACE FUNCTION getDelegatesList() RETURNS text[] LANGUAGE PLPGSQL AS $$ +DECLARE + list text[]; +BEGIN + SELECT generateDelegatesList( + -- Get current round + (SELECT CEIL((height+1) / 101::float)::int AS round FROM blocks ORDER BY height DESC LIMIT 1), + -- Get current 101 delegates sorted by rank + ARRAY(SELECT ENCODE(pk, 'hex') AS pk FROM delegates ORDER BY rank ASC LIMIT 101) + ) INTO list; + + -- Return generated delagets list + RETURN list; +END $$; + COMMIT; From b4fa22a61a827646d51874798f9cf852080adff4 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 02:15:22 +0200 Subject: [PATCH 42/88] Trigger delegates_update_on_block also on height 1 --- sql/migrations/20170521001337_roundsRewrite.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 1430d8b7f1b..a1b5af35bf7 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -92,7 +92,7 @@ CREATE FUNCTION delegates_voters_cnt_update() RETURNS TABLE(updated INT) LANGUAG BEGIN RETURN QUERY WITH - last_round AS (SELECT CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 ORDER BY height DESC LIMIT 1), + last_round AS (SELECT CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 OR height = 1 ORDER BY height DESC LIMIT 1), updated AS (UPDATE delegates SET voters_cnt = cnt FROM (SELECT d.pk, @@ -113,7 +113,7 @@ SELECT delegates_voters_cnt_update(); CREATE FUNCTION delegates_voters_balance_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ BEGIN RETURN QUERY - WITH last_round AS (SELECT height, CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 ORDER BY height DESC LIMIT 1), + WITH last_round AS (SELECT height, CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 OR height = 1 ORDER BY height DESC LIMIT 1), current_round_txs AS (SELECT t.id FROM trs t LEFT JOIN blocks b ON b.id = t."blockId" WHERE b.height > (SELECT height FROM last_round)), voters AS (SELECT DISTINCT ON (voter_address) voter_address FROM votes_details), balances AS ( @@ -286,7 +286,7 @@ CREATE CONSTRAINT TRIGGER block_insert AFTER INSERT ON blocks DEFERRABLE INITIALLY DEFERRED FOR EACH ROW - WHEN (NEW.height % 101 = 0) + WHEN (NEW.height % 101 = 0 OR NEW.height = 1) EXECUTE PROCEDURE delegates_update_on_block(); -- Create trigger that will execute 'delegates_update_on_block' after deletion of last block of round From fac26616508061820df8450f46c733b6846e7de9 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 02:34:37 +0200 Subject: [PATCH 43/88] Maintain delegates list in memory, changed finishRound event to roundChanged --- helpers/pg-notify.js | 4 +- modules/blocks/chain.js | 1 + modules/cache.js | 4 +- modules/delegates.js | 152 ++++++++++++++------------------- sql/delegates.js | 2 +- test/unit/helpers/pg-notify.js | 4 +- test/unit/modules/cache.js | 8 +- 7 files changed, 74 insertions(+), 101 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index 8f953d9ed4f..d6dc305221c 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -8,8 +8,8 @@ var db, bus, logger; var connection; // Map channels to bus.message events var channels = { - 'round-closed': 'finishRound', - 'round-reopened': 'finishRound' + 'round-closed': 'roundChanged', + 'round-reopened': 'roundChanged' }; function onNotification (data) { diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index 954b0a2c914..6f1b8b17c97 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -58,6 +58,7 @@ Chain.prototype.saveGenesisBlock = function (cb) { if (!blockId) { // If there is no block with genesis ID - save to database // WARNING: DB_WRITE + // FIXME: That will fail if we already have genesis block in database, but with different ID self.saveBlock(library.genesisblock.block, function (err) { return setImmediate(cb, err); }); diff --git a/modules/cache.js b/modules/cache.js index bbc66ba77e3..d512daccb10 100644 --- a/modules/cache.js +++ b/modules/cache.js @@ -165,11 +165,11 @@ Cache.prototype.onNewBlock = function (block, broadcast, cb) { }; /** - * This function will be triggered when a round finishes, it will clear all cache entires. + * This function will be triggered when a round changed, it will clear all cache entires. * @param {Round} round * @param {Function} cb */ -Cache.prototype.onFinishRound = function (round, cb) { +Cache.prototype.onRoundChanged = function (round, cb) { cb = cb || function () {}; if(!self.isReady()) { return cb(errorCacheDisabled); } diff --git a/modules/delegates.js b/modules/delegates.js index c257efc6f80..a8b808c1c45 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -24,6 +24,7 @@ __private.loaded = false; __private.blockReward = new BlockReward(); __private.keypairs = {}; __private.tmpKeypairs = {}; +__private.delegatesList = []; /** * Initializes library with scope content and generates a Delegate instance. @@ -67,24 +68,7 @@ function Delegates (cb, scope) { ); setImmediate(cb, null, self); -} - -// Private methods -/** - * Gets delegate public keys sorted by vote descending. - * @private - * @param {function} cb - Callback function. - * @returns {setImmediateCallback} - */ -__private.getKeysSortByVote = function (cb) { - library.db.query(sql.delegateList).then(function (rows) { - return setImmediate(cb, null, rows.map(function (el) { - return el.pk; - })); - }).catch(function (err) { - return setImmediate(cb, err); - }); -}; +} /** * Gets slot time and keypair. @@ -95,25 +79,19 @@ __private.getKeysSortByVote = function (cb) { * @returns {setImmediateCallback} error | cb | object {time, keypair}. */ __private.getBlockSlotData = function (slot, height, cb) { - self.generateDelegateList(height, function (err, activeDelegates) { - if (err) { - return setImmediate(cb, err); - } - - var currentSlot = slot; - var lastSlot = slots.getLastSlot(currentSlot); + var currentSlot = slot; + var lastSlot = slots.getLastSlot(currentSlot); - for (; currentSlot < lastSlot; currentSlot += 1) { - var delegate_pos = currentSlot % slots.delegates; - var delegate_id = activeDelegates[delegate_pos]; + for (; currentSlot < lastSlot; currentSlot += 1) { + var delegate_pos = currentSlot % slots.delegates; + var delegate_id = __private.delegatesList[delegate_pos]; - if (delegate_id && __private.keypairs[delegate_id]) { - return setImmediate(cb, null, {time: slots.getSlotTime(currentSlot), keypair: __private.keypairs[delegate_id]}); - } + if (delegate_id && __private.keypairs[delegate_id]) { + return setImmediate(cb, null, {time: slots.getSlotTime(currentSlot), keypair: __private.keypairs[delegate_id]}); } + } - return setImmediate(cb, null, null); - }); + return setImmediate(cb, null, null); }; /** @@ -332,33 +310,18 @@ __private.loadDelegates = function (cb) { }; // Public methods + /** - * Gets delegate list by vote and changes order. - * @param {number} height + * Get delegates list for current round. * @param {function} cb - Callback function. - * @returns {setImmediateCallback} err | truncated delegate list. - * @todo explain seed. + * @returns {setImmediateCallback} err */ -Delegates.prototype.generateDelegateList = function (height, cb) { - __private.getKeysSortByVote(function (err, truncDelegateList) { - if (err) { - return setImmediate(cb, err); - } - - var seedSource = modules.rounds.calc(height).toString(); - var currentSeed = crypto.createHash('sha256').update(seedSource, 'utf8').digest(); - - for (var i = 0, delCount = truncDelegateList.length; i < delCount; i++) { - for (var x = 0; x < 4 && i < delCount; i++, x++) { - var newIndex = currentSeed[x] % delCount; - var b = truncDelegateList[newIndex]; - truncDelegateList[newIndex] = truncDelegateList[i]; - truncDelegateList[i] = b; - } - currentSeed = crypto.createHash('sha256').update(currentSeed).digest(); - } - - return setImmediate(cb, null, truncDelegateList); +Delegates.prototype.generateDelegateList = function (cb) { + library.db.query(sql.delegateList).then(function (result) { + __private.delegatesList = result[0].list; + return setImmediate(cb); + }).catch(function (err) { + return setImmediate(cb, err); }); }; @@ -481,23 +444,17 @@ Delegates.prototype.fork = function (block, cause) { * @returns {setImmediateCallback} error message | cb */ Delegates.prototype.validateBlockSlot = function (block, cb) { - self.generateDelegateList(block.height, function (err, activeDelegates) { - if (err) { - return setImmediate(cb, err); - } - - var currentSlot = slots.getSlotNumber(block.timestamp); - var delegate_id = activeDelegates[currentSlot % slots.delegates]; - // var nextDelegate_id = activeDelegates[(currentSlot + 1) % slots.delegates]; - // var previousDelegate_id = activeDelegates[(currentSlot - 1) % slots.delegates]; + var currentSlot = slots.getSlotNumber(block.timestamp); + var delegate_id = __private.delegatesList[currentSlot % slots.delegates]; + // var nextDelegate_id = __private.delegatesList[(currentSlot + 1) % slots.delegates]; + // var previousDelegate_id = __private.delegatesList[(currentSlot - 1) % slots.delegates]; - if (delegate_id && block.generatorPublicKey === delegate_id) { - return setImmediate(cb); - } else { - library.logger.error('Expected generator: ' + delegate_id + ' Received generator: ' + block.generatorPublicKey); - return setImmediate(cb, 'Failed to verify slot: ' + currentSlot); - } - }); + if (delegate_id && block.generatorPublicKey === delegate_id) { + return setImmediate(cb); + } else { + library.logger.error('Expected generator: ' + delegate_id + ' Received generator: ' + block.generatorPublicKey); + return setImmediate(cb, 'Failed to verify slot: ' + currentSlot); + } }; // Events @@ -522,6 +479,25 @@ Delegates.prototype.onBind = function (scope) { ); }; +/** + * Handle node shutdown request + * + * @public + * @method onRoundChanged + * @listens module:pg-notify~event:roundChanged + * @implements module:delegates#generateDelegateList + * @param {number} round Current round + */ +Delegates.prototype.onRoundChanged = function (round) { + self.generateDelegateList(function (err) { + if (err) { + library.logger.error('Cannot get delegates list for round', err); + } + + library.network.io.sockets.emit('rounds/change', {number: round}); + }); +}; + /** * Loads delegates. * @implements module:transactions#Transactions~fillPool @@ -529,14 +505,16 @@ Delegates.prototype.onBind = function (scope) { Delegates.prototype.onBlockchainReady = function () { __private.loaded = true; - __private.loadDelegates(function (err) { - + async.waterfall([ + __private.loadDelegates, + self.generateDelegateList + ], function (err) { function nextForge (cb) { if (err) { library.logger.error('Failed to load delegates', err); } - async.series([ + async.waterfall([ __private.forge, modules.transactions.fillPool ], function () { @@ -718,23 +696,17 @@ Delegates.prototype.shared = { var currentBlock = modules.blocks.lastBlock.get(); var limit = req.body.limit || 10; - modules.delegates.generateDelegateList(currentBlock.height, function (err, activeDelegates) { - if (err) { - return setImmediate(cb, err); - } - - var currentBlockSlot = slots.getSlotNumber(currentBlock.timestamp); - var currentSlot = slots.getSlotNumber(); - var nextForgers = []; + var currentBlockSlot = slots.getSlotNumber(currentBlock.timestamp); + var currentSlot = slots.getSlotNumber(); + var nextForgers = []; - for (var i = 1; i <= slots.delegates && i <= limit; i++) { - if (activeDelegates[(currentSlot + i) % slots.delegates]) { - nextForgers.push (activeDelegates[(currentSlot + i) % slots.delegates]); - } + for (var i = 1; i <= slots.delegates && i <= limit; i++) { + if (__private.delegatesList[(currentSlot + i) % slots.delegates]) { + nextForgers.push (__private.delegatesList[(currentSlot + i) % slots.delegates]); } + } - return setImmediate(cb, null, {currentBlock: currentBlock.height, currentBlockSlot: currentBlockSlot, currentSlot: currentSlot, delegates: nextForgers}); - }); + return setImmediate(cb, null, {currentBlock: currentBlock.height, currentBlockSlot: currentBlockSlot, currentSlot: currentSlot, delegates: nextForgers}); }, search: function (req, cb) { diff --git a/sql/delegates.js b/sql/delegates.js index 423feae0050..ce3f0063e4d 100644 --- a/sql/delegates.js +++ b/sql/delegates.js @@ -18,7 +18,7 @@ var DelegatesSql = { count: 'SELECT COUNT(*)::int FROM delegates', - delegateList: 'SELECT ENCODE(pk, \'hex\') AS pk FROM delegates ORDER BY rank ASC LIMIT 101', + delegateList: 'SELECT getDelegatesList() AS list;', search: function (params) { var sql = [ diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 41395c832c4..47224f6574f 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -278,7 +278,7 @@ describe('helpers/pg-notify', function () { db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: '123'}]); expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed', 123]); - expect(bus.message.args[0]).to.deep.equal(['finishRound', 124]); + expect(bus.message.args[0]).to.deep.equal(['roundChanged', 124]); done(); }, 20)).catch(function (err) { done(err); @@ -293,7 +293,7 @@ describe('helpers/pg-notify', function () { db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: '123'}]); expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened', 123]); - expect(bus.message.args[0]).to.deep.equal(['finishRound', 123]); + expect(bus.message.args[0]).to.deep.equal(['roundChanged', 123]); done(); }, 20)).catch(function (err) { done(err); diff --git a/test/unit/modules/cache.js b/test/unit/modules/cache.js index 9d970e447b9..b22ae49c1bd 100644 --- a/test/unit/modules/cache.js +++ b/test/unit/modules/cache.js @@ -253,7 +253,7 @@ describe('cache', function () { }); }); - describe('onFinishRound', function (done) { + describe('onRoundChanged', function (done) { it('should remove all keys matching pattern /api/delegates', function (done) { var key = '/api/delegates?123'; @@ -262,7 +262,7 @@ describe('cache', function () { cache.setJsonForKey(key, value, function (err, status) { expect(err).to.not.exist; expect(status).to.equal('OK'); - cache.onFinishRound(null, function (err) { + cache.onRoundChanged(null, function (err) { expect(err).to.not.exist; cache.getJsonForKey(key, function (err, res) { expect(err).to.not.exist; @@ -281,7 +281,7 @@ describe('cache', function () { expect(err).to.not.exist; expect(status).to.equal('OK'); - cache.onFinishRound(null, function (err) { + cache.onRoundChanged(null, function (err) { expect(err).to.not.exist; cache.getJsonForKey(key, function (err, res) { expect(err).to.not.exist; @@ -301,7 +301,7 @@ describe('cache', function () { expect(status).to.equal('OK'); cache.onSyncStarted(); - cache.onFinishRound(null, function (err) { + cache.onRoundChanged(null, function (err) { expect(err).to.equal('Cache Unavailable'); cache.onSyncFinished(); cache.getJsonForKey(key, function (err, res) { From 95ce3b185a4d7d08f4d771822604f545a5e027dd Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 03:40:56 +0200 Subject: [PATCH 44/88] Remove old rounds logic --- app.js | 1 - docs/typedef/components.js | 1 - docs/typedef/modules/rounds.js | 5 - helpers/RoundChanges.js | 54 ----- helpers/slots.js | 9 + logic/inTransfer.js | 15 +- logic/multisignature.js | 11 +- logic/outTransfer.js | 13 +- logic/round.js | 237 ------------------ logic/transaction.js | 19 +- logic/transfer.js | 15 +- logic/vote.js | 17 +- modules/accounts.js | 3 +- modules/blocks/chain.js | 2 - modules/blocks/process.js | 4 +- modules/dapps.js | 2 - modules/delegates.js | 3 +- modules/loader.js | 15 +- modules/multisignatures.js | 1 - modules/rounds.js | 385 ------------------------------ modules/transactions.js | 3 +- sql/rounds.js | 25 -- test/common/initModule.js | 1 - test/index.js | 1 + test/node.js | 3 +- test/unit/helpers/RoundChanges.js | 98 -------- test/unit/helpers/slots.js | 26 ++ test/unit/logic/transaction.js | 10 +- test/unit/logic/transfer.js | 16 +- test/unit/logic/vote.js | 16 +- test/unit/modules/rounds.js | 39 --- test/unit/sql/blockRewards.js | 26 +- test/unit/sql/delegatesList.js | 53 +++- 33 files changed, 157 insertions(+), 972 deletions(-) delete mode 100644 docs/typedef/modules/rounds.js delete mode 100644 helpers/RoundChanges.js delete mode 100644 logic/round.js delete mode 100644 modules/rounds.js delete mode 100644 sql/rounds.js delete mode 100644 test/unit/helpers/RoundChanges.js create mode 100644 test/unit/helpers/slots.js delete mode 100644 test/unit/modules/rounds.js diff --git a/app.js b/app.js index 5e9381cfe4b..ca2926a50d2 100644 --- a/app.js +++ b/app.js @@ -132,7 +132,6 @@ var config = { system: './modules/system.js', peers: './modules/peers.js', delegates: './modules/delegates.js', - rounds: './modules/rounds.js', multisignatures: './modules/multisignatures.js', dapps: './modules/dapps.js', crypto: './modules/crypto.js', diff --git a/docs/typedef/components.js b/docs/typedef/components.js index 71a8985bc53..9f7988ea4be 100644 --- a/docs/typedef/components.js +++ b/docs/typedef/components.js @@ -9,7 +9,6 @@ * @property {Loader} loader * @property {Multisignatures} multisignatures * @property {Peers} peers - * @property {Rounds} rounds * @property {Server} server * @property {Signatures} signatures * @property {Sql} sql diff --git a/docs/typedef/modules/rounds.js b/docs/typedef/modules/rounds.js deleted file mode 100644 index d997442d680..00000000000 --- a/docs/typedef/modules/rounds.js +++ /dev/null @@ -1,5 +0,0 @@ -/** - * Core Module `rounds` - * - * @module rounds - */ diff --git a/helpers/RoundChanges.js b/helpers/RoundChanges.js deleted file mode 100644 index 2cf26171376..00000000000 --- a/helpers/RoundChanges.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict'; - -var bignum = require('./bignum'); -var slots = require('./slots'); -var exceptions = require('./exceptions'); - -/** - * Sets round fees and rewards - * @requires helpers/bignum - * @requires helpers/slots - * @memberof module:helpers - * @constructor - * @param {Object} scope - */ -// Constructor -function RoundChanges (scope) { - this.roundFees = Math.floor(scope.roundFees) || 0; - this.roundRewards = (scope.roundRewards || []); - - // Apply exception for round if required - if (exceptions.rounds[scope.round]) { - // Apply rewards factor - this.roundRewards.forEach(function (reward, index) { - this.roundRewards[index] = new bignum(reward.toPrecision(15)).times(exceptions.rounds[scope.round].rewards_factor).floor(); - }.bind(this)); - - // Apply fees factor and bonus - this.roundFees = new bignum(this.roundFees.toPrecision(15)).times(exceptions.rounds[scope.round].fees_factor).plus(exceptions.rounds[scope.round].fees_bonus).floor(); - } -} - -// Public methods -/** - * Calculates rewards at round position. - * Fees and feesRemaining based on slots - * @implements bignum - * @implements slots - * @param {number} index - * @return {Object} Contains fees, feesRemaining, rewards, balance - */ -RoundChanges.prototype.at = function (index) { - var fees = new bignum(this.roundFees.toPrecision(15)).dividedBy(slots.delegates).floor(); - var feesRemaining = new bignum(this.roundFees.toPrecision(15)).minus(fees.times(slots.delegates)); - var rewards = new bignum(this.roundRewards[index].toPrecision(15)).floor() || 0; - - return { - fees: Number(fees.toFixed()), - feesRemaining: Number(feesRemaining.toFixed()), - rewards: Number(rewards.toFixed()), - balance: Number(fees.add(rewards).toFixed()) - }; -}; - -module.exports = RoundChanges; diff --git a/helpers/slots.js b/helpers/slots.js index 46886d0d7e5..8d93b037b54 100644 --- a/helpers/slots.js +++ b/helpers/slots.js @@ -112,5 +112,14 @@ module.exports = { roundTime: function (date) { return Math.floor(date.getTime() / 1000) * 1000; + }, + + /** + * Calculate round number for supplied height + * @param {number} height Height from which calculate round + * @return {number} Round + */ + calcRound: function (height) { + return Math.ceil(height / this.delegates); } }; diff --git a/logic/inTransfer.js b/logic/inTransfer.js index b05e928b92a..84fea69625a 100644 --- a/logic/inTransfer.js +++ b/logic/inTransfer.js @@ -2,6 +2,7 @@ var constants = require('../helpers/constants.js'); var sql = require('../sql/dapps.js'); +var slots = require('../helpers/slots.js'); // Private fields var modules, library, shared; @@ -26,13 +27,11 @@ function InTransfer (db, schema) { /** * Binds input parameters to private variables modules and shared. * @param {Accounts} accounts - * @param {Rounds} rounds * @param {Object} sharedApi */ -InTransfer.prototype.bind = function (accounts, rounds, sharedApi) { +InTransfer.prototype.bind = function (accounts, sharedApi) { modules = { - accounts: accounts, - rounds: rounds, + accounts: accounts }; shared = sharedApi; }; @@ -137,7 +136,7 @@ InTransfer.prototype.getBytes = function (trs) { * address. * @implements {shared.getGenesis} * @implements {modules.accounts.mergeAccountAndGet} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -154,7 +153,7 @@ InTransfer.prototype.apply = function (trs, block, sender, cb) { balance: trs.amount, u_balance: trs.amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -167,7 +166,7 @@ InTransfer.prototype.apply = function (trs, block, sender, cb) { * trs amount and balance both negatives. * @implements {shared.getGenesis} * @implements {modules.accounts.mergeAccountAndGet} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -184,7 +183,7 @@ InTransfer.prototype.undo = function (trs, block, sender, cb) { balance: -trs.amount, u_balance: -trs.amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); diff --git a/logic/multisignature.js b/logic/multisignature.js index 84acb97f37b..dadb00c2a11 100644 --- a/logic/multisignature.js +++ b/logic/multisignature.js @@ -5,6 +5,7 @@ var ByteBuffer = require('bytebuffer'); var constants = require('../helpers/constants.js'); var Diff = require('../helpers/diff.js'); var exceptions = require('../helpers/exceptions.js'); +var slots = require('../helpers/slots.js'); // Private fields var modules, library, __private = {}; @@ -36,13 +37,11 @@ function Multisignature (schema, network, transaction, logger) { // Public methods /** * Binds input parameters to private variable modules - * @param {Rounds} rounds * @param {Accounts} accounts */ -Multisignature.prototype.bind = function (rounds, accounts) { +Multisignature.prototype.bind = function (accounts) { modules = { - rounds: rounds, - accounts: accounts, + accounts: accounts }; }; @@ -238,7 +237,7 @@ Multisignature.prototype.apply = function (trs, block, sender, cb) { multimin: trs.asset.multisignature.min, multilifetime: trs.asset.multisignature.lifetime, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { if (err) { return setImmediate(cb, err); @@ -279,7 +278,7 @@ Multisignature.prototype.undo = function (trs, block, sender, cb) { multimin: -trs.asset.multisignature.min, multilifetime: -trs.asset.multisignature.lifetime, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); diff --git a/logic/outTransfer.js b/logic/outTransfer.js index e9662793e0e..d1fc919d74a 100644 --- a/logic/outTransfer.js +++ b/logic/outTransfer.js @@ -2,6 +2,7 @@ var constants = require('../helpers/constants.js'); var sql = require('../sql/dapps.js'); +var slots = require('../helpers/slots.js'); // Private fields var modules, library, __private = {}; @@ -30,13 +31,11 @@ function OutTransfer (db, schema, logger) { /** * Binds input modules to private variable module. * @param {Accounts} accounts - * @param {Rounds} rounds * @param {Dapps} dapps */ -OutTransfer.prototype.bind = function (accounts, rounds, dapps) { +OutTransfer.prototype.bind = function (accounts, dapps) { modules = { accounts: accounts, - rounds: rounds, dapps: dapps, }; }; @@ -167,7 +166,7 @@ OutTransfer.prototype.getBytes = function (trs) { * mergeAccountAndGet with unconfirmed trs amount. * @implements {modules.accounts.setAccountAndGet} * @implements {modules.accounts.mergeAccountAndGet} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -187,7 +186,7 @@ OutTransfer.prototype.apply = function (trs, block, sender, cb) { balance: trs.amount, u_balance: trs.amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -200,7 +199,7 @@ OutTransfer.prototype.apply = function (trs, block, sender, cb) { * mergeAccountAndGet with unconfirmed trs amount and balance both negatives. * @implements {modules.accounts.setAccountAndGet} * @implements {modules.accounts.mergeAccountAndGet} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -219,7 +218,7 @@ OutTransfer.prototype.undo = function (trs, block, sender, cb) { balance: -trs.amount, u_balance: -trs.amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); diff --git a/logic/round.js b/logic/round.js deleted file mode 100644 index 4d86a7b1f8f..00000000000 --- a/logic/round.js +++ /dev/null @@ -1,237 +0,0 @@ -'use strict'; - -var pgp = require('pg-promise'); -var RoundChanges = require('../helpers/RoundChanges.js'); -var sql = require('../sql/rounds.js'); - -/** - * Validates required scope properties. - * @memberof module:rounds - * @class - * @classdesc Main Round logic. - * @param {Object} scope - * @param {Task} t - * @constructor - */ -// Constructor -function Round (scope, t) { - this.scope = { - backwards: scope.backwards, - round: scope.round, - roundOutsiders: scope.roundOutsiders, - roundDelegates: scope.roundDelegates, - roundFees: scope.roundFees, - roundRewards: scope.roundRewards, - library: { - logger: scope.library.logger, - }, - modules: { - accounts: scope.modules.accounts, - }, - block: { - generatorPublicKey: scope.block.generatorPublicKey, - id: scope.block.id, - height: scope.block.height, - }, - }; - this.t = t; - - // List of required scope properties - var requiredProperties = ['library', 'modules', 'block', 'round', 'backwards']; - - // Require extra scope properties when finishing round - if (scope.finishRound) { - requiredProperties = requiredProperties.concat(['roundFees', 'roundRewards', 'roundDelegates', 'roundOutsiders']); - } - - // Iterate over requiredProperties, checking for undefined scope properties - requiredProperties.forEach(function (property) { - if (scope[property] === undefined) { - throw 'Missing required scope property: ' + property; - } - }); -} - -// Public methods -/** - * Returns result from call to mergeAccountAndGet - * @implements {modules.accounts.mergeAccountAndGet} - * @return {function} Promise - */ -Round.prototype.mergeBlockGenerator = function () { - return this.t.none( - this.scope.modules.accounts.mergeAccountAndGet({ - publicKey: this.scope.block.generatorPublicKey, - producedblocks: (this.scope.backwards ? -1 : 1), - blockId: this.scope.block.id, - round: this.scope.round - }) - ); -}; - -/** - * If outsiders content, calls sql updateMissedBlocks. - * @return {} - */ -Round.prototype.updateMissedBlocks = function () { - if (this.scope.roundOutsiders.length === 0) { - return this.t; - } - - return this.t.none(sql.updateMissedBlocks(this.scope.backwards), [this.scope.roundOutsiders]); -}; - -/** - * Calls sql getVotes from `mem_round` table. - * @return {} - * @todo round must be a param option. - */ -Round.prototype.getVotes = function () { - return this.t.query(sql.getVotes, { round: this.scope.round }); -}; - -/** - * Calls getVotes with round - * @implements {getVotes} - * @implements {modules.accounts.generateAddressByPublicKey} - * @return {function} Promise - */ -Round.prototype.updateVotes = function () { - var self = this; - - return self.getVotes(self.scope.round).then(function (votes) { - var queries = votes.map(function (vote) { - return pgp.as.format(sql.updateVotes, { - address: self.scope.modules.accounts.generateAddressByPublicKey(vote.delegate), - amount: Math.floor(vote.amount) - }); - }).join(''); - - if (queries.length > 0) { - return self.t.none(queries); - } else { - return self.t; - } - }); -}; - -/** - * For backwards option calls sql updateBlockId with newID: 0. - * @return {function} Promise - */ -Round.prototype.markBlockId = function () { - if (this.scope.backwards) { - return this.t.none(sql.updateBlockId, { oldId: this.scope.block.id, newId: '0' }); - } else { - return this.t; - } -}; - -/** - * Calls sql flush: deletes round from `mem_round` table. - * @return {function} Promise - */ -Round.prototype.flushRound = function () { - return this.t.none(sql.flush, { round: this.scope.round }); -}; - -/** - * Calls sql truncateBlocks: deletes blocks greather than height from - * `blocks` table. - * @return {function} Promise - */ -Round.prototype.truncateBlocks = function () { - return this.t.none(sql.truncateBlocks, { height: this.scope.block.height }); -}; - -/** - * For each delegate calls mergeAccountAndGet and creates an address array - * @implements {helpers.RoundChanges} - * @implements {modules.accounts.mergeAccountAndGet} - * @return {function} Promise with address array - */ -Round.prototype.applyRound = function () { - var roundChanges = new RoundChanges(this.scope); - var queries = []; - - // Reverse delegates if going backwards - var delegates = (this.scope.backwards) ? this.scope.roundDelegates.reverse() : this.scope.roundDelegates; - - // Apply round changes to each delegate - for (var i = 0; i < this.scope.roundDelegates.length; i++) { - var delegate = this.scope.roundDelegates[i]; - var changes = roundChanges.at(i); - - this.scope.library.logger.trace('Delegate changes', { delegate: delegate, changes: changes }); - - queries.push(this.scope.modules.accounts.mergeAccountAndGet({ - publicKey: delegate, - balance: (this.scope.backwards ? -changes.balance : changes.balance), - u_balance: (this.scope.backwards ? -changes.balance : changes.balance), - blockId: this.scope.block.id, - round: this.scope.round, - fees: (this.scope.backwards ? -changes.fees : changes.fees), - rewards: (this.scope.backwards ? -changes.rewards : changes.rewards) - })); - } - - // Decide which delegate receives fees remainder - var remainderIndex = (this.scope.backwards) ? 0 : delegates.length - 1; - var remainderDelegate = delegates[remainderIndex]; - - // Get round changes for chosen delegate - var changes = roundChanges.at(remainderIndex); - - // Apply fees remaining to chosen delegate - if (changes.feesRemaining > 0) { - var feesRemaining = (this.scope.backwards ? -changes.feesRemaining : changes.feesRemaining); - - this.scope.library.logger.trace('Fees remaining', { index: remainderIndex, delegate: remainderDelegate, fees: feesRemaining }); - - queries.push(this.scope.modules.accounts.mergeAccountAndGet({ - publicKey: remainderDelegate, - balance: feesRemaining, - u_balance: feesRemaining, - blockId: this.scope.block.id, - round: this.scope.round, - fees: feesRemaining - })); - } - - this.scope.library.logger.trace('Applying round', queries); - - if (queries.length > 0) { - return this.t.none(queries.join('')); - } else { - return this.t; - } -}; - -/** - * Calls: - * - updateVotes - * - updateMissedBlocks - * - flushRound - * - applyRound - * - updateVotes - * - flushRound - * @implements {updateVotes} - * @implements {updateMissedBlocks} - * @implements {flushRound} - * @implements {applyRound} - * @return {function} call result - */ -Round.prototype.land = function () { - return this.updateVotes() - .then(this.updateMissedBlocks.bind(this)) - .then(this.flushRound.bind(this)) - .then(this.applyRound.bind(this)) - .then(this.updateVotes.bind(this)) - .then(this.flushRound.bind(this)) - .then(function () { - return this.t; - }.bind(this)); -}; - -// Export -module.exports = Round; diff --git a/logic/transaction.js b/logic/transaction.js index 5e0e32e92d5..0747cf82fc3 100644 --- a/logic/transaction.js +++ b/logic/transaction.js @@ -689,7 +689,7 @@ Transaction.prototype.verifyBytes = function (bytes, publicKey, signature) { * @see privateTypes * @implements {checkBalance} * @implements {account.merge} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -711,11 +711,11 @@ Transaction.prototype.apply = function (trs, block, sender, cb) { amount = amount.toNumber(); - this.scope.logger.trace('Logic/Transaction->apply', {sender: sender.address, balance: -amount, blockId: block.id, round: modules.rounds.calc(block.height)}); + this.scope.logger.trace('Logic/Transaction->apply', {sender: sender.address, balance: -amount, blockId: block.id, round: slots.calcRound(block.height)}); this.scope.account.merge(sender.address, { balance: -amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err, sender) { if (err) { return setImmediate(cb, err); @@ -729,7 +729,7 @@ Transaction.prototype.apply = function (trs, block, sender, cb) { this.scope.account.merge(sender.address, { balance: amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -745,7 +745,7 @@ Transaction.prototype.apply = function (trs, block, sender, cb) { * @see privateTypes * @implements {bignum} * @implements {account.merge} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -756,11 +756,11 @@ Transaction.prototype.undo = function (trs, block, sender, cb) { var amount = new bignum(trs.amount.toString()); amount = amount.plus(trs.fee.toString()).toNumber(); - this.scope.logger.trace('Logic/Transaction->undo', {sender: sender.address, balance: amount, blockId: block.id, round: modules.rounds.calc(block.height)}); + this.scope.logger.trace('Logic/Transaction->undo', {sender: sender.address, balance: amount, blockId: block.id, round: slots.calcRound(block.height)}); this.scope.account.merge(sender.address, { balance: amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err, sender) { if (err) { return setImmediate(cb, err); @@ -771,7 +771,7 @@ Transaction.prototype.undo = function (trs, block, sender, cb) { this.scope.account.merge(sender.address, { balance: -amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -1137,9 +1137,6 @@ Transaction.prototype.dbRead = function (raw) { */ Transaction.prototype.bindModules = function (__modules) { this.scope.logger.trace('Logic/Transaction->bindModules'); - modules = { - rounds: __modules.rounds - }; }; // Export diff --git a/logic/transfer.js b/logic/transfer.js index 91c7cb13813..4b3ca254cfa 100644 --- a/logic/transfer.js +++ b/logic/transfer.js @@ -1,6 +1,7 @@ 'use strict'; var constants = require('../helpers/constants.js'); +var slots = require('../helpers/slots.js'); // Private fields var modules; @@ -18,12 +19,10 @@ function Transfer () {} /** * Binds input parameters to private variable modules. * @param {Accounts} accounts - * @param {Rounds} rounds */ -Transfer.prototype.bind = function (accounts, rounds) { +Transfer.prototype.bind = function (accounts) { modules = { - accounts: accounts, - rounds: rounds, + accounts: accounts }; }; @@ -91,7 +90,7 @@ Transfer.prototype.getBytes = function (trs) { * mergeAccountAndGet with unconfirmed trs amount. * @implements {modules.accounts.setAccountAndGet} * @implements {modules.accounts.mergeAccountAndGet} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -109,7 +108,7 @@ Transfer.prototype.apply = function (trs, block, sender, cb) { balance: trs.amount, u_balance: trs.amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -121,7 +120,7 @@ Transfer.prototype.apply = function (trs, block, sender, cb) { * mergeAccountAndGet with unconfirmed trs amount and balance negative. * @implements {modules.accounts.setAccountAndGet} * @implements {modules.accounts.mergeAccountAndGet} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -139,7 +138,7 @@ Transfer.prototype.undo = function (trs, block, sender, cb) { balance: -trs.amount, u_balance: -trs.amount, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); diff --git a/logic/vote.js b/logic/vote.js index 2edf208381f..a379877d918 100644 --- a/logic/vote.js +++ b/logic/vote.js @@ -5,6 +5,7 @@ var constants = require('../helpers/constants.js'); var exceptions = require('../helpers/exceptions.js'); var Diff = require('../helpers/diff.js'); var _ = require('lodash'); +var slots = require('../helpers/slots.js'); // Private fields var modules, library, self; @@ -33,12 +34,10 @@ function Vote (logger, schema) { /** * Binds module content to private object modules. * @param {Delegates} delegates - * @param {Rounds} rounds */ -Vote.prototype.bind = function (delegates, rounds) { +Vote.prototype.bind = function (delegates) { modules = { - delegates: delegates, - rounds: rounds, + delegates: delegates }; }; @@ -213,7 +212,7 @@ Vote.prototype.getBytes = function (trs) { * merges account to sender address with votes as delegates. * @implements {checkConfirmedDelegates} * @implements {scope.account.merge} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -231,7 +230,7 @@ Vote.prototype.apply = function (trs, block, sender, cb) { parent.scope.account.merge(sender.address, { delegates: trs.asset.votes, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -244,7 +243,7 @@ Vote.prototype.apply = function (trs, block, sender, cb) { * sender address with inverted votes as delegates. * @implements {Diff} * @implements {scope.account.merge} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {block} block * @param {account} sender @@ -259,7 +258,7 @@ Vote.prototype.undo = function (trs, block, sender, cb) { this.scope.account.merge(sender.address, { delegates: votesInvert, blockId: block.id, - round: modules.rounds.calc(block.height) + round: slots.calcRound(block.height) }, function (err) { return setImmediate(cb, err); }); @@ -297,7 +296,7 @@ Vote.prototype.applyUnconfirmed = function (trs, sender, cb) { * sender address with inverted votes as unconfirmed delegates. * @implements {Diff} * @implements {scope.account.merge} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @param {transaction} trs * @param {account} sender * @param {function} cb - Callback function diff --git a/modules/accounts.js b/modules/accounts.js index 423d102a4d9..829fbacfaf8 100644 --- a/modules/accounts.js +++ b/modules/accounts.js @@ -230,8 +230,7 @@ Accounts.prototype.onBind = function (scope) { }; __private.assetTypes[transactionTypes.VOTE].bind( - scope.delegates, - scope.rounds + scope.delegates ); }; /** diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index 6f1b8b17c97..d6ab28e96d1 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -617,7 +617,6 @@ Chain.prototype.recoverChain = function (cb) { * Handle modules initialization: * - accounts * - blocks - * - rounds * - transactions * @param {modules} scope Exposed modules */ @@ -626,7 +625,6 @@ Chain.prototype.onBind = function (scope) { modules = { accounts: scope.accounts, blocks: scope.blocks, - rounds: scope.rounds, transactions: scope.transactions, }; diff --git a/modules/blocks/process.js b/modules/blocks/process.js index 6ca1f4e6848..f2f3dbc413d 100644 --- a/modules/blocks/process.js +++ b/modules/blocks/process.js @@ -440,7 +440,7 @@ __private.receiveBlock = function (block, cb) { library.logger.info([ 'Received new block id:', block.id, 'height:', block.height, - 'round:', modules.rounds.calc(block.height), + 'round:', slots.calcRound(block.height), 'slot:', slots.getSlotNumber(block.timestamp), 'reward:', block.reward ].join(' ')); @@ -457,7 +457,6 @@ __private.receiveBlock = function (block, cb) { * - blocks * - delegates * - loader - * - rounds * - transactions * - transport * @param {modules} scope Exposed modules @@ -469,7 +468,6 @@ Process.prototype.onBind = function (scope) { blocks: scope.blocks, delegates: scope.delegates, loader: scope.loader, - rounds: scope.rounds, transactions: scope.transactions, transport: scope.transport, }; diff --git a/modules/dapps.js b/modules/dapps.js index 97e07e497c0..4f2ef8332ca 100644 --- a/modules/dapps.js +++ b/modules/dapps.js @@ -198,13 +198,11 @@ DApps.prototype.onBind = function (scope) { __private.assetTypes[transactionTypes.IN_TRANSFER].bind( scope.accounts, - scope.rounds, shared ); __private.assetTypes[transactionTypes.OUT_TRANSFER].bind( scope.accounts, - scope.rounds, scope.dapps ); }; diff --git a/modules/delegates.js b/modules/delegates.js index a8b808c1c45..8186d63c0b2 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -163,7 +163,7 @@ __private.forge = function (cb) { library.logger.info([ 'Forged new block id:', forgedBlock.id, 'height:', forgedBlock.height, - 'round:', modules.rounds.calc(forgedBlock.height), + 'round:', slots.calcRound(forgedBlock.height), 'slot:', slots.getSlotNumber(currentBlockData.time), 'reward:' + forgedBlock.reward ].join(' ')); @@ -466,7 +466,6 @@ Delegates.prototype.validateBlockSlot = function (block, cb) { Delegates.prototype.onBind = function (scope) { modules = { loader: scope.loader, - rounds: scope.rounds, accounts: scope.accounts, blocks: scope.blocks, transport: scope.transport, diff --git a/modules/loader.js b/modules/loader.js index 5c3a9f8a321..0548e7b4903 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -6,6 +6,7 @@ var exceptions = require('../helpers/exceptions'); var jobsQueue = require('../helpers/jobsQueue.js'); var ip = require('ip'); var schema = require('../schema/loader.js'); +var slots = require('../helpers/slots.js'); var sql = require('../sql/loader.js'); require('colors'); @@ -301,7 +302,7 @@ __private.loadTransactions = function (cb) { * Loads last block and emits a bus message blockchain is ready. * @private * @implements {library.db.task} - * @implements {modules.rounds.calc} + * @implements {slots.calcRound} * @implements {library.bus.message} * @implements {library.logic.account.removeTables} * @implements {library.logic.account.createTables} @@ -319,7 +320,6 @@ __private.loadBlockChain = function () { function load (count) { verify = true; __private.total = count; - async.series({ removeTables: function (seriesCb) { library.logic.account.removeTables(function (err) { @@ -426,7 +426,7 @@ __private.loadBlockChain = function () { library.config.loading.snapshot = (round > 1) ? (round - 1) : 1; } - modules.rounds.setSnapshotRounds(library.config.loading.snapshot); + //modules.rounds.setSnapshotRounds(library.config.loading.snapshot); } library.logger.info('Snapshotting to end of round: ' + library.config.loading.snapshot); @@ -446,7 +446,7 @@ __private.loadBlockChain = function () { var count = countBlocks.count; library.logger.info('Blocks ' + count); - var round = modules.rounds.calc(count); + var round = slots.calcRound(count); if (count === 1) { return reload(count); @@ -462,9 +462,9 @@ __private.loadBlockChain = function () { var missed = !(countMemAccounts.count); - if (missed) { - return reload(count, 'Detected missed blocks in mem_accounts'); - } + // if (missed) { + // return reload(count, 'Detected missed blocks in mem_accounts'); + // } if (validateMemBalances.length) { return reload(count, 'Memory balances doesn\'t match blockchain balances'); @@ -836,7 +836,6 @@ Loader.prototype.onBind = function (scope) { transactions: scope.transactions, blocks: scope.blocks, peers: scope.peers, - rounds: scope.rounds, transport: scope.transport, multisignatures: scope.multisignatures, system: scope.system, diff --git a/modules/multisignatures.js b/modules/multisignatures.js index 6336c1ebcb2..4fce1b6bbee 100644 --- a/modules/multisignatures.js +++ b/modules/multisignatures.js @@ -178,7 +178,6 @@ Multisignatures.prototype.onBind = function (scope) { }; __private.assetTypes[transactionTypes.MULTI].bind( - scope.rounds, scope.accounts ); }; diff --git a/modules/rounds.js b/modules/rounds.js deleted file mode 100644 index 14f4220b3ba..00000000000 --- a/modules/rounds.js +++ /dev/null @@ -1,385 +0,0 @@ -'use strict'; - -var async = require('async'); -var constants = require('../helpers/constants.js'); -var Round = require('../logic/round.js'); -var slots = require('../helpers/slots.js'); -var sql = require('../sql/rounds.js'); - -// Private fields -var modules, library, self, __private = {}, shared = {}; - -__private.loaded = false; -__private.ticking = false; - -/** - * Initializes library with scope. - * @memberof module:rounds - * @class - * @classdesc Main rounds methods. - * @param {function} cb - Callback function. - * @param {scope} scope - App instance. - * @return {setImmediateCallback} Callback function with `self` as data. - * @todo apply node pattern for callbacks: callback always at the end. - */ -// Constructor -function Rounds (cb, scope) { - library = { - logger: scope.logger, - db: scope.db, - bus: scope.bus, - network: scope.network, - config: { - loading: { - snapshot: scope.config.loading.snapshot, - }, - }, - }; - self = this; - - setImmediate(cb, null, self); -} - -// Public methods -/** - * @return {boolean} __private.loaded - */ -Rounds.prototype.loaded = function () { - return __private.loaded; -}; - -/** - * @return {boolean} __private.ticking - */ -Rounds.prototype.ticking = function () { - return __private.ticking; -}; - -/** - * Returns average for each delegate based on height. - * @param {number} height - * @return {number} height / delegates - */ -Rounds.prototype.calc = function (height) { - return Math.ceil(height / slots.delegates); -}; - -/** - * Deletes from `mem_round` table records based on round. - * @implements {library.db.none} - * @param {number} round - * @param {function} cb - * @return {setImmediateCallback} error message | cb - * - */ -Rounds.prototype.flush = function (round, cb) { - library.db.none(sql.flush, {round: round}).then(function () { - return setImmediate(cb); - }).catch(function (err) { - library.logger.error(err.stack); - return setImmediate(cb, 'Rounds#flush error'); - }); -}; - -/** - * Performs backward tick on round - * @implements {calc} - * @implements {__private.getOutsiders} - * @implements {Round.mergeBlockGenerator} - * @implements {Round.markBlockId} - * @implements {Round.land} - * @implements {library.db.tx} - * @param {block} block - * @param {block} previousBlock - * @param {function} done - Callback function - * @return {function} done with error if any - */ -Rounds.prototype.backwardTick = function (block, previousBlock, done) { - var round = self.calc(block.height); - var prevRound = self.calc(previousBlock.height); - var nextRound = self.calc(block.height + 1); - - var scope = { - library: library, - modules: modules, - block: block, - round: round, - backwards: true - }; - - // Establish if finishing round or not - scope.finishRound = ( - (prevRound === round && nextRound !== round) || (block.height === 1 || block.height === 101) - ); - - function BackwardTick (t) { - var promised = new Round(scope, t); - - library.logger.debug('Performing backward tick'); - library.logger.trace(scope); - - return promised.mergeBlockGenerator().then(function () { - if (scope.finishRound) { - return promised.land().then(function () { - return promised.markBlockId(); - }); - } else { - return promised.markBlockId(); - } - }); - } - - async.series([ - function (cb) { - // Start round ticking - __private.ticking = true; - - // Sum round if finishing round - if (scope.finishRound) { - return __private.sumRound(scope, cb); - } else { - return setImmediate(cb); - } - }, - function (cb) { - // Get outsiders if finishing round - if (scope.finishRound) { - return __private.getOutsiders(scope, cb); - } else { - return setImmediate(cb); - } - }, - function (cb) { - // Perform round tick - library.db.tx(BackwardTick).then(function () { - return setImmediate(cb); - }).catch(function (err) { - library.logger.error(err.stack); - return setImmediate(cb, err); - }); - } - ], function (err) { - // Stop round ticking - __private.ticking = false; - return done(err); - }); -}; - -/** - * Sets snapshot rounds - * @param {number} rounds - */ -Rounds.prototype.setSnapshotRounds = function (rounds) { - library.config.loading.snapshot = rounds; -}; - -/** - * Generates snapshot round - * @implements {calc} - * @implements {Round.mergeBlockGenerator} - * @implements {Round.land} - * @implements {library.bus.message} - * @implements {Round.truncateBlocks} - * @implements {__private.getOutsiders} - * @param {block} block - * @param {function} done - * @return {function} done message | err - */ -Rounds.prototype.tick = function (block, done) { - var round = self.calc(block.height); - var nextRound = self.calc(block.height + 1); - - var scope = { - library: library, - modules: modules, - block: block, - round: round, - backwards: false - }; - - // Establish if snapshotting round or not - scope.snapshotRound = ( - library.config.loading.snapshot > 0 && library.config.loading.snapshot === round - ); - - // Establish if finishing round or not - scope.finishRound = ( - (round !== nextRound) || (block.height === 1 || block.height === 101) - ); - - function Tick (t) { - var promised = new Round(scope, t); - - library.logger.debug('Performing forward tick'); - library.logger.trace(scope); - - return promised.mergeBlockGenerator().then(function () { - if (scope.finishRound) { - return promised.land().then(function () { - library.bus.message('finishRound', round); - if (scope.snapshotRound) { - return promised.truncateBlocks().then(function () { - scope.finishSnapshot = true; - }); - } - }); - } - }); - } - - async.series([ - function (cb) { - // Start round ticking - __private.ticking = true; - - // Sum round if finishing round - if (scope.finishRound) { - return __private.sumRound(scope, cb); - } else { - return setImmediate(cb); - } - }, - function (cb) { - // Get outsiders if finishing round - if (scope.finishRound) { - return __private.getOutsiders(scope, cb); - } else { - return setImmediate(cb); - } - }, - // Perform round tick - function (cb) { - library.db.tx(Tick).then(function () { - return setImmediate(cb); - }).catch(function (err) { - library.logger.error(err.stack); - return setImmediate(cb, err); - }); - } - ], function (err) { - // Stop round ticking - __private.ticking = false; - - if (scope.finishSnapshot) { - return done('Snapshot finished'); - } else { - return done(err); - } - }); -}; - -// Events -/** - * Assigns modules to private variable `modules`. - * @param {modules} scope - Loaded modules. - */ -Rounds.prototype.onBind = function (scope) { - modules = { - blocks: scope.blocks, - accounts: scope.accounts, - delegates: scope.delegates, - }; -}; - -/** - * Sets private variable loaded to true. - * - * @public - * @method onBlockchainReady - * @listens module:loader~event:blockchainReady - * @param {block} block New block - */ -Rounds.prototype.onBlockchainReady = function () { - __private.loaded = true; -}; - -/** - * Emits a 'rounds/change' socket message. - * @implements {library.network.io.sockets.emit} - * @param {number} round - * @emits rounds/change - */ -Rounds.prototype.onFinishRound = function (round) { - library.network.io.sockets.emit('rounds/change', {number: round}); -}; - -/** - * Sets private variable `loaded` to false. - * @param {function} cb - * @return {setImmediateCallback} cb - */ -Rounds.prototype.cleanup = function (cb) { - __private.loaded = false; - return setImmediate(cb); -}; - -// Private methods -/** - * Generates outsiders array and pushes to param scope variable. - * Obtains delegate list and for each delegate generate address. - * @private - * @implements {modules.delegates.generateDelegateList} - * @implements {modules.accounts.generateAddressByPublicKey} - * @param {scope} scope - * @param {function} cb - * @return {setImmediateCallback} cb if block height 1 | error - */ -__private.getOutsiders = function (scope, cb) { - scope.roundOutsiders = []; - - if (scope.block.height === 1) { - return setImmediate(cb); - } - modules.delegates.generateDelegateList(scope.block.height, function (err, roundDelegates) { - if (err) { - return setImmediate(cb, err); - } - async.eachSeries(roundDelegates, function (delegate, eachCb) { - if (scope.roundDelegates.indexOf(delegate) === -1) { - scope.roundOutsiders.push(modules.accounts.generateAddressByPublicKey(delegate)); - } - return setImmediate(eachCb); - }, function (err) { - library.logger.trace('Got outsiders', scope.roundOutsiders); - return setImmediate(cb, err); - }); - }); -}; - -/** - * Gets rows from `round_blocks` and calculates rewards. Loads into scope - * variable fees, rewards and delegates. - * @private - * @implements {library.db.query} - * @param {number} round - * @param {function} cb - * @return {setImmediateCallback} err When failed to sum round | cb - */ -__private.sumRound = function (scope, cb) { - library.logger.debug('Summing round', scope.round); - - library.db.query(sql.summedRound, { round: scope.round, activeDelegates: constants.activeDelegates }).then(function (rows) { - var rewards = []; - - rows[0].rewards.forEach(function (reward) { - rewards.push(Math.floor(reward)); - }); - - scope.roundFees = Math.floor(rows[0].fees); - scope.roundRewards = rewards; - scope.roundDelegates = rows[0].delegates; - - library.logger.trace('roundFees', scope.roundFees); - library.logger.trace('roundRewards', scope.roundRewards); - library.logger.trace('roundDelegates', scope.roundDelegates); - - return setImmediate(cb); - }).catch(function (err) { - library.logger.error('Failed to sum round', scope.round); - library.logger.error(err.stack); - return setImmediate(cb, err); - }); -}; - -// Export -module.exports = Rounds; diff --git a/modules/transactions.js b/modules/transactions.js index 4da7bf7fa61..2fe04beee3e 100644 --- a/modules/transactions.js +++ b/modules/transactions.js @@ -587,8 +587,7 @@ Transactions.prototype.onBind = function (scope) { scope.loader ); __private.assetTypes[transactionTypes.SEND].bind( - scope.accounts, - scope.rounds + scope.accounts ); }; diff --git a/sql/rounds.js b/sql/rounds.js deleted file mode 100644 index 92d12d345a3..00000000000 --- a/sql/rounds.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict'; - -var RoundsSql = { - flush: 'DELETE FROM mem_round WHERE "round" = (${round})::bigint;', - - truncateBlocks: 'DELETE FROM blocks WHERE "height" > (${height})::bigint;', - - updateMissedBlocks: function (backwards) { - return [ - 'UPDATE mem_accounts SET "missedblocks" = "missedblocks"', - (backwards ? '- 1' : '+ 1'), - 'WHERE "address" IN ($1:csv);' - ].join(' '); - }, - - getVotes: 'SELECT d."delegate", d."amount" FROM (SELECT m."delegate", SUM(m."amount") AS "amount", "round" FROM mem_round m GROUP BY m."delegate", m."round") AS d WHERE "round" = (${round})::bigint', - - updateVotes: 'UPDATE mem_accounts SET "vote" = "vote" + (${amount})::bigint WHERE "address" = ${address};', - - updateBlockId: 'UPDATE mem_accounts SET "blockId" = ${newId} WHERE "blockId" = ${oldId};', - - summedRound: 'SELECT SUM(r.fee)::bigint AS "fees", ARRAY_AGG(r.reward) AS rewards, ARRAY_AGG(r.pk) AS delegates FROM (SELECT b."totalFee" AS fee, b.reward, ENCODE(b."generatorPublicKey", \'hex\') AS pk FROM blocks b WHERE CEIL(b.height / ${activeDelegates}::float)::int = ${round} ORDER BY b.height ASC) r;' -}; - -module.exports = RoundsSql; diff --git a/test/common/initModule.js b/test/common/initModule.js index e416eda515f..59843c0c36c 100644 --- a/test/common/initModule.js +++ b/test/common/initModule.js @@ -153,7 +153,6 @@ var modulesLoader = new function () { {loader: require('../../modules/loader')}, {multisignatures: require('../../modules/multisignatures')}, {peers: require('../../modules/peers')}, - {rounds: require('../../modules/rounds')}, {signatures: require('../../modules/signatures')}, {system: require('../../modules/system')}, {transactions: require('../../modules/transactions')}, diff --git a/test/index.js b/test/index.js index d332c86f9ad..39b292440de 100644 --- a/test/index.js +++ b/test/index.js @@ -1,6 +1,7 @@ require('./unit/helpers/request-limiter.js'); require('./unit/helpers/pg-notify.js'); require('./unit/helpers/jobs-queue.js'); +require('./unit/helpers/slots.js'); require('./unit/logic/blockReward.js'); require('./unit/sql/blockRewards.js'); require('./unit/sql/delegatesList.js'); diff --git a/test/node.js b/test/node.js index e75672caaad..72a6fb3b51a 100644 --- a/test/node.js +++ b/test/node.js @@ -2,7 +2,6 @@ // Root object var node = {}; -var Rounds = require('../modules/rounds.js'); var slots = require('../helpers/slots.js'); // Requires @@ -146,7 +145,7 @@ node.onNewRound = function (cb) { if (err) { return cb(err); } else { - var nextRound = Math.ceil(height / slots.delegates); + var nextRound = slots.calcRound(height); var blocksToWait = nextRound * slots.delegates - height; node.debug('blocks to wait: '.grey, blocksToWait); node.waitForNewBlock(height, blocksToWait, cb); diff --git a/test/unit/helpers/RoundChanges.js b/test/unit/helpers/RoundChanges.js deleted file mode 100644 index ad4e411b6f3..00000000000 --- a/test/unit/helpers/RoundChanges.js +++ /dev/null @@ -1,98 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var express = require('express'); -var ip = require('ip'); -var _ = require('lodash'); -var node = require('../../node.js'); -var RoundChanges = require('../../../helpers/RoundChanges.js'); - -describe('RoundChanges', function () { - - var validScope; - - beforeEach(function () { - validScope = { - round: 1, - roundFees: 500, - roundRewards: [0, 0, 100, 10] - }; - }); - - describe('constructor', function () { - - it('should accept valid scope', function () { - var roundChanges = new RoundChanges(validScope); - - node.expect(roundChanges.roundFees).equal(validScope.roundFees); - node.expect(_.isEqual(roundChanges.roundRewards, validScope.roundRewards)).to.be.ok; - }); - - it('should floor fees value', function () { - validScope.roundFees = 50.9999999999999; // Float - - var roundChanges = new RoundChanges(validScope); - - node.expect(roundChanges.roundFees).equal(50); - }); - - it('should round up fees after exceeding precision', function () { - validScope.roundFees = 50.999999999999999; // Exceeded precision - - var roundChanges = new RoundChanges(validScope); - - node.expect(roundChanges.roundFees).equal(51); - }); - - it('should accept Infinite fees as expected', function () { - validScope.roundFees = Number.MAX_VALUE * 2; // Infinity - - var roundChanges = new RoundChanges(validScope); - - node.expect(roundChanges.roundFees).equal(Infinity); - }); - }); - - describe('at', function () { - - it('should calculate round changes from valid scope', function () { - var roundChanges = new RoundChanges(validScope); - var rewardsAt = 2; - var res = roundChanges.at(rewardsAt); - - node.expect(res.fees).equal(4); - node.expect(res.feesRemaining).equal(96); - node.expect(res.rewards).equal(validScope.roundRewards[rewardsAt]); // 100 - node.expect(res.balance).equal(104); - }); - - it('should calculate round changes from Infinite fees', function () { - validScope.roundFees = Infinity; - - var roundChanges = new RoundChanges(validScope); - var rewardsAt = 2; - var res = roundChanges.at(rewardsAt); - - node.expect(res.fees).equal(Infinity); - node.expect(res.feesRemaining).to.be.NaN; - node.expect(res.rewards).equal(validScope.roundRewards[rewardsAt]); // 100 - node.expect(res.balance).equal(Infinity); - }); - - it('should calculate round changes from Number.MAX_VALUE fees', function () { - validScope.roundFees = Number.MAX_VALUE; // 1.7976931348623157e+308 - - var roundChanges = new RoundChanges(validScope); - var rewardsAt = 2; - var res = roundChanges.at(rewardsAt); - var expectedFees = 1779894192932990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099; // 1.7976931348623157e+308 / 101 (delegates num) - - node.expect(res.fees).equal(expectedFees); - node.expect(res.rewards).equal(validScope.roundRewards[rewardsAt]); // 100 - node.expect(res.feesRemaining).equal(1); - - var expectedBalance = 1779894192932990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990099009900990199; // 1.7976931348623157e+308 / 101 (delegates num) + 100 - node.expect(res.balance).equal(expectedBalance); - }); - }); -}); diff --git a/test/unit/helpers/slots.js b/test/unit/helpers/slots.js new file mode 100644 index 00000000000..43952f04158 --- /dev/null +++ b/test/unit/helpers/slots.js @@ -0,0 +1,26 @@ +'use strict'; + +var chai = require('chai'); +var express = require('express'); +var _ = require('lodash'); +var node = require('../../node.js'); +var slots = require('../../../helpers/slots.js'); + +describe('helpers/slots', function () { + + describe('calc', function () { + + it('should calculate round number from given block height', function () { + node.expect(slots.calcRound(100)).equal(1); + node.expect(slots.calcRound(200)).equal(2); + node.expect(slots.calcRound(303)).equal(3); + node.expect(slots.calcRound(304)).equal(4); + }); + + it('should calculate round number from Number.MAX_VALUE', function () { + var res = slots.calcRound(Number.MAX_VALUE); + node.expect(_.isNumber(res)).to.be.ok; + node.expect(res).to.be.below(Number.MAX_VALUE); + }); + }); +}); diff --git a/test/unit/logic/transaction.js b/test/unit/logic/transaction.js index b0d6bdcee45..ef8d0b324fe 100644 --- a/test/unit/logic/transaction.js +++ b/test/unit/logic/transaction.js @@ -14,7 +14,6 @@ var slots = require('../../../helpers/slots'); var modulesLoader = require('../../common/initModule').modulesLoader; var Transaction = require('../../../logic/transaction.js'); -var Rounds = require('../../../modules/rounds.js'); var AccountLogic = require('../../../logic/account.js'); var AccountModule = require('../../../modules/accounts.js'); @@ -133,10 +132,10 @@ describe('transaction', function () { var transaction; var accountModule; - var attachTransferAsset = function (transaction, accountLogic, rounds, done) { + var attachTransferAsset = function (transaction, accountLogic, done) { modulesLoader.initModuleWithDb(AccountModule, function (err, __accountModule) { var transfer = new Transfer(); - transfer.bind(__accountModule, rounds); + transfer.bind(__accountModule); transaction.attachAssetType(transactionTypes.SEND, transfer); accountModule = __accountModule; done(); @@ -150,9 +149,6 @@ describe('transaction', function () { before(function (done) { async.auto({ - rounds: function (cb) { - modulesLoader.initModule(Rounds, modulesLoader.scope,cb); - }, accountLogic: function (cb) { modulesLoader.initLogicWithDb(AccountLogic, cb); }, @@ -165,7 +161,7 @@ describe('transaction', function () { }, function (err, result) { transaction = result.transaction; transaction.bindModules(result); - attachTransferAsset(transaction, result.accountLogic, result.rounds, done); + attachTransferAsset(transaction, result.accountLogic, done); }); }); diff --git a/test/unit/logic/transfer.js b/test/unit/logic/transfer.js index d33e0d20fcb..a80a8e52d35 100644 --- a/test/unit/logic/transfer.js +++ b/test/unit/logic/transfer.js @@ -14,7 +14,6 @@ var transactionTypes = require('../../../helpers/transactionTypes'); var modulesLoader = require('../../common/initModule').modulesLoader; var TransactionLogic = require('../../../logic/transaction.js'); var Transfer = require('../../../logic/transfer.js'); -var Rounds = require('../../../modules/rounds.js'); var AccountLogic = require('../../../logic/account.js'); var AccountModule = require('../../../modules/accounts.js'); var DelegateModule = require('../../../modules/delegates.js'); @@ -101,15 +100,11 @@ describe('transfer', function () { before(function (done) { async.auto({ - rounds: function (cb) { - modulesLoader.initModule(Rounds, modulesLoader.scope, cb); - }, accountLogic: function (cb) { modulesLoader.initLogicWithDb(AccountLogic, cb, {}); }, - transactionLogic: ['rounds', 'accountLogic', function (result, cb) { + transactionLogic: ['accountLogic', function (result, cb) { modulesLoader.initLogicWithDb(TransactionLogic, function (err, __transaction) { - __transaction.bindModules(result.rounds); cb(err, __transaction); }, { ed: require('../../../helpers/ed'), @@ -128,10 +123,9 @@ describe('transfer', function () { expect(err).to.not.exist; transfer = new Transfer(); transferBindings = { - account: result.accountModule, - rounds: result.rounds + account: result.accountModule }; - transfer.bind(result.accountModule, result.rounds); + transfer.bind(result.accountModule); transaction = result.transactionLogic; transaction.attachAssetType(transactionTypes.SEND, transfer); accountModule = result.accountModule; @@ -143,12 +137,12 @@ describe('transfer', function () { describe('bind', function () { it('should be okay with correct params', function () { expect(function () { - transfer.bind(transferBindings.account, transferBindings.rounds); + transfer.bind(transferBindings.account); }).to.not.throw(); }); after(function () { - transfer.bind(transferBindings.account, transferBindings.rounds); + transfer.bind(transferBindings.account); }); }); diff --git a/test/unit/logic/vote.js b/test/unit/logic/vote.js index 0d23cda4d8c..4579c72af7b 100644 --- a/test/unit/logic/vote.js +++ b/test/unit/logic/vote.js @@ -16,7 +16,6 @@ var TransactionLogic = require('../../../logic/transaction.js'); var Vote = require('../../../logic/vote.js'); var Transfer = require('../../../logic/transfer.js'); var Delegate = require('../../../logic/delegate.js'); -var Rounds = require('../../../modules/rounds.js'); var AccountLogic = require('../../../logic/account.js'); var AccountModule = require('../../../modules/accounts.js'); var DelegateModule = require('../../../modules/delegates.js'); @@ -121,13 +120,10 @@ describe('vote', function () { before(function (done) { async.auto({ - rounds: function (cb) { - modulesLoader.initModule(Rounds, modulesLoader.scope, cb); - }, accountLogic: function (cb) { modulesLoader.initLogicWithDb(AccountLogic, cb, {}); }, - transactionLogic: ['rounds', 'accountLogic', function (result, cb) { + transactionLogic: ['accountLogic', function (result, cb) { modulesLoader.initLogicWithDb(TransactionLogic, function (err, __transaction) { __transaction.bindModules(result); cb(err, __transaction); @@ -148,7 +144,6 @@ describe('vote', function () { modulesLoader.initModuleWithDb(DelegateModule, function (err, __delegates) { // not all required bindings, only the ones required for votes __delegates.onBind({ - rounds: result.rounds, accounts: result.accountModule, }); cb(err, __delegates); @@ -166,10 +161,9 @@ describe('vote', function () { vote = new Vote(modulesLoader.scope.logger, modulesLoader.scope.schema); voteBindings = { delegate: result.delegateModule, - rounds: result.rounds, account: result.accountModule }; - vote.bind(result.delegateModule, result.rounds); + vote.bind(result.delegateModule); transaction = result.transactionLogic; transaction.attachAssetType(transactionTypes.VOTE, vote); accountsModule = result.accountModule; @@ -180,7 +174,7 @@ describe('vote', function () { before(function (done) { // create new account for testing; var transfer = new Transfer(); - transfer.bind(voteBindings.account, voteBindings.rounds); + transfer.bind(voteBindings.account); transaction.attachAssetType(transactionTypes.SEND, transfer); var sendTrs = { @@ -224,12 +218,12 @@ describe('vote', function () { it('should be okay with correct params', function () { expect(function () { - vote.bind(voteBindings.delegate, voteBindings.rounds); + vote.bind(voteBindings.delegate); }).to.not.throw(); }); after(function () { - vote.bind(voteBindings.delegate, voteBindings.rounds); + vote.bind(voteBindings.delegate); }); }); diff --git a/test/unit/modules/rounds.js b/test/unit/modules/rounds.js deleted file mode 100644 index 5683d6ddde9..00000000000 --- a/test/unit/modules/rounds.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var express = require('express'); -var _ = require('lodash'); -var node = require('../../node.js'); -var Rounds = require('../../../modules/rounds.js'); -var modulesLoader = require('../../common/initModule').modulesLoader; - -describe('rounds', function () { - - var rounds; - - before(function (done) { - modulesLoader.initModuleWithDb(Rounds, function (err, __rounds) { - if (err) { - return done(err); - } - rounds = __rounds; - done(); - }); - }); - - describe('calc', function () { - - it('should calculate round number from given block height', function () { - node.expect(rounds.calc(100)).equal(1); - node.expect(rounds.calc(200)).equal(2); - node.expect(rounds.calc(303)).equal(3); - node.expect(rounds.calc(304)).equal(4); - }); - - it('should calculate round number from Number.MAX_VALUE', function () { - var res = rounds.calc(Number.MAX_VALUE); - node.expect(_.isNumber(res)).to.be.ok; - node.expect(res).to.be.below(Number.MAX_VALUE); - }); - }); -}); diff --git a/test/unit/sql/blockRewards.js b/test/unit/sql/blockRewards.js index 01d2d1840b5..f5f72f8e02e 100644 --- a/test/unit/sql/blockRewards.js +++ b/test/unit/sql/blockRewards.js @@ -8,19 +8,6 @@ var constants = require('../../../helpers/constants.js'); var modulesLoader = require('../../common/initModule').modulesLoader; var db; -before(function (done) { - modulesLoader.getDbConnection(function (err, db_handle) { - if (err) { - return done(err); - } - db = db_handle; - done(); - }); -}); - -constants.rewards.distance = 3000000; -constants.rewards.offset = 1451520; - function calcBlockReward (height, reward, done) { return db.query(sql.calcBlockReward, {height: height}).then(function (rows) { expect(rows).to.be.an('array'); @@ -90,6 +77,18 @@ function calcBlockReward_test (height_start, height_end, expected_reward, done) }; describe('BlockRewardsSQL', function () { + before(function (done) { + modulesLoader.getDbConnection(function (err, db_handle) { + if (err) { + return done(err); + } + db = db_handle; + done(); + }); + }); + + constants.rewards.distance = 3000000; + constants.rewards.offset = 1451520; describe('checking SQL function getBlockRewards()', function () { @@ -117,7 +116,6 @@ describe('BlockRewardsSQL', function () { }); }); - describe('checking SQL function calcBlockReward(int)', function () { it('when height is undefined should return null', function (done) { diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index fcb0a3d6d63..7edc12c44a7 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -8,16 +8,6 @@ var sql = require('../../sql/delegatesList.js'); var modulesLoader = require('../../common/initModule').modulesLoader; var db; -before(function (done) { - modulesLoader.getDbConnection(function (err, db_handle) { - if (err) { - return done(err); - } - db = db_handle; - done(); - }); -}); - function generateDelegatesList (round, delegates) { var i, x, n, old, len; var list = []; @@ -45,6 +35,49 @@ function generateDelegatesList (round, delegates) { describe('DelegatesListSQL', function () { + before(function (done) { + modulesLoader.getDbConnection(function (err, db_handle) { + if (err) { + return done(err); + } + db = db_handle; + done(); + }); + }); + + function getKeysSortByVote (cb) { + library.db.query(sql.delegateList).then(function (rows) { + return setImmediate(cb, null, rows.map(function (el) { + return el.pk; + })); + }).catch(function (err) { + return setImmediate(cb, err); + }); + }; + + function generateDelegateList (height, cb) { + getKeysSortByVote(function (err, truncDelegateList) { + if (err) { + return setImmediate(cb, err); + } + + var seedSource = modules.rounds.calc(height).toString(); + var currentSeed = crypto.createHash('sha256').update(seedSource, 'utf8').digest(); + + for (var i = 0, delCount = truncDelegateList.length; i < delCount; i++) { + for (var x = 0; x < 4 && i < delCount; i++, x++) { + var newIndex = currentSeed[x] % delCount; + var b = truncDelegateList[newIndex]; + truncDelegateList[newIndex] = truncDelegateList[i]; + truncDelegateList[i] = b; + } + currentSeed = crypto.createHash('sha256').update(currentSeed).digest(); + } + + return setImmediate(cb, null, truncDelegateList); + }); + } + describe('checking SQL function generateDelegatesList()', function () { it('SQL generateDelegatesList() results should be equal to generateDelegatesList() - fake 101 delegates', function (done) { var round = '26381'; From 65ab9823116ace7a0cb05cb96f88720bb166fdc4 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 03:53:11 +0200 Subject: [PATCH 45/88] Removed rounds unit test from Jenkinsfile --- Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 84323893502..f3e113a39dc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -309,9 +309,6 @@ lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { export TEST=test/unit/modules/peers.js TEST_TYPE='UNIT' npm run jenkins - export TEST=test/unit/modules/rounds.js TEST_TYPE='UNIT' - npm run jenkins - # Temporarily disabled until implemented #TEST=test/unit/modules/transactions.js TEST_TYPE='UNIT' #npm run jenkins From af41657c008c3dbfc188cb99bf9cf8d18173bd2e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 04:32:39 +0200 Subject: [PATCH 46/88] Remove dapp name and link from unconfirmed tracking objects when apply --- logic/dapp.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/logic/dapp.js b/logic/dapp.js index 6c969ac6a75..7810dfbfa03 100644 --- a/logic/dapp.js +++ b/logic/dapp.js @@ -264,6 +264,9 @@ DApp.prototype.getBytes = function (trs) { * @return {setImmediateCallback} cb */ DApp.prototype.apply = function (trs, block, sender, cb) { + delete __private.unconfirmedNames[trs.asset.dapp.name]; + delete __private.unconfirmedLinks[trs.asset.dapp.link]; + return setImmediate(cb); }; From a846cc38d5a76ed78934fc346ba7b18d0f5358d6 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 17:28:44 +0200 Subject: [PATCH 47/88] Allow pg-notify helper to receive JSON in notifications --- helpers/pg-notify.js | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index d6dc305221c..fe2e1bed5bf 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -23,13 +23,21 @@ function onNotification (data) { // Process round-releated things if (data.channel === 'round-closed') { - data.payload = parseInt(data.payload); - logger.info('pg-notify: Round closed', data.payload); - // Set new round - data.payload += 1; + logger.info('pg-notify: Round closed'); + try { + data.payload = JSON.parse(data.payload); + } catch(e) { + logger.error('pg-notify: Unable to parse JSON', {err: e, data: data}); + return; + } } else if (data.channel === 'round-reopened') { - data.payload = parseInt(data.payload); - logger.warn('pg-notify: Round reopened', data.payload); + logger.warn('pg-notify: Round reopened'); + try { + data.payload = JSON.parse(data.payload); + } catch(e) { + logger.error('pg-notify: Unable to parse JSON', {err: e, data: data}); + return; + } } else { // Channel is not supported - should never happen logger.error('pg-notify: Channel not supported', data.channel); From 6c3086be5d0d67dfe86d3c4f58640e8c209e04c5 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 17:31:07 +0200 Subject: [PATCH 48/88] Changes expectations for pg-notify unit test, add test cases --- test/sql/pgNotify.js | 3 +- test/unit/helpers/pg-notify.js | 119 ++++++++++++++++++++++++++++----- 2 files changed, 105 insertions(+), 17 deletions(-) diff --git a/test/sql/pgNotify.js b/test/sql/pgNotify.js index bf974f4b7ec..254cb38ca1b 100644 --- a/test/sql/pgNotify.js +++ b/test/sql/pgNotify.js @@ -2,7 +2,8 @@ var pgNotify = { interruptConnection: 'SELECT pg_terminate_backend(${pid});', - triggerNotify: 'SELECT pg_notify(${channel}, ${message});' + triggerNotify: 'SELECT pg_notify(${channel}, json_build_object(\'round\', CEIL((SELECT height FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int, \'list\', generateDelegatesList(CEIL((SELECT height FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int, ARRAY(SELECT ENCODE(pk, \'hex\') AS pk FROM delegates ORDER BY rank ASC LIMIT 101)))::text);', + triggerNotifyWithMessage: 'SELECT pg_notify(${channel}, ${message});' }; module.exports = pgNotify; diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 47224f6574f..53419a72f79 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -1,20 +1,23 @@ 'use strict'; // Init tests dependencies -var chai = require('chai'); +var chai = require('chai'); var expect = require('chai').expect; -var sinon = require('sinon'); +var sinon = require('sinon'); var rewire = require('rewire'); // Load config file - global (not one from test directory) var config = require('../../../config.json'); -var sql = require('../../sql/pgNotify.js'); +var sql = require('../../sql/pgNotify.js'); +var slots = require('../../../helpers/slots.js'); // Init tests subject var pg_notify = rewire('../../../helpers/pg-notify.js'); // Init global variables var db, invalid_db, logger, bus; +var delegates_list_json = '{"round": 666, "list": ["2f872264534a1722e136bddf29a301fa97708f88583130770d54c1d11366e5fc","c1b5b642849729f087c862046391bc744717541cfe0b52968c705148c576fbf1","1d4941371e39987f25e2ac9f6d579d2162dd1d0bfc50bbba65694d990e7da137","0c7c2b612db9cccba57583e962bb609ea67838f8616546ce946eaf40bf73a2da","7fe4fb9ede1b715b20afcee7f5258a02a4d007c5ede2f0287c7ba1c2c66638b8","90e2f5ca74d39a76c117bc0e3be5e8b572b47fa12536fabfe5802972db943db8","54cd463ad4769270c2057dbc8ad2512452882631c952531dad09ae79b4b78fc1","9d6d602db22619c59a3932d3eba3c4e81277832bab24454676191a5fe1e7385b","8ec002db73e38a22f76f70c9f94968f99d77f1c009f23860400427ab63296785","0a4178fbca14a0cf4b5eb934343da94d766b24a673d6cf197b5d1dcaaf6f9f55","8f375813bfa12a510f556b1fbd3796d5ac08c5356c2982471cc79a68d73f3308","60d36c7ba3b76c2bb3e2d2f8bedf61f17a54de9cfd8f5e49c1fe5658e8832dcc","326544de0b5ae1d73c5b7c233f873de57b02fbec890c90847aa46e621a164b3d","49664663158a141d18b44954959728545fd0597317ccd8f99d5f206dd424cc23","2a899596ae3a8ab438f7efc7ef2b2b7f48d20315e9c669ecb91d7d679f993378","c97d92df8210680610637751b571563120da4e8aa78cc757356efbfdfd0f4d25","366cfff7cc7f96e16c512b9df04f5bc05b694cb253a46e52f1ac8d79be223b06","2052d8fdd3fab6334f77730c4d56de4be35cea5bae706b945ae8f19ea4d62c81","f8453e3b8eb53e26e4486a50ad85fa5be5e9aeb30542d5b71d6c6d20be09c298","12c8dd3b4ca12fdade63c27f767ed85e012d54847932b5c65dd51aacb0d7d25b","c7fc591a74e216fde01fbe89346f919688e9d8d304c593a6d985ec48533a3f48","ed48864abffe2aa66b2f0bdfe43f151e87f653543f4ea5f61f469fdc655754cc","09a4491e247a8a871e48904c69fcd58e92bc5d06075fb23f5813736de516e2b3","6dceabf4f0f8cb0101165cb00689372d60617cd24894d46824a620011c3f297d","ad26f937b4e8d2f663df8b148ff2b56cd768766eac59ba78551f5a415fd94924","f8313e19ad3e3b61cab3c591147d0fce90429696b96f82243ffab10f58a20678","0cfb773eac46cd18609323e298711c427f382d847a5f078c690a21ce09ef1c9c","3fc7907abb6ca45b3c9f315f61a488d0999744a590918ed4f4673d615fc7f35f","9dc80749af77db925345ccc83e4cb74a9548ce11713b74e886efeb5ba13204d1","4b61c3180e4d50681c8e8ba22bd8624d1fb990d3430832993b96d6e177f67a79","b0968d8fd6ab885b1df6ec2af2cacc87c8ee56a3429d39064a69f59a7533a5ad","5510cff50bb32a2f38dd0a6d3d95653810e20483fb280c50f152c2e23bfb69ca","fdd2b4a9842d626bc89028bb5a766fd17ecb292a8003f06b8f824fac86f2cf53","f8d8aa286b689fa7279ad99f0fdfaeb3c0d2b22bee62230df2ba2fe40b9eb628","0a47b151eafe8cfc278721ba14305071cae727395abf4c00bd298296c851dab9","ec5e69549e2b278cc81278822310f040a6fa25472e73ed3fa31d2854d905b7db","1b4662cbd609141211518497e524d401087ab051abf753f6b1ddd41baa43b35a","79de262387a6d51f5510d3dca33c40e614510f00f7bfcc5f9a1ba3a704104fcf","baa1a74353d4dc44caeda1a6b1fdb9a2e2089ab25678cf9e8ceea54b574fa745","82018190e97c950275aa73546510c8acb9203254234cf15ac5376970c0bfb8fa","95c5f754d58fe448ea4013b48abe3fd95250a47e0b3ffb1f62fb026b7e59e87f","8162b08bc15732b4220814693d16eb2db9eee0801eca9fa2015b127d32be4f8d","d18a5f22a1402275fe9d40f4609e5688c1c2a200fb1ba962310c5145114cf34a","2be0301710d1295f9afeba6c469f7447e6915e3e63fcbdd69c3fa54a50184803","14b7969b93494a1e8345e8330148e4a88da6d30dacf02388fdd8a2ba63f2c69d","7d76d68a1fabf8a00ff97b7ae8514d23923e2acfc5594d1f47e3ace5e6a1649e","2c3f411ca51c70e46cd19835ec687f49ae490dc405ffae7bf4ea23607d9ea5cc","83811ea7f9b00e2d540f33c7fa431ac71e734abde89dffd0ff8eb9b09277fc13","9bb219513cfdcf75d096e46ec338732bf78f1453c1c9e949fb14590bccdee31c","33a8d3d41f3276cef20c6ed9c468c31a75ad9643dc3347e8a9370b1ba38524d1","35ab9b36db1207eee23ca6d0706c2fedc82ccf4856453ce7079b1de12fc882b7","f14daa7a42093f0d1b60b8ca0fb70f3ce9dd25f1dd80bdd0ff8cf3875ddf1d16","f555bf37565aab4a500a8225a178bae5afa20c749ce2266d1010728b4578c8cf","1e52b5f94256b29d3978f1a3e1933d2c917376994abe1e03af1332d5f56a83a1","f42203fbd0e6a781530f8e60e41603b04b54cc148b8fc7b975cebe33a682dbb2","362ec7711dd52f97e24af8823e1abc58f545353f6ab13c3855f0020f86f2cd15","4326747318f97e25bbb1873e63a2b54065851175125d9769f3e731eb7e865cf2","ec8efaf175bc9b292302da2de8595cb2e17cc37fbba4dd606b98e1717aa281b1","855e01df4e59df149773aac715340fcc3a9f8e79b18fc63483b8e78c70d08707","e34473ed004f6b39e0af599fe4f43f5bc36286680140f80c026c25e91955f03d","aa9bac4ccaab8206a0bafb3fd8979b13db29a56c632586c0fc2e43452f1704c7","02a968172428da5c33c1f12ecc6a3117886018d42e7a996a263da362ec3aa9bb","d277e465d3b802e51adc7897440f1b2c2037b4dbb59347ccfe0db253df259477","4a18c604dae421566de114ccaef1da51f9ee108a2cfccc33b4e98ac817875ab9","89e546e33eadad6d04d0d89946432058652b2491739f73128ee7b7cfd2a3776c","c379455ea222666817e8b6d7673fb47f3594ab0516441efbcf93c0ab0d9ab15f","1e82d980d55167ece64111a175f6ce558714e4a5a9e3f69fc8776abeedc3e214","a1f23beac0af1cf5d973509c872e8552ef02d585de729ed07e1aa27cc224d262","b3484d1993d9dee4c2225311ff81222c8f8b57e21d1a9f2540ecee31be48b2d9","b55fb628cc9dad2f4073a402941ffc2f53cd0b3e882b7f2b1770d58d2e439df9","80a15f589177338c086095cd80f157d964d78b463684388d9e0c7a126c9313a8","1a8de1c82a2bb79b51fc452580ddcd30f802de268cf914bee54e5e3c72505ae0","687f54af22b69a15d685fe4aa3ee157a3a17c6c62cad86cde3066bdc26dbb69f","14d6e1db90c94afc0d59bf52061311c20b030a13919842b1af06e94951219f4d","9548a003ec975142d99e0e9a720655b4be0a08801f940dc0fbc242de2ecd5558","f654f16a9d538120b391fb2de6683802d5926408b7b2c95f7307ebd83156eb77","703ee0695471a0c274dc225d691fb2387e061bbb28f9c729c09149f20a7efe44","95f0cc8ee70052aa78866ec0be9146917d05a05db814b2c4d4ed151b4e5e6f7b","a21f655cf396727c186254f23d9f266481884442f6338a9d55f603a7abbdcb61","dae47d539641710c933136522794c56f2152c8a41fb1f8599666f01f25c977eb","5351f73cc176500d968f50aed518fb1fe9c748d14af905e740f03fbd5ee4e51e","b6f1017fc2d51dbe5e30115e1cbd7619269969bee098359da0eef83916fa49ff","09e13e1c72143c9b75013f0d5fe13e1e978e608ea883bb93a3a9c38f0c8826f3","b06fc6c45bfccb5ab0da94a06f083b08ee069f849cccafd68da3ce334f330da7","129049f0f3fb76738123e483060fd83cbff2d90d67833763f54717d915a23cb3","ecfb82f80f204e508b83c0eab2543b33b946fe31958f3d7ce91015dbe3c7f31a","05c3190c0bb57d9908c2b82f8b8d6d0f2b69f1ce7dff0c7fccaff23c00b8072b","02a70630d4f4eb9722763ac91abf2048fcaa9172e8734a32dbd1f70c6c5668a8","38de3395db59b478a2a2a1bc5b24fca3c2ffea77f101ff9d4e39809afac842f9","8e094cebc4cdafdf379b5ce5097fe5910f2dc34ae5fe223a1cc3ef6fe55ca51b","3046ab9cd2cf06e8483873852595fbaee6c3078caa169d58e17a6b2371ce05b7","08b81a3228a70b6a96d7e432718e3ee36d4a691c34a9986b202378da0ebe4357","66e6564444ce14caab3e5c92084e163ce7af71847f32669a42b4959f430aa999","fa7e7da4d339d0cd07290364509b0b90758e1654de6cd9cbb04672b555680597","084b559ee2fec7d755aa458387e5276f830ddf4b08dcfc2ff11e3dc38d451493","7d395718ae51f1dedd3ceff4fb3fb807c57bee88ae93d05684ede9d3152d53ce","37d70714bf4e447ba80bdb348ae38f0fced03551fccf5e52dcab8cbbdf7b4e15","7f31ab028c700d4cf0ca933866c40a5249bc78cbc4d108cbb468e0bcd190715d","09702cba04017cf2847925e6d072d708a7933ab9a385ac77549fda28991907dc","6122ac1fd71b437014ddbc4ec01e07879f5af1853536efaa0233bc12907c684b","0f6a1ac6c2a98dc611a1de1ed8fb81186ec6d1852f9054841dc36e73bdd9e20d"]}'; +var delegates_list = JSON.parse(delegates_list_json); describe('helpers/pg-notify', function () { @@ -74,6 +77,14 @@ describe('helpers/pg-notify', function () { reconnect(done); }); + afterEach(function () { + var connection = pg_notify.__get__('connection'); + // Release the connection + if (connection) { + connection.done(); + } + }); + describe('init', function () { it('try to estabilish initial connection with valid params should succeed', function (done) { pg_notify.init(db, bus, logger, function (err) { @@ -270,15 +281,55 @@ describe('helpers/pg-notify', function () { }); describe('onNotification', function () { + it('should notify about round-closed event (message generated by query)', function (done) { + var channel = 'round-closed'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { + expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); + expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); + expect(logger.warn.args[0]).to.be.undefined; + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0][0]).to.equal('roundChanged'); + expect(bus.message.args[0][1].round).to.be.a('number'); + expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); + + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + it('should notify about round-closed event', function (done) { var channel = 'round-closed'; - var message = '123'; // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: '123'}]); - expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed', 123]); - expect(bus.message.args[0]).to.deep.equal(['roundChanged', 124]); + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: delegates_list_json}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: delegates_list_json}]); + expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); + expect(logger.warn.args[0]).to.be.undefined; + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); + + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should notify about round-reopened event (message generated by query)', function (done) { + var channel = 'round-reopened'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { + expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0][0]).to.equal('roundChanged'); + expect(bus.message.args[0][1].round).to.be.a('number'); + expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); + done(); }, 20)).catch(function (err) { done(err); @@ -287,13 +338,48 @@ describe('helpers/pg-notify', function () { it('should notify about round-reopened event', function (done) { var channel = 'round-reopened'; - var message = '123'; // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: '123'}]); - expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened', 123]); - expect(bus.message.args[0]).to.deep.equal(['roundChanged', 123]); + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: delegates_list_json}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: delegates_list_json}]); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should not notify about round-closed event when payload is not valid JSON', function (done) { + var channel = 'round-closed'; + var message = 'not_json'; + + // Execute query that trigger notify + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: message}]); + expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); + expect(logger.warn.args[0]).to.be.undefined; + expect(logger.error.args[0][0]).to.equal('pg-notify: Unable to parse JSON'); + expect(bus.message.args[0]).to.be.undefined; + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should not notify about round-reopened event when payload is not valid JSON', function (done) { + var channel = 'round-reopened'; + var message = 'not_json'; + + // Execute query that trigger notify + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: message}]); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); + expect(logger.error.args[0][0]).to.equal('pg-notify: Unable to parse JSON'); + expect(bus.message.args[0]).to.be.undefined; done(); }, 20)).catch(function (err) { done(err); @@ -305,10 +391,11 @@ describe('helpers/pg-notify', function () { var message = 'unknown'; // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { expect(logger.debug.args[0]).to.be.undefined; expect(logger.info.args[0]).to.be.undefined; expect(logger.warn.args[0]).to.be.undefined; + expect(logger.error.args[0]).to.be.undefined; expect(bus.message.args[0]).to.be.undefined; done(); }, 20)).catch(function (err) { @@ -324,7 +411,7 @@ describe('helpers/pg-notify', function () { var restore = pg_notify.__set__('channels', {}); // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: '123'}]); expect(logger.error.args[0]).to.deep.equal(['pg-notify: Invalid channel', 'round-reopened']); expect(logger.info.args[0]).to.be.undefined; @@ -348,7 +435,7 @@ describe('helpers/pg-notify', function () { resetSpiesState(); // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel, message: message}).then(setTimeout(function () { + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'test', data: '123'}]); expect(logger.error.args[0]).to.deep.equal(['pg-notify: Channel not supported', 'test']); expect(logger.info.args[0]).to.be.undefined; From 34a0c941a1d8071f50d21d9acffa6d446c20301b Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 17:35:18 +0200 Subject: [PATCH 49/88] Return round number and delegates list via notifications --- sql/migrations/20170521001337_roundsRewrite.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index a1b5af35bf7..b97ff20639d 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -272,10 +272,10 @@ CREATE FUNCTION delegates_update_on_block() RETURNS TRIGGER LANGUAGE PLPGSQL AS -- Perform notification to backend that round changed IF (TG_OP = 'INSERT') THEN -- Last block of round inserted - round is closed here and processing is done - PERFORM pg_notify('round-closed', CEIL(NEW.height / 101::float)::text); + PERFORM pg_notify('round-closed', json_build_object('round', CEIL((NEW.height+1) / 101::float)::int, 'list', generateDelegatesList(CEIL((NEW.height+1) / 101::float)::int, ARRAY(SELECT ENCODE(pk, 'hex') AS pk FROM delegates ORDER BY rank ASC LIMIT 101)))::text); ELSIF (TG_OP = 'DELETE') THEN -- Last block of round deleted - round reopened, processing is done here - PERFORM pg_notify('round-reopened', CEIL(OLD.height / 101::float)::text); + PERFORM pg_notify('round-reopened', json_build_object('round', CEIL((OLD.height) / 101::float)::int, 'list', generateDelegatesList(CEIL((OLD.height) / 101::float)::int, ARRAY(SELECT ENCODE(pk, 'hex') AS pk FROM delegates ORDER BY rank ASC LIMIT 101)))::text); END IF; RETURN NULL; END $$; From 269c88018f6998f090eb8b8dcaa8b8c6ad38bcdb Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 17:47:34 +0200 Subject: [PATCH 50/88] Support new data fromat in onRoundChanged events --- modules/cache.js | 6 ++++-- modules/delegates.js | 19 ++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/modules/cache.js b/modules/cache.js index d512daccb10..917c3013dc3 100644 --- a/modules/cache.js +++ b/modules/cache.js @@ -166,10 +166,12 @@ Cache.prototype.onNewBlock = function (block, broadcast, cb) { /** * This function will be triggered when a round changed, it will clear all cache entires. - * @param {Round} round + * @param {object} data Data received from postgres + * @param {object} data.round Current round + * @param {object} data.list Delegates list used for slot calculations * @param {Function} cb */ -Cache.prototype.onRoundChanged = function (round, cb) { +Cache.prototype.onRoundChanged = function (data, cb) { cb = cb || function () {}; if(!self.isReady()) { return cb(errorCacheDisabled); } diff --git a/modules/delegates.js b/modules/delegates.js index 8186d63c0b2..a759ebf7b88 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -479,22 +479,19 @@ Delegates.prototype.onBind = function (scope) { }; /** - * Handle node shutdown request + * Trigger when get notification from postgres that round changed * * @public * @method onRoundChanged * @listens module:pg-notify~event:roundChanged - * @implements module:delegates#generateDelegateList - * @param {number} round Current round + * @param {object} data Data received from postgres + * @param {object} data.round Current round + * @param {object} data.list Delegates list used for slot calculations */ -Delegates.prototype.onRoundChanged = function (round) { - self.generateDelegateList(function (err) { - if (err) { - library.logger.error('Cannot get delegates list for round', err); - } - - library.network.io.sockets.emit('rounds/change', {number: round}); - }); +Delegates.prototype.onRoundChanged = function (data) { + __private.delegatesList = data.list; + library.network.io.sockets.emit('rounds/change', {number: data.round}); + library.logger.info('Round changed, current round', data.round); }; /** From 300e55e45bb8848c4710b543855d9f2736f0d913 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 18:32:38 +0200 Subject: [PATCH 51/88] Added note about pgcrypto extension to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 22265af3e7f..6d67efdd5f7 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ Lisk is a next generation crypto-currency and decentralized application platform sudo -u postgres psql -d lisk_main -c "alter user "$USER" with password 'password';" ``` + **NOTE:** Database user require privileges to `CREATE EXTENSION pgcrypto`. + - Bower () -- Bower helps to install required JavaScript dependencies. `npm install -g bower` From 726feaf45cb19c4a63aaeb0a05f99c5f556ffed0 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 20:48:19 +0200 Subject: [PATCH 52/88] Added test for SQL function getDelegatesList --- test/sql/delegatesList.js | 3 +++ test/unit/sql/delegatesList.js | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/test/sql/delegatesList.js b/test/sql/delegatesList.js index e09ae042d4e..f2a1b852f9f 100644 --- a/test/sql/delegatesList.js +++ b/test/sql/delegatesList.js @@ -3,6 +3,9 @@ var DelegatesList = { generateDelegatesList: 'SELECT generateDelegatesList AS delegates FROM generateDelegatesList(${round}, ${delegates});', generateDelegatesListCast: 'SELECT generateDelegatesList AS delegates FROM generateDelegatesList(${round}, ${delegates}::text[]);', + getDelegatesList: 'SELECT getDelegatesList() AS list;', + getActiveDelegates: 'SELECT ARRAY(SELECT ENCODE(pk, \'hex\') AS pk FROM delegates ORDER BY rank ASC LIMIT 101) AS delegates', + getRound: 'SELECT CEIL((SELECT height+1 FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int AS round;' }; module.exports = DelegatesList; diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index 7edc12c44a7..9af3876f999 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -425,4 +425,27 @@ describe('DelegatesListSQL', function () { }); }); }); + + describe('checking SQL function getDelegatesList()', function () { + it('SQL getDelegatesList() results should be equal to generateDelegatesList() - real 101 delegates from current database', function (done) { + db.task(function (t) { + return t.batch([ + t.query(sql.getDelegatesList), + t.query(sql.getActiveDelegates), + t.query(sql.getRound) + ]); + }).then(function (res) { + var delegates_list = res[0][0].list, + active_delegates = res[1][0].delegates, + round = res[2][0].round; + + var expectedDelegates = generateDelegatesList(round.toString(), active_delegates); + expect(delegates_list).to.deep.equal(expectedDelegates); + + done(); + }).catch(function (err) { + done(err); + }); + }); + }); }); From 6ac28a5d3c38c77f8f249eee57bfd7f44124629c Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 14 Jul 2017 20:50:11 +0200 Subject: [PATCH 53/88] Reorganise pg-notify tests to prevent interrupt on working node --- test/sql/pgNotify.js | 6 +- test/unit/helpers/pg-notify.js | 169 ++++++++++++++++++--------------- 2 files changed, 98 insertions(+), 77 deletions(-) diff --git a/test/sql/pgNotify.js b/test/sql/pgNotify.js index 254cb38ca1b..96bf1268f66 100644 --- a/test/sql/pgNotify.js +++ b/test/sql/pgNotify.js @@ -2,8 +2,10 @@ var pgNotify = { interruptConnection: 'SELECT pg_terminate_backend(${pid});', - triggerNotify: 'SELECT pg_notify(${channel}, json_build_object(\'round\', CEIL((SELECT height FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int, \'list\', generateDelegatesList(CEIL((SELECT height FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int, ARRAY(SELECT ENCODE(pk, \'hex\') AS pk FROM delegates ORDER BY rank ASC LIMIT 101)))::text);', - triggerNotifyWithMessage: 'SELECT pg_notify(${channel}, ${message});' + triggerNotify: 'SELECT pg_notify(${channel}, json_build_object(\'round\', CEIL((SELECT height+1 FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int, \'list\', getDelegatesList())::text);', + triggerNotifyWithMessage: 'SELECT pg_notify(${channel}, ${message});', + getDelegatesList: 'SELECT getDelegatesList() AS list;', + getRound: 'SELECT CEIL((SELECT height+1 FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int AS round;' }; module.exports = pgNotify; diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 53419a72f79..636bbc46f6e 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -281,73 +281,21 @@ describe('helpers/pg-notify', function () { }); describe('onNotification', function () { - it('should notify about round-closed event (message generated by query)', function (done) { - var channel = 'round-closed'; - - // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { - expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); - expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); - expect(logger.warn.args[0]).to.be.undefined; - expect(logger.error.args[0]).to.be.undefined; - expect(bus.message.args[0][0]).to.equal('roundChanged'); - expect(bus.message.args[0][1].round).to.be.a('number'); - expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); - - done(); - }, 20)).catch(function (err) { + var delegates_list_db; + var round; + + before(function (done) { + // Get valid data from database, so we can compare notifications results with them + db.query(sql.getDelegatesList).then(function (result) { + delegates_list_db = result[0].list; + }).catch(function (err) { done(err); }); - }); - - it('should notify about round-closed event', function (done) { - var channel = 'round-closed'; - - // Execute query that trigger notify - db.query(sql.triggerNotifyWithMessage, {channel: channel, message: delegates_list_json}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: delegates_list_json}]); - expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); - expect(logger.warn.args[0]).to.be.undefined; - expect(logger.error.args[0]).to.be.undefined; - expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); + db.query(sql.getRound).then(function (result) { + round = result[0].round; done(); - }, 20)).catch(function (err) { - done(err); - }); - }); - - it('should notify about round-reopened event (message generated by query)', function (done) { - var channel = 'round-reopened'; - - // Execute query that trigger notify - db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { - expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); - expect(logger.info.args[0]).to.be.undefined; - expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); - expect(logger.error.args[0]).to.be.undefined; - expect(bus.message.args[0][0]).to.equal('roundChanged'); - expect(bus.message.args[0][1].round).to.be.a('number'); - expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); - - done(); - }, 20)).catch(function (err) { - done(err); - }); - }); - - it('should notify about round-reopened event', function (done) { - var channel = 'round-reopened'; - - // Execute query that trigger notify - db.query(sql.triggerNotifyWithMessage, {channel: channel, message: delegates_list_json}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: delegates_list_json}]); - expect(logger.info.args[0]).to.be.undefined; - expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); - expect(logger.error.args[0]).to.be.undefined; - expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); - done(); - }, 20)).catch(function (err) { + }).catch(function (err) { done(err); }); }); @@ -358,7 +306,7 @@ describe('helpers/pg-notify', function () { // Execute query that trigger notify db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: message}]); + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-closed', data: 'not_json'}]); expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); expect(logger.warn.args[0]).to.be.undefined; expect(logger.error.args[0][0]).to.equal('pg-notify: Unable to parse JSON'); @@ -375,7 +323,7 @@ describe('helpers/pg-notify', function () { // Execute query that trigger notify db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: message}]); + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: 'not_json'}]); expect(logger.info.args[0]).to.be.undefined; expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); expect(logger.error.args[0][0]).to.equal('pg-notify: Unable to parse JSON'); @@ -388,10 +336,9 @@ describe('helpers/pg-notify', function () { it('should not notify about unknown event', function (done) { var channel = 'unknown'; - var message = 'unknown'; // Execute query that trigger notify - db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { expect(logger.debug.args[0]).to.be.undefined; expect(logger.info.args[0]).to.be.undefined; expect(logger.warn.args[0]).to.be.undefined; @@ -405,15 +352,16 @@ describe('helpers/pg-notify', function () { it('should not notify about event on invalid channel, but log it', function (done) { var channel = 'round-reopened'; - var message = '123'; // Overwrite channels object with custom ones var restore = pg_notify.__set__('channels', {}); // Execute query that trigger notify - db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'round-reopened', data: '123'}]); - expect(logger.error.args[0]).to.deep.equal(['pg-notify: Invalid channel', 'round-reopened']); + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { + expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); + expect(logger.debug.args[0][1].channel).to.equal(channel); + expect(JSON.parse(logger.debug.args[0][1].data)).to.deep.equal({round: round, list: delegates_list_db}); + expect(logger.error.args[0]).to.deep.equal(['pg-notify: Invalid channel', channel]); expect(logger.info.args[0]).to.be.undefined; expect(logger.warn.args[0]).to.be.undefined; expect(bus.message.args[0]).to.be.undefined; @@ -426,7 +374,6 @@ describe('helpers/pg-notify', function () { it('should not notify about event on not supported channel, but log it', function (done) { var channel = 'test'; - var message = '123'; // Overwrite channels object with custom ones var restore = pg_notify.__set__('channels', {test: 'test'}); @@ -435,9 +382,11 @@ describe('helpers/pg-notify', function () { resetSpiesState(); // Execute query that trigger notify - db.query(sql.triggerNotifyWithMessage, {channel: channel, message: message}).then(setTimeout(function () { - expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: 'test', data: '123'}]); - expect(logger.error.args[0]).to.deep.equal(['pg-notify: Channel not supported', 'test']); + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { + expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); + expect(logger.debug.args[0][1].channel).to.equal(channel); + expect(JSON.parse(logger.debug.args[0][1].data)).to.deep.equal({round: round, list: delegates_list_db}); + expect(logger.error.args[0]).to.deep.equal(['pg-notify: Channel not supported', channel]); expect(logger.info.args[0]).to.be.undefined; expect(logger.warn.args[0]).to.be.undefined; expect(bus.message.args[0]).to.be.undefined; @@ -449,5 +398,75 @@ describe('helpers/pg-notify', function () { }); }); + it('should notify about round-reopened event', function (done) { + var channel = 'round-reopened'; + + // Execute query that trigger notify + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: delegates_list_json}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: channel, data: delegates_list_json}]); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should notify about round-closed event', function (done) { + var channel = 'round-closed'; + + // Execute query that trigger notify + db.query(sql.triggerNotifyWithMessage, {channel: channel, message: delegates_list_json}).then(setTimeout(function () { + expect(logger.debug.args[0]).to.deep.equal(['pg-notify: Notification received', {channel: channel, data: delegates_list_json}]); + expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); + expect(logger.warn.args[0]).to.be.undefined; + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); + + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should notify about round-reopened event (message generated by query)', function (done) { + var channel = 'round-reopened'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { + expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); + expect(logger.info.args[0]).to.be.undefined; + expect(logger.warn.args[0]).to.deep.equal(['pg-notify: Round reopened']); + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0][0]).to.equal('roundChanged'); + expect(bus.message.args[0][1].round).to.be.a('number'); + expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); + + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); + + it('should notify about round-closed event (message generated by query)', function (done) { + var channel = 'round-closed'; + + // Execute query that trigger notify + db.query(sql.triggerNotify, {channel: channel}).then(setTimeout(function () { + expect(logger.debug.args[0][0]).to.equal('pg-notify: Notification received'); + expect(logger.info.args[0]).to.deep.equal(['pg-notify: Round closed']); + expect(logger.warn.args[0]).to.be.undefined; + expect(logger.error.args[0]).to.be.undefined; + expect(bus.message.args[0][0]).to.equal('roundChanged'); + expect(bus.message.args[0][1].round).to.be.a('number'); + expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); + + done(); + }, 20)).catch(function (err) { + done(err); + }); + }); }); }); From bfb0e70fcdd35b9cc9758525eefed7b9a7924264 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sun, 16 Jul 2017 21:06:17 +0200 Subject: [PATCH 54/88] Fix DApps module initialisation --- modules/dapps.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dapps.js b/modules/dapps.js index 4f2ef8332ca..87206af9766 100644 --- a/modules/dapps.js +++ b/modules/dapps.js @@ -80,7 +80,7 @@ function DApps (cb, scope) { process.on('exit', function () { }); - return setImmediate(cb, null, self); + setImmediate(cb, null, self); } // Private methods From 6150efb09573f03113e89ebcfd9dfe4811a08945 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sat, 22 Jul 2017 03:35:21 +0200 Subject: [PATCH 55/88] Fix spelling and code standards --- README.md | 2 +- helpers/pg-notify.js | 4 ++-- helpers/slots.js | 4 ++-- modules/blocks/chain.js | 2 +- modules/cache.js | 2 +- modules/delegates.js | 4 +--- modules/loader.js | 22 +++++++++---------- .../20170508101337_generateDelegatesList.sql | 4 ++-- .../20170521001337_roundsRewrite.sql | 2 +- test/sql/delegatesList.js | 4 ++++ test/sql/pgNotify.js | 4 ++++ test/unit/helpers/jobs-queue.js | 8 +++---- test/unit/helpers/pg-notify.js | 20 ++++++++--------- test/unit/sql/delegatesList.js | 6 ++--- 14 files changed, 47 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 6d67efdd5f7..301de43a418 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Lisk is a next generation crypto-currency and decentralized application platform sudo -u postgres psql -d lisk_main -c "alter user "$USER" with password 'password';" ``` - **NOTE:** Database user require privileges to `CREATE EXTENSION pgcrypto`. + **NOTE:** Database user requires privileges to `CREATE EXTENSION pgcrypto`. - Bower () -- Bower helps to install required JavaScript dependencies. diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index fe2e1bed5bf..43e63b1cd94 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -21,7 +21,7 @@ function onNotification (data) { return; } - // Process round-releated things + // Process round-related things if (data.channel === 'round-closed') { logger.info('pg-notify: Round closed'); try { @@ -70,7 +70,7 @@ function setListeners (client, cb) { }); } -// Generate list of queries for unlisten to every supported channels +// Generate list of unlisten queries for every supported channel function unlistenQueries (t) { var queries = []; Object.keys(channels).forEach(function (channel) { diff --git a/helpers/slots.js b/helpers/slots.js index 8d93b037b54..d4f922ad35e 100644 --- a/helpers/slots.js +++ b/helpers/slots.js @@ -115,8 +115,8 @@ module.exports = { }, /** - * Calculate round number for supplied height - * @param {number} height Height from which calculate round + * Calculates round number from the given height + * @param {number} height Height from which round is calculated * @return {number} Round */ calcRound: function (height) { diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index d6ab28e96d1..104581fc877 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -58,7 +58,7 @@ Chain.prototype.saveGenesisBlock = function (cb) { if (!blockId) { // If there is no block with genesis ID - save to database // WARNING: DB_WRITE - // FIXME: That will fail if we already have genesis block in database, but with different ID + // FIXME: This will fail if we already have genesis block in database, but with different ID self.saveBlock(library.genesisblock.block, function (err) { return setImmediate(cb, err); }); diff --git a/modules/cache.js b/modules/cache.js index 917c3013dc3..e2bd56dfcd6 100644 --- a/modules/cache.js +++ b/modules/cache.js @@ -165,7 +165,7 @@ Cache.prototype.onNewBlock = function (block, broadcast, cb) { }; /** - * This function will be triggered when a round changed, it will clear all cache entires. + *This function will be triggered when round has changed, it will clear all cache entires. * @param {object} data Data received from postgres * @param {object} data.round Current round * @param {object} data.list Delegates list used for slot calculations diff --git a/modules/delegates.js b/modules/delegates.js index a759ebf7b88..06bef7d42c8 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -446,8 +446,6 @@ Delegates.prototype.fork = function (block, cause) { Delegates.prototype.validateBlockSlot = function (block, cb) { var currentSlot = slots.getSlotNumber(block.timestamp); var delegate_id = __private.delegatesList[currentSlot % slots.delegates]; - // var nextDelegate_id = __private.delegatesList[(currentSlot + 1) % slots.delegates]; - // var previousDelegate_id = __private.delegatesList[(currentSlot - 1) % slots.delegates]; if (delegate_id && block.generatorPublicKey === delegate_id) { return setImmediate(cb); @@ -479,7 +477,7 @@ Delegates.prototype.onBind = function (scope) { }; /** - * Trigger when get notification from postgres that round changed + * Triggered on receiving notification from postgres, indicating round has changed. * * @public * @method onRoundChanged diff --git a/modules/loader.js b/modules/loader.js index 0548e7b4903..481f705c74e 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -437,11 +437,11 @@ __private.loadBlockChain = function () { } library.db.task(checkMemTables).then(function (res) { - var countBlocks = res[0], - getGenesisBlock = res[1], - countMemAccounts = res[2], - validateMemBalances = res[3], - dbRoundsExceptions = res[4]; + var countBlocks = res[0]; + var getGenesisBlock = res[1]; + var countMemAccounts = res[2]; + var validateMemBalances = res[3]; + var dbRoundsExceptions = res[4]; var count = countBlocks.count; library.logger.info('Blocks ' + count); @@ -467,18 +467,18 @@ __private.loadBlockChain = function () { // } if (validateMemBalances.length) { - return reload(count, 'Memory balances doesn\'t match blockchain balances'); + return reload(count, 'Memory table balances do not match balances of blockchain'); } // Compare rounds exceptions with database layer var roundsExceptions = Object.keys(exceptions.rounds); if (roundsExceptions.length !== dbRoundsExceptions.length) { - throw 'Rounds exceptions count doesn\'t match database layer'; + throw 'Rounds exceptions count does not match database layer'; } else { dbRoundsExceptions.forEach(function (row) { var ex = exceptions.rounds[row.round]; if (!ex || ex.rewards_factor !== row.rewards_factor || ex.fees_factor !== row.fees_factor || ex.fees_bonus !== Number(row.fees_bonus)) { - throw 'Rounds exceptions values doesn\'t match database layer'; + throw 'Rounds exception values do not match database layer'; } }); } @@ -494,9 +494,9 @@ __private.loadBlockChain = function () { } library.db.task(updateMemAccounts).then(function (res) { - var updateMemAccounts = res[0], - getOrphanedMemAccounts = res[1], - getDelegates = res[2]; + var updateMemAccounts = res[0]; + var getOrphanedMemAccounts = res[1]; + var getDelegates = res[2]; if (getOrphanedMemAccounts.length > 0) { return reload(count, 'Detected orphaned blocks in mem_accounts'); diff --git a/sql/migrations/20170508101337_generateDelegatesList.sql b/sql/migrations/20170508101337_generateDelegatesList.sql index 41584d902f4..c2e5f491f52 100644 --- a/sql/migrations/20170508101337_generateDelegatesList.sql +++ b/sql/migrations/20170508101337_generateDelegatesList.sql @@ -50,7 +50,7 @@ BEGIN i := i + 1; END LOOP; - -- Return generated delagets list + -- Return generated delegates list RETURN delegates; END $$; @@ -66,7 +66,7 @@ BEGIN ARRAY(SELECT ENCODE(pk, 'hex') AS pk FROM delegates ORDER BY rank ASC LIMIT 101) ) INTO list; - -- Return generated delagets list + -- Return generated delegates list RETURN list; END $$; diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index b97ff20639d..0ee8871bb49 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -325,7 +325,7 @@ CREATE OR REPLACE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGS b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb FROM blocks b LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round - WHERE CEIL(height / 101::float)::int = CEIL(NEW.height / 101::float)::int + WHERE CEIL(b.height / 101::float)::int = CEIL(NEW.height / 101::float)::int ), -- Calculating total fees of round, apply exception fees bonus fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), diff --git a/test/sql/delegatesList.js b/test/sql/delegatesList.js index f2a1b852f9f..9ff70aab1ee 100644 --- a/test/sql/delegatesList.js +++ b/test/sql/delegatesList.js @@ -2,9 +2,13 @@ var DelegatesList = { generateDelegatesList: 'SELECT generateDelegatesList AS delegates FROM generateDelegatesList(${round}, ${delegates});', + generateDelegatesListCast: 'SELECT generateDelegatesList AS delegates FROM generateDelegatesList(${round}, ${delegates}::text[]);', + getDelegatesList: 'SELECT getDelegatesList() AS list;', + getActiveDelegates: 'SELECT ARRAY(SELECT ENCODE(pk, \'hex\') AS pk FROM delegates ORDER BY rank ASC LIMIT 101) AS delegates', + getRound: 'SELECT CEIL((SELECT height+1 FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int AS round;' }; diff --git a/test/sql/pgNotify.js b/test/sql/pgNotify.js index 96bf1268f66..14b2abaa89c 100644 --- a/test/sql/pgNotify.js +++ b/test/sql/pgNotify.js @@ -2,9 +2,13 @@ var pgNotify = { interruptConnection: 'SELECT pg_terminate_backend(${pid});', + triggerNotify: 'SELECT pg_notify(${channel}, json_build_object(\'round\', CEIL((SELECT height+1 FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int, \'list\', getDelegatesList())::text);', + triggerNotifyWithMessage: 'SELECT pg_notify(${channel}, ${message});', + getDelegatesList: 'SELECT getDelegatesList() AS list;', + getRound: 'SELECT CEIL((SELECT height+1 FROM blocks ORDER BY height DESC LIMIT 1) / 101::float)::int AS round;' }; diff --git a/test/unit/helpers/jobs-queue.js b/test/unit/helpers/jobs-queue.js index 3f67c44fec6..eba4c8b45a1 100644 --- a/test/unit/helpers/jobs-queue.js +++ b/test/unit/helpers/jobs-queue.js @@ -99,7 +99,7 @@ describe('helpers/jobsQueue', function () { testExecution(job, name, spy); }); - it('should throw an error imediatelly when try to register same job twice', function () { + it('should throw an error immediately when trying to register same job twice', function () { var name = 'job4'; var spy = sinon.spy(dummyFunction); var job = jobsQueue.register(name, spy, recallInterval); @@ -126,19 +126,19 @@ describe('helpers/jobsQueue', function () { expect(jobsQueuePeers).to.equal(jobsQueue); }); - it('should throw an error when try to pass job that is not a function', function () { + it('should throw an error when trying to pass job that is not a function', function () { expect(function () { jobsQueue.register('test_job', 'test', recallInterval); }).to.throw('Syntax error - invalid parameters supplied'); }); - it('should throw an error when try to pass name that is not a string', function () { + it('should throw an error when trying to pass name that is not a string', function () { expect(function () { jobsQueue.register(123, dummyFunction, recallInterval); }).to.throw('Syntax error - invalid parameters supplied'); }); - it('should throw an error when try to pass time that is not integer', function () { + it('should throw an error when trying to pass time that is not integer', function () { expect(function () { jobsQueue.register('test_job', dummyFunction, 0.22); }).to.throw('Syntax error - invalid parameters supplied'); diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 636bbc46f6e..bba84ddf632 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -86,7 +86,7 @@ describe('helpers/pg-notify', function () { }); describe('init', function () { - it('try to estabilish initial connection with valid params should succeed', function (done) { + it('should establish initial connection using valid params', function (done) { pg_notify.init(db, bus, logger, function (err) { // Should be no error expect(err).to.be.undefined; @@ -95,7 +95,7 @@ describe('helpers/pg-notify', function () { }); }); - it('try to estabilish initial connection with invalid params should fail after 1 retry', function (done) { + it('should fail (after 1 retry) to establish initial connection using invalid params', function (done) { pg_notify.init(invalid_db, bus, logger, function (err) { var err_msg = 'password authentication failed for user "invalidUser"'; // Error should propagate @@ -113,7 +113,7 @@ describe('helpers/pg-notify', function () { }); }); - it('try to estabilish initial connection with valid params but error during LISTEN queries should fail after 1 retry', function (done) { + it('should establish initial connection using valid params and fail (after 1 retry) if execution of LISTEN queries encounters error', function (done) { // Spy private functions var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); @@ -137,7 +137,7 @@ describe('helpers/pg-notify', function () { }); describe('setListeners', function () { - it('listeners should be set correctly after successfull connection', function (done) { + it('should set listeners correctly after successful connection', function (done) { // Spy private functions var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); @@ -150,7 +150,7 @@ describe('helpers/pg-notify', function () { done(); }); - it('should fail if error occurred during LISTEN queries', function (done) { + it('should fail if execution of LISTEN encounters error', function (done) { // Spy private functions var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); @@ -189,7 +189,7 @@ describe('helpers/pg-notify', function () { expect(logger.error.args[0][1].message).to.equal('terminating connection due to administrator command'); var errors = logger.error.args.slice(1, 11); - // Iterating over errors (failed retires) + // Iterating over errors (failed retries) for (var i = errors.length - 1; i >= 0; i--) { expect(errors[i][0]).to.equal('pg-notify: Error connecting'); expect(errors[i][1]).to.be.an('error'); @@ -199,7 +199,7 @@ describe('helpers/pg-notify', function () { // Last error - function should fail to reconnect expect(logger.error.args[11][0]).to.equal('pg-notify: Failed to reconnect - connection lost'); - //Connection should be cleared + // Connection should be cleared connection = pg_notify.__get__('connection'); expect(connection).to.be.an('null'); @@ -235,7 +235,7 @@ describe('helpers/pg-notify', function () { }); describe('removeListeners', function () { - it('listeners should be removed correctly', function (done) { + it('should remove listeners correctly', function (done) { // Spy private functions var removeListeners = pg_notify.__get__('removeListeners'); var connection = pg_notify.__get__('connection'); @@ -248,7 +248,7 @@ describe('helpers/pg-notify', function () { }); }); - it('listeners should be removed correctly even if error occurred during UNLISTEN queries', function (done) { + it('should remove listeners correctly even if error encountered executing UNLISTEN queries', function (done) { // Spy private functions var removeListeners = pg_notify.__get__('removeListeners'); var connection = pg_notify.__get__('connection'); @@ -263,7 +263,7 @@ describe('helpers/pg-notify', function () { }); }); - it('listeners should be removed correctly even if connection is null', function (done) { + it('should remove listeners correctly even if connection is null', function (done) { // Spy private functions var removeListeners = pg_notify.__get__('removeListeners'); var connection = pg_notify.__get__('connection'); diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index 9af3876f999..7db081a0938 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -435,9 +435,9 @@ describe('DelegatesListSQL', function () { t.query(sql.getRound) ]); }).then(function (res) { - var delegates_list = res[0][0].list, - active_delegates = res[1][0].delegates, - round = res[2][0].round; + var delegates_list = res[0][0].list; + var active_delegates = res[1][0].delegates; + var round = res[2][0].round; var expectedDelegates = generateDelegatesList(round.toString(), active_delegates); expect(delegates_list).to.deep.equal(expectedDelegates); From 5af4103d7a3651e3d185ce02aa834a4ec81b9f2d Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sat, 22 Jul 2017 21:59:48 +0200 Subject: [PATCH 56/88] Change vote_insert_delete trigger to deffered --- sql/migrations/20170521001337_roundsRewrite.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 0ee8871bb49..bdc1da74d7d 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -258,8 +258,9 @@ CREATE FUNCTION delegates_forged_blocks_cnt_update() RETURNS TRIGGER LANGUAGE PL END $$; -- Create trigger that will execute 'delegates_forged_blocks_cnt_update' after insertion or deletion of block -CREATE TRIGGER vote_insert_delete +CREATE CONSTRAINT TRIGGER vote_insert_delete AFTER INSERT OR DELETE ON blocks + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE PROCEDURE delegates_forged_blocks_cnt_update(); From 11eecabc1ca1f7dbbe7f4506593feb98d15c5f31 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sat, 22 Jul 2017 22:02:13 +0200 Subject: [PATCH 57/88] Add tests for rounds-related things - genesisBlock --- test/index.js | 1 + test/unit/sql/rounds.js | 310 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 test/unit/sql/rounds.js diff --git a/test/index.js b/test/index.js index 39b292440de..3b5aa021557 100644 --- a/test/index.js +++ b/test/index.js @@ -5,6 +5,7 @@ require('./unit/helpers/slots.js'); require('./unit/logic/blockReward.js'); require('./unit/sql/blockRewards.js'); require('./unit/sql/delegatesList.js'); +require('./unit/sql/rounds.js'); require('./unit/modules/peers.js'); require('./unit/modules/blocks.js'); diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js new file mode 100644 index 00000000000..70c97ea6c86 --- /dev/null +++ b/test/unit/sql/rounds.js @@ -0,0 +1,310 @@ +'use strict'; + +var _ = require('lodash'); +var async = require('async'); +var chai = require('chai'); +var expect = require('chai').expect; +var rewire = require('rewire'); +var sinon = require('sinon'); + +var config = require('../../../config.json'); +var Sequence = require('../../../helpers/sequence.js'); + +describe('Rounds-related SQL triggers', function () { + var db, logger, library, rewiredModules = {}, modules = []; + + before(function (done) { + // Init dummy connection with database - valid, used for tests here + var pgp = require('pg-promise')(); + config.db.user = config.db.user || process.env.USER; + db = pgp(config.db); + + // Clear tables + db.none('DELETE FROM blocks; DELETE FROM mem_accounts;').then(function (rows) { + done(); + }).catch(function (err) { + done(err); + }); + }); + + before(function (done) { + logger = { + trace: sinon.spy(), + debug: sinon.spy(), + info: sinon.spy(), + warn: sinon.spy(), + error: sinon.spy() + }; + + var modulesInit = { + accounts: '../../../modules/accounts.js', + transactions: '../../../modules/transactions.js', + blocks: '../../../modules/blocks.js', + signatures: '../../../modules/signatures.js', + // transport: '../../../modules/transport.js', + // loader: '../../../modules/loader.js', + // system: '../../../modules/system.js', + // peers: '../../../modules/peers.js', + delegates: '../../../modules/delegates.js', + multisignatures: '../../../modules/multisignatures.js', + dapps: '../../../modules/dapps.js', + crypto: '../../../modules/crypto.js', + // cache: '../../../modules/cache.js' + }; + + // Init limited application layer + async.auto({ + config: function (cb) { + cb(null, config); + }, + genesisblock: function (cb) { + var genesisblock = require('../../../genesisBlock.json'); + cb(null, {block: genesisblock}); + }, + + schema: function (cb) { + var z_schema = require('../../../helpers/z_schema.js'); + cb(null, new z_schema()); + }, + network: function (cb) { + cb(null, { + io: {sockets: {emit: sinon.spy()}}, + }); + }, + logger: function (cb) { + cb(null, logger); + }, + dbSequence: ['logger', function (scope, cb) { + var sequence = new Sequence({ + onWarning: function (current, limit) { + scope.logger.warn('DB queue', current); + } + }); + cb(null, sequence); + }], + sequence: ['logger', function (scope, cb) { + var sequence = new Sequence({ + onWarning: function (current, limit) { + scope.logger.warn('Main queue', current); + } + }); + cb(null, sequence); + }], + balancesSequence: ['logger', function (scope, cb) { + var sequence = new Sequence({ + onWarning: function (current, limit) { + scope.logger.warn('Balance queue', current); + } + }); + cb(null, sequence); + }], + ed: function (cb) { + cb(null, require('../../../helpers/ed.js')); + }, + + bus: ['ed', function (scope, cb) { + var changeCase = require('change-case'); + var bus = function () { + this.message = function () { + var args = []; + Array.prototype.push.apply(args, arguments); + var topic = args.shift(); + var eventName = 'on' + changeCase.pascalCase(topic); + + // executes the each module onBind function + modules.forEach(function (module) { + if (typeof(module[eventName]) === 'function') { + module[eventName].apply(module[eventName], args); + } + if (module.submodules) { + async.each(module.submodules, function (submodule) { + if (submodule && typeof(submodule[eventName]) === 'function') { + submodule[eventName].apply(submodule[eventName], args); + } + }); + } + }); + }; + }; + cb(null, new bus()); + }], + db: function (cb) { + cb(null, db); + }, + pg_notify: ['db', 'bus', 'logger', function (scope, cb) { + var pg_notify = require('../../../helpers/pg-notify.js'); + pg_notify.init(scope.db, scope.bus, scope.logger, cb); + }], + logic: ['db', 'bus', 'schema', 'genesisblock', function (scope, cb) { + var Transaction = require('../../../logic/transaction.js'); + var Block = require('../../../logic/block.js'); + var Account = require('../../../logic/account.js'); + var Peers = require('../../../logic/peers.js'); + + async.auto({ + bus: function (cb) { + cb(null, scope.bus); + }, + db: function (cb) { + cb(null, scope.db); + }, + ed: function (cb) { + cb(null, scope.ed); + }, + logger: function (cb) { + cb(null, scope.logger); + }, + schema: function (cb) { + cb(null, scope.schema); + }, + genesisblock: function (cb) { + cb(null, { + block: scope.genesisblock.block + }); + }, + account: ['db', 'bus', 'ed', 'schema', 'genesisblock', 'logger', function (scope, cb) { + new Account(scope.db, scope.schema, scope.logger, cb); + }], + transaction: ['db', 'bus', 'ed', 'schema', 'genesisblock', 'account', 'logger', function (scope, cb) { + new Transaction(scope.db, scope.ed, scope.schema, scope.genesisblock, scope.account, scope.logger, cb); + }], + block: ['db', 'bus', 'ed', 'schema', 'genesisblock', 'account', 'transaction', function (scope, cb) { + new Block(scope.ed, scope.schema, scope.transaction, cb); + }], + peers: ['logger', function (scope, cb) { + new Peers(scope.logger, cb); + }] + }, cb); + }], + modules: ['network', 'logger', 'bus', 'sequence', 'dbSequence', 'balancesSequence', 'db', 'logic', function (scope, cb) { + var tasks = {}; + Object.keys(modulesInit).forEach(function (name) { + tasks[name] = function (cb) { + var Instance = rewire(modulesInit[name]); + rewiredModules[name] = Instance; + var obj = new Instance(cb, scope); + modules.push(obj); + }; + }); + + async.parallel(tasks, function (err, results) { + cb(err, results); + }); + }], + ready: ['modules', 'bus', 'logic', function (scope, cb) { + scope.bus.message('bind', scope.modules); + scope.logic.transaction.bindModules(scope.modules); + scope.logic.peers.bindModules(scope.modules); + cb(); + }] + }, function (err, scope) { + library = scope; + done(err); + }); + }); + + describe('genesisBlock', function () { + var genesisBlock, delegatesList; + var genesisAccount; + var accounts; + + before(function() { + // Get genesis accounts address - should be senderId from first transaction + genesisAccount = library.genesisblock.block.transactions[0].senderId; + + // Get unique accounts from genesis block + accounts = _.reduce(library.genesisblock.block.transactions, function(accounts, tx) { + if (tx.senderId && accounts.indexOf(tx.senderId) === -1) { + accounts.push(tx.senderId); + } + if (tx.recipientId && accounts.indexOf(tx.recipientId) === -1) { + accounts.push(tx.recipientId); + } + return accounts; + }, []); + }) + + it('should not populate mem_accounts', function (done) { + db.query('SELECT * FROM mem_accounts').then(function (rows) { + expect(rows.length).to.equal(0); + done(); + }).catch(function (err) { + done(err); + }); + }); + + it('should load genesis block with transactions into database (native)', function (done) { + db.query('SELECT * FROM full_blocks_list WHERE b_height = 1').then(function (rows) { + genesisBlock = library.modules.blocks.utils.readDbRows(rows)[0]; + expect(genesisBlock.id).to.equal(library.genesisblock.block.id); + expect(genesisBlock.transactions.length).to.equal(library.genesisblock.block.transactions.length); + done(); + }).catch(function (err) { + done(err); + }); + }); + + it('should populate delegates table (native) and set data (trigger block_insert->delegates_update_on_block)', function (done) { + db.query('SELECT * FROM delegates').then(function (rows) { + _.each(rows, function (delegate) { + expect(delegate.tx_id).that.is.an('string'); + + // Search for that transaction in genesis block + var found = _.find(library.genesisblock.block.transactions, {id: delegate.tx_id}); + expect(found).to.be.an('object'); + + expect(delegate.name).to.equal(found.asset.delegate.username); + expect(delegate.address).to.equal(found.senderId); + expect(delegate.pk.toString('hex')).to.equal(found.senderPublicKey); + + // Data populated by trigger + expect(delegate.rank).that.is.an('number'); + expect(Number(delegate.voters_balance)).to.equal(10000000000000000); + expect(delegate.voters_cnt).to.equal(1); + expect(delegate.blocks_forged_cnt).to.equal(0); + expect(delegate.blocks_missed_cnt).to.equal(0); + }); + done(); + }).catch(function (err) { + done(err); + }); + }); + + it('should populate modules.delegates.__private.delegatesList with 101 public keys (pg-notify)', function () { + delegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(delegatesList.length).to.equal(101); + _.each(delegatesList, function (pk) { + // Search for that pk in genesis block + var found = _.find(library.genesisblock.block.transactions, {senderPublicKey: pk}); + expect(found).to.be.an('object'); + }) + }); + + it('should apply transactions of genesis block to mem_accounts (native)', function (done) { + library.modules.blocks.chain.applyGenesisBlock(genesisBlock, function (err) { + db.query('SELECT * FROM mem_accounts').then(function (rows) { + // Number of returned accounts should be equal to number of unique accounts in genesis block + expect(rows.length).to.equal(accounts.length); + + _.each(rows, function (account) { + account.balance = Number(account.balance); + if (account.address === genesisAccount) { + // Genesis account should have negative balance + expect(account.balance).to.be.below(0); + } else if (account.isDelegate) { + // Delegates accounts should have balances of 0 + expect(account.balance).to.be.equal(0); + } else { + // Other accounts (with funds) should have positive balance + expect(account.balance).to.be.above(0); + } + }); + done(); + }).catch(function (err) { + done(err); + }); + }) + }); + }); + +}); \ No newline at end of file From 29eb954499e3360a6aedf88d08f36f745a8e1fe2 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sat, 22 Jul 2017 22:48:54 +0200 Subject: [PATCH 58/88] Improve code readability, standards in tests --- test/unit/helpers/pg-notify.js | 52 +++++++++------------------------- test/unit/sql/blockRewards.js | 28 +++++------------- test/unit/sql/delegatesList.js | 12 ++------ test/unit/sql/rounds.js | 31 ++++++-------------- 4 files changed, 32 insertions(+), 91 deletions(-) diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index bba84ddf632..3a46c64b9f1 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -207,9 +207,7 @@ describe('helpers/pg-notify', function () { exit.restore(); done(); - }, 60000)).catch(function (err) { - done(err); - }); + }, 60000)).catch(done); }); }); @@ -228,9 +226,7 @@ describe('helpers/pg-notify', function () { expect(logger.info.args[0][0]).to.equal('pg-notify: Reconnected successfully'); done(); - }, 10000)).catch(function (err) { - done(err); - }); + }, 10000)).catch(done); }); }); @@ -288,16 +284,12 @@ describe('helpers/pg-notify', function () { // Get valid data from database, so we can compare notifications results with them db.query(sql.getDelegatesList).then(function (result) { delegates_list_db = result[0].list; - }).catch(function (err) { - done(err); - }); + }).catch(done); db.query(sql.getRound).then(function (result) { round = result[0].round; done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); it('should not notify about round-closed event when payload is not valid JSON', function (done) { @@ -312,9 +304,7 @@ describe('helpers/pg-notify', function () { expect(logger.error.args[0][0]).to.equal('pg-notify: Unable to parse JSON'); expect(bus.message.args[0]).to.be.undefined; done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should not notify about round-reopened event when payload is not valid JSON', function (done) { @@ -329,9 +319,7 @@ describe('helpers/pg-notify', function () { expect(logger.error.args[0][0]).to.equal('pg-notify: Unable to parse JSON'); expect(bus.message.args[0]).to.be.undefined; done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should not notify about unknown event', function (done) { @@ -345,9 +333,7 @@ describe('helpers/pg-notify', function () { expect(logger.error.args[0]).to.be.undefined; expect(bus.message.args[0]).to.be.undefined; done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should not notify about event on invalid channel, but log it', function (done) { @@ -367,9 +353,7 @@ describe('helpers/pg-notify', function () { expect(bus.message.args[0]).to.be.undefined; restore(); done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should not notify about event on not supported channel, but log it', function (done) { @@ -392,9 +376,7 @@ describe('helpers/pg-notify', function () { expect(bus.message.args[0]).to.be.undefined; restore(); done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); }); @@ -409,9 +391,7 @@ describe('helpers/pg-notify', function () { expect(logger.error.args[0]).to.be.undefined; expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should notify about round-closed event', function (done) { @@ -426,9 +406,7 @@ describe('helpers/pg-notify', function () { expect(bus.message.args[0]).to.deep.equal(['roundChanged', delegates_list]); done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should notify about round-reopened event (message generated by query)', function (done) { @@ -445,9 +423,7 @@ describe('helpers/pg-notify', function () { expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); it('should notify about round-closed event (message generated by query)', function (done) { @@ -464,9 +440,7 @@ describe('helpers/pg-notify', function () { expect(bus.message.args[0][1].list).to.be.an('array').and.lengthOf(slots.delegates); done(); - }, 20)).catch(function (err) { - done(err); - }); + }, 20)).catch(done); }); }); }); diff --git a/test/unit/sql/blockRewards.js b/test/unit/sql/blockRewards.js index f5f72f8e02e..e85585d1a40 100644 --- a/test/unit/sql/blockRewards.js +++ b/test/unit/sql/blockRewards.js @@ -19,9 +19,7 @@ function calcBlockReward (height, reward, done) { expect(Number(rows[0].reward)).to.equal(reward); } done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }; function calcSupply (height, supply, done) { @@ -35,9 +33,7 @@ function calcSupply (height, supply, done) { expect(Number(rows[0].supply)).to.equal(supply); } done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }; function calcSupply_test (height_start, height_end, expected_reward, done) { @@ -47,9 +43,7 @@ function calcSupply_test (height_start, height_end, expected_reward, done) { expect(rows[0]).to.be.an('object'); expect(rows[0].result).to.equal(true); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }; function calcSupply_test_fail (height_start, height_end, expected_reward, done) { @@ -59,9 +53,7 @@ function calcSupply_test_fail (height_start, height_end, expected_reward, done) expect(rows[0]).to.be.an('object'); expect(rows[0].result).to.equal(false); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }; function calcBlockReward_test (height_start, height_end, expected_reward, done) { @@ -71,9 +63,7 @@ function calcBlockReward_test (height_start, height_end, expected_reward, done) expect(rows[0]).to.be.an('object'); expect(Number(rows[0].result)).to.equal(0); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }; describe('BlockRewardsSQL', function () { @@ -110,9 +100,7 @@ describe('BlockRewardsSQL', function () { expect(Number(rows[0].milestones[3])).to.equal(constants.rewards.milestones[3]); expect(Number(rows[0].milestones[4])).to.equal(constants.rewards.milestones[4]); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); }); @@ -359,9 +347,7 @@ describe('BlockRewardsSQL', function () { expect(rows[0]).to.be.an('object'); expect(Number(rows[0].result)).to.equal(1000); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); }); diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index 7db081a0938..77b7bcf5880 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -95,9 +95,7 @@ describe('DelegatesListSQL', function () { expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); } done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); it('SQL generateDelegatesList() results should be equal to generateDelegatesList() - real 101 delegates, exact order', function (done) { @@ -316,9 +314,7 @@ describe('DelegatesListSQL', function () { expect(rows[0].delegates[99]).to.equal('e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f'); expect(rows[0].delegates[100]).to.equal('eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7'); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); it('SQL generateDelegatesList() should raise exception for round 0', function (done) { @@ -443,9 +439,7 @@ describe('DelegatesListSQL', function () { expect(delegates_list).to.deep.equal(expectedDelegates); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); }); }); diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 70c97ea6c86..dbefc3c6675 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -20,11 +20,7 @@ describe('Rounds-related SQL triggers', function () { db = pgp(config.db); // Clear tables - db.none('DELETE FROM blocks; DELETE FROM mem_accounts;').then(function (rows) { - done(); - }).catch(function (err) { - done(err); - }); + db.none('DELETE FROM blocks; DELETE FROM mem_accounts;').then(done).catch(done); }); before(function (done) { @@ -67,9 +63,8 @@ describe('Rounds-related SQL triggers', function () { cb(null, new z_schema()); }, network: function (cb) { - cb(null, { - io: {sockets: {emit: sinon.spy()}}, - }); + // Init with empty function + cb(null, {io: {sockets: {emit: function () {}}}}); }, logger: function (cb) { cb(null, logger); @@ -208,12 +203,12 @@ describe('Rounds-related SQL triggers', function () { var genesisAccount; var accounts; - before(function() { + before(function () { // Get genesis accounts address - should be senderId from first transaction genesisAccount = library.genesisblock.block.transactions[0].senderId; // Get unique accounts from genesis block - accounts = _.reduce(library.genesisblock.block.transactions, function(accounts, tx) { + accounts = _.reduce(library.genesisblock.block.transactions, function (accounts, tx) { if (tx.senderId && accounts.indexOf(tx.senderId) === -1) { accounts.push(tx.senderId); } @@ -228,9 +223,7 @@ describe('Rounds-related SQL triggers', function () { db.query('SELECT * FROM mem_accounts').then(function (rows) { expect(rows.length).to.equal(0); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); it('should load genesis block with transactions into database (native)', function (done) { @@ -239,9 +232,7 @@ describe('Rounds-related SQL triggers', function () { expect(genesisBlock.id).to.equal(library.genesisblock.block.id); expect(genesisBlock.transactions.length).to.equal(library.genesisblock.block.transactions.length); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); it('should populate delegates table (native) and set data (trigger block_insert->delegates_update_on_block)', function (done) { @@ -265,9 +256,7 @@ describe('Rounds-related SQL triggers', function () { expect(delegate.blocks_missed_cnt).to.equal(0); }); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }); it('should populate modules.delegates.__private.delegatesList with 101 public keys (pg-notify)', function () { @@ -300,9 +289,7 @@ describe('Rounds-related SQL triggers', function () { } }); done(); - }).catch(function (err) { - done(err); - }); + }).catch(done); }) }); }); From e77c4cb248f18971fb454f92a1cd6e97c8cd373a Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sun, 23 Jul 2017 02:51:11 +0200 Subject: [PATCH 59/88] Add manual forging control for rounds-related tests --- test/unit/sql/rounds.js | 182 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 172 insertions(+), 10 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index dbefc3c6675..31f435655ad 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -7,11 +7,25 @@ var expect = require('chai').expect; var rewire = require('rewire'); var sinon = require('sinon'); -var config = require('../../../config.json'); -var Sequence = require('../../../helpers/sequence.js'); +var config = require('../../../config.json'); +var node = require('../../node.js'); +var Sequence = require('../../../helpers/sequence.js'); +var slots = require('../../../helpers/slots.js'); describe('Rounds-related SQL triggers', function () { var db, logger, library, rewiredModules = {}, modules = []; + var mem_state; + + function normalizeMemAccounts (mem_accounts) { + var accounts = {}; + _.map(mem_accounts, function (acc) { + acc.balance = Number(acc.balance); + acc.u_balance = Number(acc.u_balance); + acc.fees = Number(acc.fees); + accounts[acc.address] = acc; + }); + return accounts; + } before(function (done) { // Init dummy connection with database - valid, used for tests here @@ -24,10 +38,14 @@ describe('Rounds-related SQL triggers', function () { }); before(function (done) { + // Set block time to 1 second, so we can forge valid block every second + slots.interval = 1; + logger = { trace: sinon.spy(), debug: sinon.spy(), info: sinon.spy(), + log: sinon.spy(), warn: sinon.spy(), error: sinon.spy() }; @@ -37,10 +55,10 @@ describe('Rounds-related SQL triggers', function () { transactions: '../../../modules/transactions.js', blocks: '../../../modules/blocks.js', signatures: '../../../modules/signatures.js', - // transport: '../../../modules/transport.js', - // loader: '../../../modules/loader.js', - // system: '../../../modules/system.js', - // peers: '../../../modules/peers.js', + transport: '../../../modules/transport.js', + loader: '../../../modules/loader.js', + system: '../../../modules/system.js', + peers: '../../../modules/peers.js', delegates: '../../../modules/delegates.js', multisignatures: '../../../modules/multisignatures.js', dapps: '../../../modules/dapps.js', @@ -177,7 +195,7 @@ describe('Rounds-related SQL triggers', function () { tasks[name] = function (cb) { var Instance = rewire(modulesInit[name]); rewiredModules[name] = Instance; - var obj = new Instance(cb, scope); + var obj = new rewiredModules[name](cb, scope); modules.push(obj); }; }); @@ -194,6 +212,8 @@ describe('Rounds-related SQL triggers', function () { }] }, function (err, scope) { library = scope; + // Overwrite onBlockchainReady function to prevent automatic forging + library.modules.delegates.onBlockchainReady = function () {}; done(err); }); }); @@ -221,6 +241,7 @@ describe('Rounds-related SQL triggers', function () { it('should not populate mem_accounts', function (done) { db.query('SELECT * FROM mem_accounts').then(function (rows) { + mem_state = normalizeMemAccounts(rows); expect(rows.length).to.equal(0); done(); }).catch(done); @@ -272,11 +293,11 @@ describe('Rounds-related SQL triggers', function () { it('should apply transactions of genesis block to mem_accounts (native)', function (done) { library.modules.blocks.chain.applyGenesisBlock(genesisBlock, function (err) { db.query('SELECT * FROM mem_accounts').then(function (rows) { + mem_state = normalizeMemAccounts(rows); // Number of returned accounts should be equal to number of unique accounts in genesis block expect(rows.length).to.equal(accounts.length); - _.each(rows, function (account) { - account.balance = Number(account.balance); + _.each(mem_state, function (account) { if (account.address === genesisAccount) { // Genesis account should have negative balance expect(account.balance).to.be.below(0); @@ -294,4 +315,145 @@ describe('Rounds-related SQL triggers', function () { }); }); -}); \ No newline at end of file + describe('round', function () { + var transactions = []; + + function addTransaction (transaction, cb) { + // Add transaction to transactions pool - we use shortcut here to bypass transport module, but logic is the same + // See: modules.transport.__private.receiveTransaction + transaction = library.logic.transaction.objectNormalize(transaction); + library.balancesSequence.add(function (sequenceCb) { + library.modules.transactions.processUnconfirmedTransaction(transaction, true, function (err) { + if (err) { + return setImmediate(sequenceCb, err.toString()); + } else { + return setImmediate(sequenceCb, null, transaction.id); + } + }); + }, cb); + } + + function forge (cb) { + var transactionPool = rewiredModules.transactions.__get__('__private.transactionPool'); + var forge = rewiredModules.delegates.__get__('__private.forge'); + + async.series([ + transactionPool.fillPool, + forge, + ], function (err) { + cb(err); + }); + } + + function addTransactionsAndForge (transactions, cb) { + async.waterfall([ + function addTransactions (waterCb) { + async.eachSeries(transactions, function (transaction, eachSeriesCb) { + addTransaction(transaction, eachSeriesCb); + }, waterCb); + }, + forge + ], function (err) { + cb(err); + }); + } + + function expectedMemState (transactions) { + _.each(transactions, function (tx) { + var block_id = library.modules.blocks.lastBlock.get().id; + + var address = tx.senderId + if (mem_state[address]) { + // Update sender + mem_state[address].balance -= (tx.fee+tx.amount); + mem_state[address].u_balance -= (tx.fee+tx.amount); + mem_state[address].blockId = block_id; + mem_state[address].virgin = 0; + } + + address = tx.recipientId; + if (mem_state[address]) { + // Update recipient + found = true; + mem_state[address].balance += tx.amount; + mem_state[address].u_balance += tx.amount; + mem_state[address].blockId = block_id; + } else { + // Funds sent to new account + mem_state[address] = { + address: address, + balance: tx.amount, + blockId: block_id, + delegates: null, + fees: 0, + isDelegate: 0, + missedblocks: 0, + multilifetime: 0, + multimin: 0, + multisignatures: null, + nameexist: 0, + producedblocks: 0, + publicKey: null, + rate: '0', + rewards: '0', + secondPublicKey: null, + secondSignature: 0, + u_balance: tx.amount, + u_delegates: null, + u_isDelegate: 0, + u_multilifetime: 0, + u_multimin: 0, + u_multisignatures: null, + u_nameexist: 0, + u_secondSignature: 0, + u_username: null, + username: null, + virgin: 1, + vote: '0' + } + } + }); + return mem_state; + } + + before(function () { + // Set delegates module as loaded to allow manual forging + rewiredModules.delegates.__set__('__private.loaded', true); + }); + + it('should load all secrets of 101 delegates and set modules.delegates.__private.keypairs (native)', function (done) { + var loadDelegates = rewiredModules.delegates.__get__('__private.loadDelegates'); + loadDelegates(function (err) { + if (err) { done(err); } + var keypairs = rewiredModules.delegates.__get__('__private.keypairs'); + expect(Object.keys(keypairs).length).to.equal(config.forging.secret.length); + _.each(keypairs, function (keypair, pk) { + expect(keypair.publicKey).to.be.instanceOf(Buffer); + expect(keypair.privateKey).to.be.instanceOf(Buffer); + expect(pk).to.equal(keypair.publicKey.toString('hex')); + }); + done(); + }); + }); + + it('should forge block with 1 TRANSFER transaction and update mem_accounts', function (done) { + var tx = node.lisk.transaction.createTransaction( + node.randomAccount().address, + node.randomNumber(100000000, 1000000000), + node.gAccount.password + ); + transactions.push(tx); + + addTransactionsAndForge(transactions, function (err) { + if (err) { done(err); } + var expected_mem_state = expectedMemState(transactions); + db.query('SELECT * FROM mem_accounts').then(function (rows) { + mem_state = normalizeMemAccounts(rows); + expect(mem_state).to.deep.equal(expected_mem_state); + done(); + }).catch(done); + }); + }); + + }); +}); From e48dd7907e11f32c6fa1515f5e40c4a99b4df5ea Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sun, 23 Jul 2017 03:55:03 +0200 Subject: [PATCH 60/88] Fix wrong trigger name --- sql/migrations/20170521001337_roundsRewrite.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index bdc1da74d7d..46a30868d19 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -258,7 +258,7 @@ CREATE FUNCTION delegates_forged_blocks_cnt_update() RETURNS TRIGGER LANGUAGE PL END $$; -- Create trigger that will execute 'delegates_forged_blocks_cnt_update' after insertion or deletion of block -CREATE CONSTRAINT TRIGGER vote_insert_delete +CREATE CONSTRAINT TRIGGER block_insert_delete AFTER INSERT OR DELETE ON blocks DEFERRABLE INITIALLY DEFERRED FOR EACH ROW From 70974e0f612189ad4feeff20e805d67cf4e3815e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sun, 23 Jul 2017 14:00:00 +0200 Subject: [PATCH 61/88] Do not calculate rewards for forger of genesis block --- sql/migrations/20170507001337_roundsRewards.sql | 4 ++-- sql/migrations/20170521001337_roundsRewrite.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/migrations/20170507001337_roundsRewards.sql b/sql/migrations/20170507001337_roundsRewards.sql index 4a2015a3297..9477fce5a6b 100644 --- a/sql/migrations/20170507001337_roundsRewards.sql +++ b/sql/migrations/20170507001337_roundsRewards.sql @@ -59,7 +59,7 @@ CREATE FUNCTION rounds_rewards_init() RETURNS void LANGUAGE PLPGSQL AS $$ b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb FROM blocks b LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round - WHERE CEIL(b.height / 101::float)::int = row.round + WHERE CEIL(b.height / 101::float)::int = row.round AND b.height > 1 ), -- Calculating total fees of round, apply exception fees bonus fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), @@ -115,7 +115,7 @@ CREATE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb FROM blocks b LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round - WHERE CEIL(height / 101::float)::int = CEIL(NEW.height / 101::float)::int + WHERE CEIL(height / 101::float)::int = CEIL(NEW.height / 101::float)::int AND b.height > 1 ), -- Calculating total fees of round, apply exception fees bonus fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 46a30868d19..71af8faa3c2 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -326,7 +326,7 @@ CREATE OR REPLACE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGS b.reward * COALESCE(e.rewards_factor, 1) AS reward, COALESCE(e.fees_bonus, 0) AS fb FROM blocks b LEFT JOIN rounds_exceptions e ON CEIL(b.height / 101::float)::int = e.round - WHERE CEIL(b.height / 101::float)::int = CEIL(NEW.height / 101::float)::int + WHERE CEIL(b.height / 101::float)::int = CEIL(NEW.height / 101::float)::int AND b.height > 1 ), -- Calculating total fees of round, apply exception fees bonus fees AS (SELECT SUM(fees) + fb AS total, FLOOR((SUM(fees) + fb) / 101) AS single FROM round GROUP BY fb), From 2011949c06cfe7aa76cbce4598cd8728b1d82e7c Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Sun, 23 Jul 2017 14:17:40 +0200 Subject: [PATCH 62/88] Add first round test to rounds-related tests --- test/unit/sql/rounds.js | 293 ++++++++++++++++++++++++++++++++++------ 1 file changed, 248 insertions(+), 45 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 31f435655ad..d31e6a9180d 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -6,15 +6,17 @@ var chai = require('chai'); var expect = require('chai').expect; var rewire = require('rewire'); var sinon = require('sinon'); +var Promise = require('bluebird'); -var config = require('../../../config.json'); -var node = require('../../node.js'); -var Sequence = require('../../../helpers/sequence.js'); -var slots = require('../../../helpers/slots.js'); +var config = require('../../../config.json'); +var node = require('../../node.js'); +var Sequence = require('../../../helpers/sequence.js'); +var slots = require('../../../helpers/slots.js'); +var bignum = require('../../../helpers/bignum.js'); describe('Rounds-related SQL triggers', function () { var db, logger, library, rewiredModules = {}, modules = []; - var mem_state; + var mem_state, delegates_state, round_blocks = []; function normalizeMemAccounts (mem_accounts) { var accounts = {}; @@ -27,14 +29,115 @@ describe('Rounds-related SQL triggers', function () { return accounts; } + function normalizeDelegates (db_delegates) { + var delegates = {}; + _.map(db_delegates, function (d) { + d.pk = d.pk.toString('hex'); + d.rank = Number(d.rank); + d.fees = Number(d.fees); + d.rewards = Number(d.rewards); + d.voters_balance = Number(d.voters_balance); + d.voters_cnt = Number(d.voters_cnt); + d.blocks_forged_cnt = Number(d.blocks_forged_cnt); + d.blocks_missed_cnt = Number(d.blocks_missed_cnt); + delegates[d.pk] = d; + }); + return delegates; + } + + function getMemAccounts () { + return db.query('SELECT * FROM mem_accounts').then(function (rows) { + rows = normalizeMemAccounts(rows); + mem_state = rows; + return rows; + }); + } + + function getDelegates (normalize) { + return db.query('SELECT * FROM delegates').then(function (rows) { + delegates_state = normalizeDelegates(rows); + return rows; + }); + } + + function getBlocks (round) { + return db.query('SELECT * FROM blocks WHERE CEIL(height / 101::float)::int = ${round} AND height > 1 ORDER BY height ASC', {round: round}).then(function (rows) { + return rows; + }); + } + + function getRoundRewards (round) { + return db.query('SELECT ENCODE(pk, \'hex\') AS pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = ${round} GROUP BY pk', {round: round}).then(function (rows) { + var rewards = {}; + _.each(rows, function (row) { + rewards[row.pk] = { + pk: row.pk, + fees: Number(row.fees), + rewards: Number(row.rewards) + }; + }); + return rewards; + }); + } + + function getExpectedRoundRewards (blocks) { + var rewards = {}; + + var feesTotal = _.reduce(blocks, function (fees, block) { + return new bignum(fees).plus(block.totalFee); + }, 0); + + var feesPerDelegate = new bignum(feesTotal.toPrecision(15)).dividedBy(slots.delegates).floor(); + var feesRemaining = new bignum(feesTotal.toPrecision(15)).minus(feesPerDelegate.times(slots.delegates)); + + node.debug(' Total fees: ' + feesTotal.toString() + ' Fees per delegates: ' + feesPerDelegate.toString() + ' Remaining fees: ' + feesRemaining); + + _.each(blocks, function (block, index) { + var pk = block.generatorPublicKey.toString('hex'); + if (rewards[pk]) { + rewards[pk].fees = rewards[pk].fees.plus(feesPerDelegate); + rewards[pk].rewards = rewards[pk].rewards.plus(block.reward); + } else { + rewards[pk] = { + pk: pk, + fees: new bignum(feesPerDelegate), + rewards: new bignum(block.reward) + }; + } + + if (index === blocks.length - 1) { + // Apply remaining fees to last delegate + rewards[pk].fees = rewards[pk].fees.plus(feesRemaining); + } + }); + + _.each(rewards, function (delegate) { + delegate.fees = Number(delegate.fees.toFixed()); + delegate.rewards = Number(delegate.rewards.toFixed()); + }); + + return rewards; + }; + before(function (done) { // Init dummy connection with database - valid, used for tests here - var pgp = require('pg-promise')(); + var options = { + promiseLib: Promise + }; + var pgp = require('pg-promise')(options); config.db.user = config.db.user || process.env.USER; db = pgp(config.db); // Clear tables - db.none('DELETE FROM blocks; DELETE FROM mem_accounts;').then(done).catch(done); + db.task(function (t) { + return t.batch([ + t.none('DELETE FROM blocks WHERE height > 1'), + t.none('DELETE FROM blocks'), + t.none('DELETE FROM mem_accounts') + ]); + }).then(function () { + done(); + }).catch(done); }); before(function (done) { @@ -221,14 +324,14 @@ describe('Rounds-related SQL triggers', function () { describe('genesisBlock', function () { var genesisBlock, delegatesList; var genesisAccount; - var accounts; + var genesisAccounts; before(function () { // Get genesis accounts address - should be senderId from first transaction genesisAccount = library.genesisblock.block.transactions[0].senderId; // Get unique accounts from genesis block - accounts = _.reduce(library.genesisblock.block.transactions, function (accounts, tx) { + genesisAccounts = _.reduce(library.genesisblock.block.transactions, function (accounts, tx) { if (tx.senderId && accounts.indexOf(tx.senderId) === -1) { accounts.push(tx.senderId); } @@ -239,12 +342,10 @@ describe('Rounds-related SQL triggers', function () { }, []); }) - it('should not populate mem_accounts', function (done) { - db.query('SELECT * FROM mem_accounts').then(function (rows) { - mem_state = normalizeMemAccounts(rows); - expect(rows.length).to.equal(0); - done(); - }).catch(done); + it('should not populate mem_accounts', function () { + return getMemAccounts().then(function (accounts) { + expect(Object.keys(accounts).length).to.equal(0); + }); }); it('should load genesis block with transactions into database (native)', function (done) { @@ -256,9 +357,9 @@ describe('Rounds-related SQL triggers', function () { }).catch(done); }); - it('should populate delegates table (native) and set data (trigger block_insert->delegates_update_on_block)', function (done) { - db.query('SELECT * FROM delegates').then(function (rows) { - _.each(rows, function (delegate) { + it('should populate delegates table (native) and set data (trigger block_insert)', function () { + return getDelegates().then(function () { + _.each(delegates_state, function (delegate) { expect(delegate.tx_id).that.is.an('string'); // Search for that transaction in genesis block @@ -267,17 +368,16 @@ describe('Rounds-related SQL triggers', function () { expect(delegate.name).to.equal(found.asset.delegate.username); expect(delegate.address).to.equal(found.senderId); - expect(delegate.pk.toString('hex')).to.equal(found.senderPublicKey); + expect(delegate.pk).to.equal(found.senderPublicKey); // Data populated by trigger expect(delegate.rank).that.is.an('number'); - expect(Number(delegate.voters_balance)).to.equal(10000000000000000); + expect(delegate.voters_balance).to.equal(10000000000000000); expect(delegate.voters_cnt).to.equal(1); expect(delegate.blocks_forged_cnt).to.equal(0); expect(delegate.blocks_missed_cnt).to.equal(0); }); - done(); - }).catch(done); + }); }); it('should populate modules.delegates.__private.delegatesList with 101 public keys (pg-notify)', function () { @@ -292,12 +392,13 @@ describe('Rounds-related SQL triggers', function () { it('should apply transactions of genesis block to mem_accounts (native)', function (done) { library.modules.blocks.chain.applyGenesisBlock(genesisBlock, function (err) { - db.query('SELECT * FROM mem_accounts').then(function (rows) { - mem_state = normalizeMemAccounts(rows); + if (err) { done(err); } + + getMemAccounts().then(function (accounts) { // Number of returned accounts should be equal to number of unique accounts in genesis block - expect(rows.length).to.equal(accounts.length); + expect(Object.keys(accounts).length).to.equal(genesisAccounts.length); - _.each(mem_state, function (account) { + _.each(accounts, function (account) { if (account.address === genesisAccount) { // Genesis account should have negative balance expect(account.balance).to.be.below(0); @@ -309,8 +410,7 @@ describe('Rounds-related SQL triggers', function () { expect(account.balance).to.be.above(0); } }); - done(); - }).catch(done); + }).then(done).catch(done); }) }); }); @@ -319,6 +419,7 @@ describe('Rounds-related SQL triggers', function () { var transactions = []; function addTransaction (transaction, cb) { + node.debug(' Add transaction ID: ' + transaction.id); // Add transaction to transactions pool - we use shortcut here to bypass transport module, but logic is the same // See: modules.transport.__private.receiveTransaction transaction = library.logic.transaction.objectNormalize(transaction); @@ -358,32 +459,58 @@ describe('Rounds-related SQL triggers', function () { }); } + function tickAndValidate (transactions) { + var last_block = library.modules.blocks.lastBlock.get(); + + return new Promise(function (resolve, reject) { + addTransactionsAndForge(transactions, function(err) { + if (err) { reject(err); } + resolve(); + }) + }) + .then(function () { + var new_block = library.modules.blocks.lastBlock.get(); + expect(new_block.id).to.not.equal(last_block.id); + last_block = new_block; + round_blocks.push(new_block); + }) + .then(getMemAccounts) + .then(function (accounts) { + var expected_mem_state = expectedMemState(transactions); + expect(accounts).to.deep.equal(expected_mem_state); + }) + .then(getDelegates) + .then(function () { + var expected_delegates_state = expectedDelegatesState(); + expect(delegates_state).to.deep.equal(expected_delegates_state); + }) + } + function expectedMemState (transactions) { _.each(transactions, function (tx) { - var block_id = library.modules.blocks.lastBlock.get().id; + var last_block = library.modules.blocks.lastBlock.get(); var address = tx.senderId if (mem_state[address]) { // Update sender mem_state[address].balance -= (tx.fee+tx.amount); mem_state[address].u_balance -= (tx.fee+tx.amount); - mem_state[address].blockId = block_id; + mem_state[address].blockId = last_block.id; mem_state[address].virgin = 0; } address = tx.recipientId; if (mem_state[address]) { // Update recipient - found = true; mem_state[address].balance += tx.amount; mem_state[address].u_balance += tx.amount; - mem_state[address].blockId = block_id; + mem_state[address].blockId = last_block.id; } else { // Funds sent to new account mem_state[address] = { address: address, balance: tx.amount, - blockId: block_id, + blockId: last_block.id, delegates: null, fees: 0, isDelegate: 0, @@ -416,6 +543,16 @@ describe('Rounds-related SQL triggers', function () { return mem_state; } + function expectedDelegatesState () { + var last_block = library.modules.blocks.lastBlock.get(); + _.each(delegates_state, function (delegate) { + if (delegate.pk === last_block.generatorPublicKey) { + delegate.blocks_forged_cnt += 1; + } + }); + return delegates_state; + } + before(function () { // Set delegates module as loaded to allow manual forging rewiredModules.delegates.__set__('__private.loaded', true); @@ -424,7 +561,6 @@ describe('Rounds-related SQL triggers', function () { it('should load all secrets of 101 delegates and set modules.delegates.__private.keypairs (native)', function (done) { var loadDelegates = rewiredModules.delegates.__get__('__private.loadDelegates'); loadDelegates(function (err) { - if (err) { done(err); } var keypairs = rewiredModules.delegates.__get__('__private.keypairs'); expect(Object.keys(keypairs).length).to.equal(config.forging.secret.length); _.each(keypairs, function (keypair, pk) { @@ -432,11 +568,11 @@ describe('Rounds-related SQL triggers', function () { expect(keypair.privateKey).to.be.instanceOf(Buffer); expect(pk).to.equal(keypair.publicKey.toString('hex')); }); - done(); + done(err); }); }); - it('should forge block with 1 TRANSFER transaction and update mem_accounts', function (done) { + it('should forge block with 1 TRANSFER transaction to random account, update mem_accounts (native) and delegates (trigger block_insert_delete) tables', function () { var tx = node.lisk.transaction.createTransaction( node.randomAccount().address, node.randomNumber(100000000, 1000000000), @@ -444,16 +580,83 @@ describe('Rounds-related SQL triggers', function () { ); transactions.push(tx); - addTransactionsAndForge(transactions, function (err) { - if (err) { done(err); } - var expected_mem_state = expectedMemState(transactions); - db.query('SELECT * FROM mem_accounts').then(function (rows) { - mem_state = normalizeMemAccounts(rows); - expect(mem_state).to.deep.equal(expected_mem_state); - done(); - }).catch(done); - }); + return tickAndValidate(transactions); + }); + + it('should forge block with 25 TRANSFER transactions to random accounts, update mem_accounts (native) and delegates (trigger block_insert_delete) tables', function () { + var tx_cnt = 25; + var transactions = []; + + for (var i = tx_cnt - 1; i >= 0; i--) { + var tx = node.lisk.transaction.createTransaction( + node.randomAccount().address, + node.randomNumber(100000000, 1000000000), + node.gAccount.password + ); + transactions.push(tx); + } + + return tickAndValidate(transactions); }); + it('should forge 98 blocks with 1 TRANSFER transaction each to random account, update mem_accounts (native) and delegates (trigger block_insert_delete) tables', function (done) { + var blocks_cnt = 98; + var blocks_processed = 0; + var tx_cnt = 1; + + async.doUntil(function (untilCb) { + ++blocks_processed; + var transactions = []; + for (var t = tx_cnt - 1; t >= 0; t--) { + var tx = node.lisk.transaction.createTransaction( + node.randomAccount().address, + node.randomNumber(100000000, 1000000000), + node.gAccount.password + ); + transactions.push(tx); + } + node.debug(' Processing block ' + blocks_processed + ' of ' + blocks_cnt + ' with ' + transactions.length + ' transactions'); + + // We need to wait 1 second between forge because of block time + setTimeout(function () { + tickAndValidate(transactions).then(untilCb).catch(untilCb); + }, 1000); + + }, function (err) { + return err || blocks_processed >= blocks_cnt; + }, done); + }); + + it('should calculate rewards for round 1 correctly - all should be the same (native, rounds_rewards, delegates)', function () { + var round = 1; + var expectedRewards; + + return getBlocks(round) + .then(function (blocks) { + expectedRewards = getExpectedRoundRewards(blocks); + }) + .then(function () { + return getRoundRewards(round); + }) + .then(function (rewards) { + expect(rewards).to.deep.equal(expectedRewards); + }) + .then(function () { + return getDelegates(); + }) + .reduce(function (delegates, d) { + if (d.fees > 0 || d.rewards > 0) { + delegates[d.pk] = { + pk: d.pk, + fees: Number(d.fees), + rewards: Number(d.rewards) + } + } + return delegates; + }, {}) + .then(function (delegates) { + expect(delegates).to.deep.equal(expectedRewards); + }); + }); }); }); From a89f367e20ab45fb728ac6194035992379401d0b Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 31 Jul 2017 09:57:53 +0200 Subject: [PATCH 63/88] Run SQL tests in sequence --- Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 77f86b1ea15..63c6d5677d8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -264,9 +264,15 @@ lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { "Unit - SQL" : { node('node-03'){ sh ''' - export TEST=test/unit/sql TEST_TYPE='UNIT' NODE_ENV='TEST' + export TEST=test/unit/sql/blockRewards.js TEST_TYPE='UNIT' NODE_ENV='TEST' cd "$(echo $WORKSPACE | cut -f 1 -d '@')" npm run jenkins + + export TEST=test/unit/sql/delegatesList.js TEST_TYPE='UNIT' NODE_ENV='TEST' + npm run jenkins + + export TEST=test/unit/sql/rounds.js TEST_TYPE='UNIT' NODE_ENV='TEST' + npm run jenkins ''' } }, From 4e60dd633f488c7b699403910d52f99c6399d888 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 31 Jul 2017 10:11:46 +0200 Subject: [PATCH 64/88] Reorder unit tests --- Jenkinsfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 63c6d5677d8..8fc08b2e929 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -233,15 +233,6 @@ lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { ''' } }, // End Node-02 Tests - "Unit - Helpers" : { - node('node-03'){ - sh ''' - export TEST=test/unit/helpers TEST_TYPE='UNIT' NODE_ENV='TEST' - cd "$(echo $WORKSPACE | cut -f 1 -d '@')" - npm run jenkins - ''' - } - }, "Unit - Modules" : { node('node-03'){ sh ''' @@ -276,6 +267,15 @@ lock(resource: "Lisk-Core-Nodes", inversePrecedence: true) { ''' } }, + "Unit - Helpers" : { + node('node-03'){ + sh ''' + export TEST=test/unit/helpers TEST_TYPE='UNIT' NODE_ENV='TEST' + cd "$(echo $WORKSPACE | cut -f 1 -d '@')" + npm run jenkins + ''' + } + }, "Unit - Logic" : { node('node-03'){ sh ''' From 7d02046fb5f3740635f01102e4eefad1fa4cc716 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 31 Jul 2017 14:10:22 +0200 Subject: [PATCH 65/88] Fix transaction unit test --- test/unit/logic/transaction.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/logic/transaction.js b/test/unit/logic/transaction.js index 90ffb13341e..b1d9efb6f89 100644 --- a/test/unit/logic/transaction.js +++ b/test/unit/logic/transaction.js @@ -132,7 +132,7 @@ describe('transaction', function () { var transactionLogic; var accountModule; - var attachTransferAsset = function (transactionLogic, accountLogic, rounds, done) { + var attachTransferAsset = function (transactionLogic, accountLogic, done) { modulesLoader.initModuleWithDb(AccountModule, function (err, __accountModule) { var transfer = new Transfer(); transfer.bind(__accountModule); From 216e4e3abc3e752432352baace1eb1e73ac638e6 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 31 Jul 2017 14:11:18 +0200 Subject: [PATCH 66/88] Init webSocket for rounds-related tests --- test/unit/sql/rounds.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index d31e6a9180d..5096e20f7fd 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -187,6 +187,19 @@ describe('Rounds-related SQL triggers', function () { // Init with empty function cb(null, {io: {sockets: {emit: function () {}}}}); }, + webSocket: ['config', 'logger', 'network', function (scope, cb) { + // Init with empty functions + var MasterWAMPServer = require('wamp-socket-cluster/MasterWAMPServer'); + + var dummySocketCluster = {on: function () {}}; + var dummyWAMPServer = new MasterWAMPServer(dummySocketCluster, {}); + var wsRPC = require('../../../api/ws/rpc/wsRPC.js').wsRPC; + + wsRPC.setServer(dummyWAMPServer); + wsRPC.getServer().registerRPCEndpoints({status: function () {}}); + + cb(); + }], logger: function (cb) { cb(null, logger); }, @@ -416,8 +429,6 @@ describe('Rounds-related SQL triggers', function () { }); describe('round', function () { - var transactions = []; - function addTransaction (transaction, cb) { node.debug(' Add transaction ID: ' + transaction.id); // Add transaction to transactions pool - we use shortcut here to bypass transport module, but logic is the same @@ -573,6 +584,7 @@ describe('Rounds-related SQL triggers', function () { }); it('should forge block with 1 TRANSFER transaction to random account, update mem_accounts (native) and delegates (trigger block_insert_delete) tables', function () { + var transactions = []; var tx = node.lisk.transaction.createTransaction( node.randomAccount().address, node.randomNumber(100000000, 1000000000), From b4310084189617a824eb9cff3db98ef73acb6356 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 31 Jul 2017 15:26:35 +0200 Subject: [PATCH 67/88] Wait 1 sec for network initialisation in rounds-related tests --- test/unit/sql/rounds.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 5096e20f7fd..bfb65b1af34 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -565,8 +565,11 @@ describe('Rounds-related SQL triggers', function () { } before(function () { - // Set delegates module as loaded to allow manual forging - rewiredModules.delegates.__set__('__private.loaded', true); + return new Promise(function (resolve) { + // Set delegates module as loaded to allow manual forging + rewiredModules.delegates.__set__('__private.loaded', true); + setTimeout(resolve, 1000); + }); }); it('should load all secrets of 101 delegates and set modules.delegates.__private.keypairs (native)', function (done) { From 0fb92c402960daa9cba87513d8c75245e84b01c8 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 00:40:05 +0200 Subject: [PATCH 68/88] Fix/improve comments, add test for last block of round deletion (type 0) --- app.js | 4 +- modules/blocks/chain.js | 2 +- modules/cache.js | 2 +- modules/loader.js | 2 +- .../20170521001337_roundsRewrite.sql | 4 +- test/unit/index.js | 7 +- test/unit/sql/rounds.js | 95 ++++++++++++++----- 7 files changed, 84 insertions(+), 32 deletions(-) diff --git a/app.js b/app.js index f6f0fea53fb..cc169d8f06d 100644 --- a/app.js +++ b/app.js @@ -394,7 +394,7 @@ d.run(function () { var topic = args.shift(); var eventName = 'on' + changeCase.pascalCase(topic); - // executes the each module onBind function + // Iterate over modules and execute event functions (on*) modules.forEach(function (module) { if (typeof(module[eventName]) === 'function') { module[eventName].apply(module[eventName], args); @@ -537,7 +537,9 @@ d.run(function () { }], ready: ['modules', 'bus', 'logic', function (scope, cb) { + // Fire onBind event in every module scope.bus.message('bind', scope.modules); + scope.logic.transaction.bindModules(scope.modules); scope.logic.peers.bindModules(scope.modules); cb(); diff --git a/modules/blocks/chain.js b/modules/blocks/chain.js index 104581fc877..e8e147d13eb 100644 --- a/modules/blocks/chain.js +++ b/modules/blocks/chain.js @@ -544,7 +544,7 @@ __private.popLastBlock = function (oldLastBlock, cb) { } // Delete last block from blockchain - // WARNING: Db_WRITE + // WARNING: DB_WRITE self.deleteBlock(oldLastBlock.id, function (err) { if (err) { // Fatal error, memory tables will be inconsistent diff --git a/modules/cache.js b/modules/cache.js index e2bd56dfcd6..6c312f5d961 100644 --- a/modules/cache.js +++ b/modules/cache.js @@ -165,7 +165,7 @@ Cache.prototype.onNewBlock = function (block, broadcast, cb) { }; /** - *This function will be triggered when round has changed, it will clear all cache entires. + * This function will be triggered when round has changed, it will clear all cache entries. * @param {object} data Data received from postgres * @param {object} data.round Current round * @param {object} data.list Delegates list used for slot calculations diff --git a/modules/loader.js b/modules/loader.js index 6ba25d4aad8..69db34af0fa 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -301,7 +301,7 @@ __private.loadTransactions = function (cb) { * @implements {modules.blocks.deleteAfterBlock} * @implements {modules.blocks.loadLastBlock} * @emits exit - * @throws {string} When fails to match genesis block with database or rounds exceptions doesn't match database + * @throws {string} On failure to match genesis block with database, or when rounds exceptions do not match database */ __private.loadBlockChain = function () { var offset = 0, limit = Number(library.config.loading.loadPerIteration) || 1000; diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 71af8faa3c2..79f788040e7 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -302,7 +302,7 @@ CREATE CONSTRAINT TRIGGER block_delete -- Replace function for deleting round rewards when last block of round is deleted CREATE OR REPLACE FUNCTION round_rewards_delete() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN - -- Update 'delagate' table with round rewards + -- Update 'delegates' table with round rewards WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(OLD.height / 101::float)::int) GROUP BY pk) UPDATE delegates SET rewards = delegates.rewards-r.rewards, fees = delegates.fees-r.fees FROM r WHERE delegates.pk = r.pk; @@ -351,7 +351,7 @@ CREATE OR REPLACE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGS -- Sort fees by block height ORDER BY round.height ASC; - -- Update 'delagate' table with round rewards + -- Update 'delegates' table with round rewards WITH r AS (SELECT pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = (CEIL(NEW.height / 101::float)::int) GROUP BY pk) UPDATE delegates SET rewards = delegates.rewards+r.rewards, fees = delegates.fees+r.fees FROM r WHERE delegates.pk = r.pk; diff --git a/test/unit/index.js b/test/unit/index.js index 30bcf961d05..1277a1e0ccd 100644 --- a/test/unit/index.js +++ b/test/unit/index.js @@ -1,7 +1,8 @@ require('./helpers/RPC'); -require('./helpers/RoundChanges'); require('./helpers/request-limiter.js'); require('./helpers/wsApi'); +require('./helpers/jobs-queue.js'); +require('./helpers/pg-notify.js'); require('./logic/blockReward.js'); require('./logic/peer'); @@ -14,7 +15,7 @@ require('./modules/blocks.js'); require('./modules/cache.js'); require('./modules/loader.js'); require('./modules/peers.js'); -require('./modules/rounds.js'); require('./sql/blockRewards.js'); - +require('./sql/delegatesList.js'); +require('./sql/rounds.js'); diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index bfb65b1af34..009f3ab5d46 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -17,6 +17,8 @@ var bignum = require('../../../helpers/bignum.js'); describe('Rounds-related SQL triggers', function () { var db, logger, library, rewiredModules = {}, modules = []; var mem_state, delegates_state, round_blocks = []; + var round_transactions = []; + var delegatesList; function normalizeMemAccounts (mem_accounts) { var accounts = {}; @@ -240,7 +242,7 @@ describe('Rounds-related SQL triggers', function () { var topic = args.shift(); var eventName = 'on' + changeCase.pascalCase(topic); - // executes the each module onBind function + // Iterate over modules and execute event functions (on*) modules.forEach(function (module) { if (typeof(module[eventName]) === 'function') { module[eventName].apply(module[eventName], args); @@ -321,7 +323,9 @@ describe('Rounds-related SQL triggers', function () { }); }], ready: ['modules', 'bus', 'logic', function (scope, cb) { + // Fire onBind event in every module scope.bus.message('bind', scope.modules); + scope.logic.transaction.bindModules(scope.modules); scope.logic.peers.bindModules(scope.modules); cb(); @@ -335,7 +339,7 @@ describe('Rounds-related SQL triggers', function () { }); describe('genesisBlock', function () { - var genesisBlock, delegatesList; + var genesisBlock; var genesisAccount; var genesisAccounts; @@ -429,11 +433,21 @@ describe('Rounds-related SQL triggers', function () { }); describe('round', function () { + var round_mem_acc, round_delegates; + + before(function () { + // Copy initial round states for later comparison + round_mem_acc = _.clone(mem_state); + round_delegates = _.clone(delegates_state); + }) + function addTransaction (transaction, cb) { node.debug(' Add transaction ID: ' + transaction.id); // Add transaction to transactions pool - we use shortcut here to bypass transport module, but logic is the same // See: modules.transport.__private.receiveTransaction transaction = library.logic.transaction.objectNormalize(transaction); + // Add transaction to round_transactions + round_transactions.push(transaction); library.balancesSequence.add(function (sequenceCb) { library.modules.transactions.processUnconfirmedTransaction(transaction, true, function (err) { if (err) { @@ -473,12 +487,7 @@ describe('Rounds-related SQL triggers', function () { function tickAndValidate (transactions) { var last_block = library.modules.blocks.lastBlock.get(); - return new Promise(function (resolve, reject) { - addTransactionsAndForge(transactions, function(err) { - if (err) { reject(err); } - resolve(); - }) - }) + return Promise.promisify(addTransactionsAndForge)(transactions) .then(function () { var new_block = library.modules.blocks.lastBlock.get(); expect(new_block.id).to.not.equal(last_block.id); @@ -494,7 +503,7 @@ describe('Rounds-related SQL triggers', function () { .then(function () { var expected_delegates_state = expectedDelegatesState(); expect(delegates_state).to.deep.equal(expected_delegates_state); - }) + }); } function expectedMemState (transactions) { @@ -646,21 +655,16 @@ describe('Rounds-related SQL triggers', function () { var round = 1; var expectedRewards; - return getBlocks(round) - .then(function (blocks) { - expectedRewards = getExpectedRoundRewards(blocks); - }) - .then(function () { - return getRoundRewards(round); - }) - .then(function (rewards) { - expect(rewards).to.deep.equal(expectedRewards); - }) - .then(function () { - return getDelegates(); - }) - .reduce(function (delegates, d) { + return Promise.join(getBlocks(round), getRoundRewards(round), getDelegates(), function (blocks, rewards, delegates) { + // Get expected rewards for round (native) + expectedRewards = getExpectedRoundRewards(blocks); + // Rewards from database table rounds_rewards should match native rewards + expect(rewards).to.deep.equal(expectedRewards); + + + return Promise.reduce(delegates, function (delegates, d) { if (d.fees > 0 || d.rewards > 0) { + // Normalize database data delegates[d.pk] = { pk: d.pk, fees: Number(d.fees), @@ -672,6 +676,51 @@ describe('Rounds-related SQL triggers', function () { .then(function (delegates) { expect(delegates).to.deep.equal(expectedRewards); }); + }); + }); + + it('delegates list should be different than one generated at the beginning of round 1', function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.not.deep.equal(delegatesList); + }); + + describe('Delete last block of round 1, block contain 1 transaction type SEND', function () { + var round = 1; + //var last_block = library.modules.blocks.lastBlock.get(); + + it('round rewards should be empty (rewards for round 1 deleted from rounds_rewards table)', function () { + return Promise.promisify(library.modules.blocks.chain.deleteLastBlock)().then(function () { + return getRoundRewards(round); + }).then(function (rewards) { + expect(rewards).to.deep.equal({}); + }); + }); + + it('delegates table should be equal to one generated at the beginning of round 1 with updated blocks_forged_cnt', function () { + return Promise.join(getDelegates(), getBlocks(round), function (delegates, blocks) { + // Apply blocks_forged_cnt to round_delegates + _.each(blocks, function (block) { + round_delegates[block.generatorPublicKey.toString('hex')].blocks_forged_cnt += 1; + }); + expect(delegates_state).to.deep.equal(round_delegates); + }); + }); + + it('mem_accounts table should not contain changes from transaction included in deleted block', function () { + return getMemAccounts() + .then(function (accounts) { + var last_transaction = round_transactions[round_transactions.length - 1]; + last_transaction.amount = -last_transaction.amount; + last_transaction.fees = -last_transaction.fee; + var expected_mem_state = expectedMemState([last_transaction]); + expect(accounts).to.deep.equal(expected_mem_state); + }); + }); + + it('delegates list should be equal to one generated at the beginning of round 1', function () { + var newDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(newDelegatesList).to.deep.equal(delegatesList); + }); }); }); }); From 22b61079529faed5924580652633f3ac2d744610 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 02:38:18 +0200 Subject: [PATCH 69/88] Use generateBlock instead of forge to speed up forging process --- test/unit/sql/rounds.js | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 009f3ab5d46..90153b654d4 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -18,7 +18,7 @@ describe('Rounds-related SQL triggers', function () { var db, logger, library, rewiredModules = {}, modules = []; var mem_state, delegates_state, round_blocks = []; var round_transactions = []; - var delegatesList; + var delegatesList, keypairs; function normalizeMemAccounts (mem_accounts) { var accounts = {}; @@ -143,9 +143,6 @@ describe('Rounds-related SQL triggers', function () { }); before(function (done) { - // Set block time to 1 second, so we can forge valid block every second - slots.interval = 1; - logger = { trace: sinon.spy(), debug: sinon.spy(), @@ -459,13 +456,32 @@ describe('Rounds-related SQL triggers', function () { }, cb); } + function getNextForger(offset) { + offset = !offset ? 1 : offset; + + var last_block = library.modules.blocks.lastBlock.get(); + var slot = slots.getSlotNumber(last_block.timestamp); + return delegatesList[(slot + offset) % slots.delegates]; + } + function forge (cb) { var transactionPool = rewiredModules.transactions.__get__('__private.transactionPool'); - var forge = rewiredModules.delegates.__get__('__private.forge'); async.series([ transactionPool.fillPool, - forge, + function (seriesCb) { + var last_block = library.modules.blocks.lastBlock.get(); + var slot = slots.getSlotNumber(last_block.timestamp) + 1; + var delegate = getNextForger(); + var keypair = keypairs[delegate]; + node.debug(' Last block height: ' + last_block.height + ' Last block ID: ' + last_block.id + ' Last block timestamp: ' + last_block.timestamp + ' Next slot: ' + slot + ' Next delegate PK: ' + delegate + ' Next block timestamp: ' + slots.getSlotTime(slot)); + library.modules.blocks.process.generateBlock(keypair, slots.getSlotTime(slot), function (err) { + if (err) { return seriesCb(err); } + last_block = library.modules.blocks.lastBlock.get(); + node.debug(' New last block height: ' + last_block.height + ' New last block ID: ' + last_block.id); + return seriesCb(err); + }); + } ], function (err) { cb(err); }); @@ -584,7 +600,7 @@ describe('Rounds-related SQL triggers', function () { it('should load all secrets of 101 delegates and set modules.delegates.__private.keypairs (native)', function (done) { var loadDelegates = rewiredModules.delegates.__get__('__private.loadDelegates'); loadDelegates(function (err) { - var keypairs = rewiredModules.delegates.__get__('__private.keypairs'); + keypairs = rewiredModules.delegates.__get__('__private.keypairs'); expect(Object.keys(keypairs).length).to.equal(config.forging.secret.length); _.each(keypairs, function (keypair, pk) { expect(keypair.publicKey).to.be.instanceOf(Buffer); @@ -641,11 +657,7 @@ describe('Rounds-related SQL triggers', function () { } node.debug(' Processing block ' + blocks_processed + ' of ' + blocks_cnt + ' with ' + transactions.length + ' transactions'); - // We need to wait 1 second between forge because of block time - setTimeout(function () { - tickAndValidate(transactions).then(untilCb).catch(untilCb); - }, 1000); - + tickAndValidate(transactions).then(untilCb).catch(untilCb); }, function (err) { return err || blocks_processed >= blocks_cnt; }, done); From 196ad488086e24b19a843d8ebc228e65684753dc Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 06:09:28 +0200 Subject: [PATCH 70/88] Base voters calculations on block height instead of round --- .../20170521001337_roundsRewrite.sql | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 79f788040e7..4363071959b 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -52,14 +52,14 @@ CREATE TABLE IF NOT EXISTS "votes_details"( "voter_address" VARCHAR(22) NOT NULL, "type" VARCHAR(3) NOT NULL, "timestamp" INT NOT NULL, - "round" INT NOT NULL, + "height" INT NOT NULL, "delegate_pk" BYTEA REFERENCES delegates(pk) ON DELETE CASCADE ); -- Populate 'votes_details' table from blockchain ('votes', 'trs', 'blocks') INSERT INTO votes_details -SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'add' ELSE 'rem' END) AS type, r.timestamp, r.round, DECODE(substring(vote, 2), 'hex') AS delegate_pk FROM ( - SELECT v."transactionId" AS tx_id, t."senderId" AS voter_address, b.timestamp AS timestamp, CEIL(b.height / 101::float)::int AS round, regexp_split_to_table(v.votes, ',') AS vote +SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'add' ELSE 'rem' END) AS type, r.timestamp, r.height, DECODE(substring(vote, 2), 'hex') AS delegate_pk FROM ( + SELECT v."transactionId" AS tx_id, t."senderId" AS voter_address, b.timestamp AS timestamp, b.height, regexp_split_to_table(v.votes, ',') AS vote FROM votes v, trs t, blocks b WHERE v."transactionId" = t.id AND b.id = t."blockId" ) AS r ORDER BY r.timestamp ASC; @@ -67,8 +67,8 @@ SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'ad CREATE FUNCTION vote_insert() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN INSERT INTO votes_details - SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'add' ELSE 'rem' END) AS type, r.timestamp, r.round, DECODE(substring(vote, 2), 'hex') AS delegate_pk FROM ( - SELECT v."transactionId" AS tx_id, t."senderId" AS voter_address, b.timestamp AS timestamp, CEIL(b.height / 101::float)::int AS round, regexp_split_to_table(v.votes, ',') AS vote + SELECT r.tx_id, r.voter_address, (CASE WHEN substring(vote, 1, 1) = '+' THEN 'add' ELSE 'rem' END) AS type, r.timestamp, r.height, DECODE(substring(vote, 2), 'hex') AS delegate_pk FROM ( + SELECT v."transactionId" AS tx_id, t."senderId" AS voter_address, b.timestamp AS timestamp, b.height, regexp_split_to_table(v.votes, ',') AS vote FROM votes v, trs t, blocks b WHERE v."transactionId" = NEW."transactionId" AND v."transactionId" = t.id AND b.id = t."blockId" ) AS r ORDER BY r.timestamp ASC; RETURN NULL; @@ -83,7 +83,7 @@ CREATE TRIGGER vote_insert -- Create indexes on 'votes_details' CREATE INDEX votes_details_voter_address ON votes_details(voter_address); CREATE INDEX votes_details_type ON votes_details(type); -CREATE INDEX votes_details_round ON votes_details(round); +CREATE INDEX votes_details_height ON votes_details(height); CREATE INDEX votes_details_sort ON votes_details(voter_address ASC, timestamp DESC); CREATE INDEX votes_details_dpk ON votes_details(delegate_pk); @@ -92,13 +92,13 @@ CREATE FUNCTION delegates_voters_cnt_update() RETURNS TABLE(updated INT) LANGUAG BEGIN RETURN QUERY WITH - last_round AS (SELECT CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 OR height = 1 ORDER BY height DESC LIMIT 1), + last_round AS (SELECT (CASE WHEN height < 101 THEN 1 ELSE height END) AS height FROM blocks WHERE height % 101 = 0 OR height = 1 ORDER BY height DESC LIMIT 1), updated AS (UPDATE delegates SET voters_cnt = cnt FROM (SELECT d.pk, (SELECT COUNT(1) AS cnt FROM (SELECT DISTINCT ON (voter_address) voter_address, delegate_pk, type FROM votes_details - WHERE delegate_pk = d.pk AND round <= (SELECT round FROM last_round) + WHERE delegate_pk = d.pk AND height <= (SELECT height FROM last_round) ORDER BY voter_address, timestamp DESC ) v WHERE type = 'add' ) FROM delegates d @@ -113,7 +113,8 @@ SELECT delegates_voters_cnt_update(); CREATE FUNCTION delegates_voters_balance_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ BEGIN RETURN QUERY - WITH last_round AS (SELECT height, CEIL(height / 101::float)::int AS round FROM blocks WHERE height % 101 = 0 OR height = 1 ORDER BY height DESC LIMIT 1), + WITH + last_round AS (SELECT (CASE WHEN height < 101 THEN 1 ELSE height END) AS height FROM blocks WHERE height % 101 = 0 OR height = 1 ORDER BY height DESC LIMIT 1), current_round_txs AS (SELECT t.id FROM trs t LEFT JOIN blocks b ON b.id = t."blockId" WHERE b.height > (SELECT height FROM last_round)), voters AS (SELECT DISTINCT ON (voter_address) voter_address FROM votes_details), balances AS ( @@ -134,7 +135,7 @@ CREATE FUNCTION delegates_voters_balance_update() RETURNS TABLE(updated INT) LAN (SELECT COALESCE(SUM(balance), 0) AS balance FROM accounts WHERE address IN (SELECT v.voter_address FROM (SELECT DISTINCT ON (voter_address) voter_address, type FROM votes_details - WHERE delegate_pk = d.pk AND round <= (SELECT round FROM last_round) + WHERE delegate_pk = d.pk AND height <= (SELECT height FROM last_round) ORDER BY voter_address, timestamp DESC ) v WHERE v.type = 'add' From b33de41709f93eb9d5da80dff7d50ea661413bf6 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 06:31:45 +0200 Subject: [PATCH 71/88] Add tests for rollback when forger of last block of round is unvoted --- test/unit/sql/rounds.js | 103 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 7 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 90153b654d4..d9ce8b1642e 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -62,6 +62,12 @@ describe('Rounds-related SQL triggers', function () { }); } + function getFullBlock(height) { + return db.query('SELECT * FROM full_blocks_list WHERE b_height = ${height}', {height: height}).then(function (rows) { + return rows; + }); + } + function getBlocks (round) { return db.query('SELECT * FROM blocks WHERE CEIL(height / 101::float)::int = ${round} AND height > 1 ORDER BY height ASC', {round: round}).then(function (rows) { return rows; @@ -404,10 +410,9 @@ describe('Rounds-related SQL triggers', function () { }) }); - it('should apply transactions of genesis block to mem_accounts (native)', function (done) { - library.modules.blocks.chain.applyGenesisBlock(genesisBlock, function (err) { - if (err) { done(err); } - + it('transactions of genesis block should be applied to mem_accounts (native)', function (done) { + // Wait 10 seconds for proper initialisation + setTimeout(function () { getMemAccounts().then(function (accounts) { // Number of returned accounts should be equal to number of unique accounts in genesis block expect(Object.keys(accounts).length).to.equal(genesisAccounts.length); @@ -425,17 +430,20 @@ describe('Rounds-related SQL triggers', function () { } }); }).then(done).catch(done); - }) + }, 10000); }); }); describe('round', function () { var round_mem_acc, round_delegates; + var deleteLastBlockPromise; before(function () { // Copy initial round states for later comparison round_mem_acc = _.clone(mem_state); round_delegates = _.clone(delegates_state); + + deleteLastBlockPromise = Promise.promisify(library.modules.blocks.chain.deleteLastBlock); }) function addTransaction (transaction, cb) { @@ -461,7 +469,7 @@ describe('Rounds-related SQL triggers', function () { var last_block = library.modules.blocks.lastBlock.get(); var slot = slots.getSlotNumber(last_block.timestamp); - return delegatesList[(slot + offset) % slots.delegates]; + return rewiredModules.delegates.__get__('__private.delegatesList')[(slot + offset) % slots.delegates]; } function forge (cb) { @@ -701,7 +709,7 @@ describe('Rounds-related SQL triggers', function () { //var last_block = library.modules.blocks.lastBlock.get(); it('round rewards should be empty (rewards for round 1 deleted from rounds_rewards table)', function () { - return Promise.promisify(library.modules.blocks.chain.deleteLastBlock)().then(function () { + return deleteLastBlockPromise().then(function () { return getRoundRewards(round); }).then(function (rewards) { expect(rewards).to.deep.equal({}); @@ -734,5 +742,86 @@ describe('Rounds-related SQL triggers', function () { expect(newDelegatesList).to.deep.equal(delegatesList); }); }); + + describe('Round rollback (table delegates) when forger of last block of round is unvoted', function() { + var last_block_forger; + + before(function () { + // Set last block forger + last_block_forger = getNextForger(); + // Delete one block more + return deleteLastBlockPromise(); + }); + + it('we should be on height 99 after delete one more block', function () { + var last_block = library.modules.blocks.lastBlock.get(); + expect(last_block.height).to.equal(99); + }); + + it('expected forger of last block of round should have proper votes', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[last_block_forger]; + expect(delegate.voters_balance).to.equal(10000000000000000); + expect(delegate.voters_cnt).to.equal(1); + }); + }); + + it('should unvote expected forger of last block of round', function () { + var transactions = []; + var tx = node.lisk.vote.createVote( + node.gAccount.password, + ['-' + last_block_forger] + ); + transactions.push(tx); + + return tickAndValidate(transactions) + .then(function () { + var last_block = library.modules.blocks.lastBlock.get(); + return getFullBlock(last_block.height); + }) + .then(function (rows) { + // Normalize blocks + var blocks = library.modules.blocks.utils.readDbRows(rows); + expect(blocks[0].transactions[0].asset.votes[0]).to.equal('-' + last_block_forger); + }); + }); + + it('finish round, delegates list should be different than one generated at the beginning of round 1', function () { + var transactions = []; + + return tickAndValidate(transactions) + .then(function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.not.deep.equal(delegatesList); + }); + }); + + it('forger of last block of previous round should have voters_balance and voters_cnt 0', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[last_block_forger]; + expect(delegate.voters_balance).to.equal(0); + expect(delegate.voters_cnt).to.equal(0); + }); + }); + + it('delete last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { + return deleteLastBlockPromise() + .then(function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.deep.equal(delegatesList); + }); + }); + + it('expected forger of last block of round should have proper votes again', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[last_block_forger]; + expect(delegate.voters_balance).to.equal(10000000000000000); + expect(delegate.voters_cnt).to.equal(1); + }); + }); + }); }); }); From 3898dc3af1587be665c588008410dad3dd1ce3cf Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 07:29:35 +0200 Subject: [PATCH 72/88] Exec validateMemBalances after every tests --- test/unit/sql/rounds.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index d9ce8b1642e..6810997cbc1 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -47,6 +47,14 @@ describe('Rounds-related SQL triggers', function () { return delegates; } + afterEach(function () { + // Perform validation of mem_accounts balances against blockchain after every test + return validateMemBalances() + .then(function (results) { + expect(results.length).to.equal(0); + }); + }); + function getMemAccounts () { return db.query('SELECT * FROM mem_accounts').then(function (rows) { rows = normalizeMemAccounts(rows); @@ -74,6 +82,12 @@ describe('Rounds-related SQL triggers', function () { }); } + function validateMemBalances () { + return db.query('SELECT * FROM validateMemBalances()').then(function (rows) { + return rows; + }); + } + function getRoundRewards (round) { return db.query('SELECT ENCODE(pk, \'hex\') AS pk, SUM(fees) AS fees, SUM(reward) AS rewards FROM rounds_rewards WHERE round = ${round} GROUP BY pk', {round: round}).then(function (rows) { var rewards = {}; From b9e694d797aa4d89f4e1ca1fad3b1e7c33c6f0dc Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 08:34:39 +0200 Subject: [PATCH 73/88] Add test for round rollback replace last block forger at last block --- test/unit/sql/rounds.js | 113 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 6810997cbc1..e4cc2a475c7 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -837,5 +837,118 @@ describe('Rounds-related SQL triggers', function () { }); }); }); + + describe('Round rollback (table delegates) when forger of last block of round is replaced in last block of round', function() { + var last_block_forger, tmp_account; + + before(function () { + // Set last block forger + last_block_forger = getNextForger(); + // Delete two blocks more + return deleteLastBlockPromise() + .then(function () { + return deleteLastBlockPromise(); + }) + .then(function () { + // Fund random account + var transactions = []; + tmp_account = node.randomAccount(); + var tx = node.lisk.transaction.createTransaction(tmp_account.address, 5000000000, node.gAccount.password); + transactions.push(tx); + return tickAndValidate(transactions); + }) + .then(function () { + // Register random delegate + var transactions = []; + var tx = node.lisk.delegate.createDelegate(tmp_account.password, 'my_little_delegate'); + transactions.push(tx); + return tickAndValidate(transactions); + }); + }); + + it('we should be on height 100', function () { + var last_block = library.modules.blocks.lastBlock.get(); + expect(last_block.height).to.equal(100); + }); + + it('finish round, should unvote expected forger of last block of round and vote new delegate', function () { + var transactions = []; + var tx = node.lisk.vote.createVote( + node.gAccount.password, + ['-' + last_block_forger, '+' + tmp_account.publicKey] + ); + transactions.push(tx); + + return tickAndValidate(transactions) + .then(function () { + var last_block = library.modules.blocks.lastBlock.get(); + return getFullBlock(last_block.height); + }) + .then(function (rows) { + // Normalize blocks + var blocks = library.modules.blocks.utils.readDbRows(rows); + expect(blocks[0].transactions[0].asset.votes).to.deep.equal(['-' + last_block_forger, '+' + tmp_account.publicKey]); + }); + }); + + it('delegates list should be different than one generated at the beginning of round 1', function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.not.deep.equal(delegatesList); + }); + + it('unvoted delegate should not be on list', function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.not.contain(last_block_forger); + }); + + it('delegate who replaced unvoted one should be on list', function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.contain(tmp_account.publicKey); + }); + + it('forger of last block of previous round should have voters_balance and voters_cnt 0', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[last_block_forger]; + expect(delegate.voters_balance).to.equal(0); + expect(delegate.voters_cnt).to.equal(0); + }); + }); + + it('delegate who replaced last block forger should have proper votes', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[tmp_account.publicKey]; + expect(delegate.voters_balance).to.be.above(0); + expect(delegate.voters_cnt).to.equal(1); + }); + }); + + it('delete last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { + return deleteLastBlockPromise() + .then(function () { + var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); + expect(tmpDelegatesList).to.deep.equal(delegatesList); + }); + }); + + it('expected forger of last block of round should have proper votes again', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[last_block_forger]; + expect(delegate.voters_balance).to.equal(10000000000000000); + expect(delegate.voters_cnt).to.equal(1); + }); + }); + + it('delegate who replaced last block forger should have voters_balance and voters_cnt 0', function () { + return getDelegates() + .then(function () { + var delegate = delegates_state[tmp_account.publicKey]; + expect(delegate.voters_balance).to.equal(0); + expect(delegate.voters_cnt).to.equal(0); + }); + }); + }); }); }); From 19ff1f554c603986866b7a58bf86aee133feb1a1 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 10:11:40 +0200 Subject: [PATCH 74/88] Replace setTimeout with Promise.delay, wait 20 ms for pg-notify --- test/unit/sql/rounds.js | 47 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index e4cc2a475c7..bf499bd7375 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -424,27 +424,27 @@ describe('Rounds-related SQL triggers', function () { }) }); - it('transactions of genesis block should be applied to mem_accounts (native)', function (done) { + it('transactions of genesis block should be applied to mem_accounts (native)', function () { // Wait 10 seconds for proper initialisation - setTimeout(function () { - getMemAccounts().then(function (accounts) { - // Number of returned accounts should be equal to number of unique accounts in genesis block - expect(Object.keys(accounts).length).to.equal(genesisAccounts.length); - - _.each(accounts, function (account) { - if (account.address === genesisAccount) { - // Genesis account should have negative balance - expect(account.balance).to.be.below(0); - } else if (account.isDelegate) { - // Delegates accounts should have balances of 0 - expect(account.balance).to.be.equal(0); - } else { - // Other accounts (with funds) should have positive balance - expect(account.balance).to.be.above(0); - } - }); - }).then(done).catch(done); - }, 10000); + return Promise.delay(10000).then(function () { + return getMemAccounts(); + }).then(function (accounts) { + // Number of returned accounts should be equal to number of unique accounts in genesis block + expect(Object.keys(accounts).length).to.equal(genesisAccounts.length); + + _.each(accounts, function (account) { + if (account.address === genesisAccount) { + // Genesis account should have negative balance + expect(account.balance).to.be.below(0); + } else if (account.isDelegate) { + // Delegates accounts should have balances of 0 + expect(account.balance).to.be.equal(0); + } else { + // Other accounts (with funds) should have positive balance + expect(account.balance).to.be.above(0); + } + }); + }); }); }); @@ -612,10 +612,9 @@ describe('Rounds-related SQL triggers', function () { } before(function () { - return new Promise(function (resolve) { + return Promise.delay(1000).then(function () { // Set delegates module as loaded to allow manual forging rewiredModules.delegates.__set__('__private.loaded', true); - setTimeout(resolve, 1000); }); }); @@ -821,7 +820,7 @@ describe('Rounds-related SQL triggers', function () { }); it('delete last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { - return deleteLastBlockPromise() + return deleteLastBlockPromise().delay(20) .then(function () { var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); expect(tmpDelegatesList).to.deep.equal(delegatesList); @@ -925,7 +924,7 @@ describe('Rounds-related SQL triggers', function () { }); it('delete last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { - return deleteLastBlockPromise() + return deleteLastBlockPromise().delay(20) .then(function () { var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); expect(tmpDelegatesList).to.deep.equal(delegatesList); From d8add1963c9ff79376101023caeb527d5d994e34 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 15:21:47 +0200 Subject: [PATCH 75/88] Create functions for maintain outsiders (missed blocks) --- .../20170521001337_roundsRewrite.sql | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170521001337_roundsRewrite.sql index 4363071959b..801aba5619d 100644 --- a/sql/migrations/20170521001337_roundsRewrite.sql +++ b/sql/migrations/20170521001337_roundsRewrite.sql @@ -268,9 +268,21 @@ CREATE CONSTRAINT TRIGGER block_insert_delete -- Create function 'delegates_update_on_block' for updating 'delegates' table data CREATE FUNCTION delegates_update_on_block() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN + -- Update outsiders of round + IF (TG_OP = 'INSERT') AND (NEW.height != 1) THEN + PERFORM outsiders_update(); + END IF; + PERFORM delegates_voters_cnt_update(); PERFORM delegates_voters_balance_update(); PERFORM delegates_rank_update(); + + -- Rollback outsiders of round + IF (TG_OP = 'DELETE') THEN + -- Pass public key of delegate who forged deleted block + PERFORM outsiders_rollback(ENCODE(OLD."generatorPublicKey", 'hex')); + END IF; + -- Perform notification to backend that round changed IF (TG_OP = 'INSERT') THEN -- Last block of round inserted - round is closed here and processing is done @@ -363,6 +375,42 @@ CREATE OR REPLACE FUNCTION round_rewards_insert() RETURNS TRIGGER LANGUAGE PLPGS RETURN NULL; END $$; +-- Create function for updating blocks_missed_cnt (outsiders of round) +CREATE OR REPLACE FUNCTION outsiders_update() RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ + BEGIN + RETURN QUERY + WITH + -- Calculate round from highest block + last_round AS (SELECT CEIL(height / 101::float)::int AS round FROM blocks ORDER BY height DESC LIMIT 1), + -- Increase delegates.blocks_missed_cnt + updated AS (UPDATE delegates d SET blocks_missed_cnt = blocks_missed_cnt+1 WHERE ENCODE(d.pk, 'hex') IN ( + -- Delegates who are on list and did not forged a block during round are treatment as round outsiders + SELECT outsider FROM UNNEST(getDelegatesList()) outsider WHERE outsider NOT IN ( + SELECT ENCODE(b."generatorPublicKey", 'hex') FROM blocks b WHERE CEIL(b.height / 101::float)::int = (SELECT round FROM last_round) + ) + ) RETURNING 1 + ) + SELECT COUNT(1)::INT FROM updated; +END $$; + +-- Create function for rollback blocks_missed_cnt (outsiders of round) in case of delete last block of round +CREATE OR REPLACE FUNCTION outsiders_rollback(last_block_forger text) RETURNS TABLE(updated INT) LANGUAGE PLPGSQL AS $$ + BEGIN + RETURN QUERY + WITH + -- Calculate round from highest block + last_round AS (SELECT CEIL(height / 101::float)::int AS round FROM blocks ORDER BY height DESC LIMIT 1), + -- Decrease delegates.blocks_missed_cnt + updated AS (UPDATE delegates d SET blocks_missed_cnt = blocks_missed_cnt-1 WHERE ENCODE(d.pk, 'hex') IN ( + -- Delegates who are on list and did not forged a block during round are treatment as round outsiders + SELECT outsider FROM UNNEST(getDelegatesList()) outsider WHERE outsider NOT IN ( + SELECT ENCODE(b."generatorPublicKey", 'hex') FROM blocks b WHERE CEIL(b.height / 101::float)::int = (SELECT round FROM last_round) + ) AND outsider <> last_block_forger -- Forger of deleted block cannot be outsider + ) RETURNING 1 + ) + SELECT COUNT(1)::INT FROM updated; +END $$; + -- Drop mem_round table DROP TABLE IF EXISTS mem_round; From 02d4894bd8f70e80895c576a8cda557d77b40a51 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Fri, 4 Aug 2017 15:22:20 +0200 Subject: [PATCH 76/88] Added tests for outsiders --- test/unit/sql/rounds.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index bf499bd7375..eadbcb772c7 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -451,6 +451,7 @@ describe('Rounds-related SQL triggers', function () { describe('round', function () { var round_mem_acc, round_delegates; var deleteLastBlockPromise; + var outsider_pk = '948b8b509579306694c00833ec1c0f81e964487db2206ddb1517bfeca2b0dc1b'; before(function () { // Copy initial round states for later comparison @@ -694,7 +695,7 @@ describe('Rounds-related SQL triggers', function () { // Rewards from database table rounds_rewards should match native rewards expect(rewards).to.deep.equal(expectedRewards); - + expect(delegates_state[outsider_pk].blocks_missed_cnt).to.equal(1); return Promise.reduce(delegates, function (delegates, d) { if (d.fees > 0 || d.rewards > 0) { // Normalize database data @@ -719,7 +720,6 @@ describe('Rounds-related SQL triggers', function () { describe('Delete last block of round 1, block contain 1 transaction type SEND', function () { var round = 1; - //var last_block = library.modules.blocks.lastBlock.get(); it('round rewards should be empty (rewards for round 1 deleted from rounds_rewards table)', function () { return deleteLastBlockPromise().then(function () { @@ -813,6 +813,7 @@ describe('Rounds-related SQL triggers', function () { it('forger of last block of previous round should have voters_balance and voters_cnt 0', function () { return getDelegates() .then(function () { + expect(delegates_state[outsider_pk].blocks_missed_cnt).to.equal(1); var delegate = delegates_state[last_block_forger]; expect(delegate.voters_balance).to.equal(0); expect(delegate.voters_cnt).to.equal(0); @@ -830,6 +831,7 @@ describe('Rounds-related SQL triggers', function () { it('expected forger of last block of round should have proper votes again', function () { return getDelegates() .then(function () { + expect(delegates_state[outsider_pk].blocks_missed_cnt).to.equal(0); var delegate = delegates_state[last_block_forger]; expect(delegate.voters_balance).to.equal(10000000000000000); expect(delegate.voters_cnt).to.equal(1); @@ -908,6 +910,7 @@ describe('Rounds-related SQL triggers', function () { it('forger of last block of previous round should have voters_balance and voters_cnt 0', function () { return getDelegates() .then(function () { + expect(delegates_state[outsider_pk].blocks_missed_cnt).to.equal(1); var delegate = delegates_state[last_block_forger]; expect(delegate.voters_balance).to.equal(0); expect(delegate.voters_cnt).to.equal(0); @@ -934,6 +937,7 @@ describe('Rounds-related SQL triggers', function () { it('expected forger of last block of round should have proper votes again', function () { return getDelegates() .then(function () { + expect(delegates_state[outsider_pk].blocks_missed_cnt).to.equal(0); var delegate = delegates_state[last_block_forger]; expect(delegate.voters_balance).to.equal(10000000000000000); expect(delegate.voters_cnt).to.equal(1); From 9f8fe9eed3b75337c2ff71563e7e8eaefd5ca1f7 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 08:16:34 +0200 Subject: [PATCH 77/88] Add tests for round rewards consistency --- test/unit/sql/rounds.js | 130 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 12 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index eadbcb772c7..3dc324b0609 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -1,18 +1,21 @@ 'use strict'; -var _ = require('lodash'); -var async = require('async'); -var chai = require('chai'); -var expect = require('chai').expect; -var rewire = require('rewire'); -var sinon = require('sinon'); +// Utils +var _ = require('lodash'); +var async = require('async'); +var chai = require('chai'); +var expect = require('chai').expect; var Promise = require('bluebird'); +var rewire = require('rewire'); +var sinon = require('sinon'); -var config = require('../../../config.json'); -var node = require('../../node.js'); -var Sequence = require('../../../helpers/sequence.js'); -var slots = require('../../../helpers/slots.js'); -var bignum = require('../../../helpers/bignum.js'); +// Application specific +var bignum = require('../../../helpers/bignum.js'); +var config = require('../../../config.json'); +var constants = require('../../../helpers/constants'); +var node = require('../../node.js'); +var Sequence = require('../../../helpers/sequence.js'); +var slots = require('../../../helpers/slots.js'); describe('Rounds-related SQL triggers', function () { var db, logger, library, rewiredModules = {}, modules = []; @@ -109,10 +112,14 @@ describe('Rounds-related SQL triggers', function () { return new bignum(fees).plus(block.totalFee); }, 0); + var rewardsTotal = _.reduce(blocks, function (reward, block) { + return new bignum(reward).plus(block.reward); + }, 0); + var feesPerDelegate = new bignum(feesTotal.toPrecision(15)).dividedBy(slots.delegates).floor(); var feesRemaining = new bignum(feesTotal.toPrecision(15)).minus(feesPerDelegate.times(slots.delegates)); - node.debug(' Total fees: ' + feesTotal.toString() + ' Fees per delegates: ' + feesPerDelegate.toString() + ' Remaining fees: ' + feesRemaining); + node.debug(' Total fees: ' + feesTotal.toString() + ' Fees per delegates: ' + feesPerDelegate.toString() + ' Remaining fees: ' + feesRemaining + 'Total rewards: ' + rewardsTotal); _.each(blocks, function (block, index) { var pk = block.generatorPublicKey.toString('hex'); @@ -163,6 +170,9 @@ describe('Rounds-related SQL triggers', function () { }); before(function (done) { + // Force rewards start at 150-th block + constants.rewards.offset = 150; + logger = { trace: sinon.spy(), debug: sinon.spy(), @@ -953,5 +963,101 @@ describe('Rounds-related SQL triggers', function () { }); }); }); + + describe('Rounds rewards consistency - round 2', function() { + var expected_reward; + var round; + + before(function (done) { + // Set expected reward per block as first milestone + expected_reward = constants.rewards.milestones[0]; + // Get height of last block + var current_height = library.modules.blocks.lastBlock.get().height; + // Calculate how many block to forge before rewards start + var blocks_to_forge = constants.rewards.offset - current_height - 1; // 1 before rewards start, co we can check + var blocks_processed = 0; + + async.doUntil(function (untilCb) { + ++blocks_processed; + node.debug(' Processing block ' + blocks_processed + ' of ' + blocks_to_forge); + + tickAndValidate([]).then(untilCb).catch(untilCb); + }, function (err) { + return err || blocks_processed >= blocks_to_forge; + }, done); + }); + + it('block just before rewards start should have reward 0', function () { + var last_block = library.modules.blocks.lastBlock.get(); + expect(last_block.reward).to.equal(0); + }); + + it('all blocks from now until round end should have proper rewards (' + expected_reward + ')', function (done) { + var blocks_processed = 0; + var last_block; + + // Forge blocks until end of a round + async.doUntil(function (untilCb) { + ++blocks_processed; + node.debug(' Processing block ' + blocks_processed); + + tickAndValidate([]).then(function () { + last_block = library.modules.blocks.lastBlock.get(); + // All blocks from now should have proper rewards + expect(last_block.reward).to.equal(expected_reward); + untilCb(); + }).catch(untilCb); + }, function (err) { + return err || last_block.height % 101 === 0; + }, done); + }); + + it('rewards from table rounds_rewards should match rewards from blockchian', function () { + var last_block = library.modules.blocks.lastBlock.get(); + round = slots.calcRound(last_block.height); + + return Promise.join(getBlocks(round), getRoundRewards(round), getDelegates(), function (blocks, rewards) { + // Get expected rewards for round (native) + var expectedRewards = getExpectedRoundRewards(blocks); + // Rewards from database table rounds_rewards should match native rewards + expect(rewards).to.deep.equal(expectedRewards); + }); + }); + + it('rewards from table delegates should match rewards from blockchain', function () { + var blocks_rewards, delegates_rewards; + return Promise.join(getBlocks(round), getDelegates(), function (blocks, delegates) { + return Promise.reduce(delegates, function (delegates, d) { + // Skip delegates who not forged + if (d.blocks_forged_cnt) { + delegates[d.pk] = { + pk: d.pk, + rewards: Number(d.rewards) + } + } + return delegates; + }, {}) + .then(function (delegates) { + delegates_rewards = delegates; + return Promise.reduce(blocks, function (blocks, b) { + var pk; + pk = b.generatorPublicKey.toString('hex'); + if (blocks[pk]) { + blocks.rewards += Number(b.reward); + } else { + blocks[pk] = { + pk: pk, + rewards: Number(b.reward) + } + } + return blocks; + }, {}) + .then (function (blocks) { + expect(delegates_rewards).to.deep.equal(blocks); + }); + }); + }); + }); + }); }); }); From 52562c10a30730f36db1fb7d028bd82f9fc2f117 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 17:38:36 +0200 Subject: [PATCH 78/88] Remove unused modules var and bindModules event --- app.js | 1 - logic/transaction.js | 11 +---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/app.js b/app.js index cc169d8f06d..3f541ff4b34 100644 --- a/app.js +++ b/app.js @@ -540,7 +540,6 @@ d.run(function () { // Fire onBind event in every module scope.bus.message('bind', scope.modules); - scope.logic.transaction.bindModules(scope.modules); scope.logic.peers.bindModules(scope.modules); cb(); }], diff --git a/logic/transaction.js b/logic/transaction.js index 0eb32964a48..e900eba2a0e 100644 --- a/logic/transaction.js +++ b/logic/transaction.js @@ -11,7 +11,7 @@ var slots = require('../helpers/slots.js'); var sql = require('../sql/transactions.js'); // Private fields -var self, modules, __private = {}; +var self, __private = {}; /** * @typedef {Object} privateTypes @@ -1085,14 +1085,5 @@ Transaction.prototype.dbRead = function (raw) { } }; -// Events -/** - * Binds input parameters to private variables modules. - * @param {Object} __modules - */ -Transaction.prototype.bindModules = function (__modules) { - this.scope.logger.trace('Logic/Transaction->bindModules'); -}; - // Export module.exports = Transaction; From 0c90a8b69f0293479d006322fb22c85c6d20e904 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 17:39:23 +0200 Subject: [PATCH 79/88] Fix standards --- modules/delegates.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/delegates.js b/modules/delegates.js index 65d80a08c75..eb9da794fd5 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -746,7 +746,7 @@ Delegates.prototype.shared = { for (var i = 1; i <= slots.delegates && i <= limit; i++) { if (__private.delegatesList[(currentSlot + i) % slots.delegates]) { - nextForgers.push (__private.delegatesList[(currentSlot + i) % slots.delegates]); + nextForgers.push(__private.delegatesList[(currentSlot + i) % slots.delegates]); } } From f011aaa1058ad67efa16f81dec90721e2457b579 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 17:40:10 +0200 Subject: [PATCH 80/88] Rename migrations so they can run after last applied one --- ...7001337_roundsRewards.sql => 20170807001337_roundsRewards.sql} | 0 ...validateMemTables.sql => 20170807101337_validateMemTables.sql} | 0 ...DelegatesList.sql => 20170807201337_generateDelegatesList.sql} | 0 ...1001337_roundsRewrite.sql => 20170807301337_roundsRewrite.sql} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename sql/migrations/{20170507001337_roundsRewards.sql => 20170807001337_roundsRewards.sql} (100%) rename sql/migrations/{20170508001337_validateMemTables.sql => 20170807101337_validateMemTables.sql} (100%) rename sql/migrations/{20170508101337_generateDelegatesList.sql => 20170807201337_generateDelegatesList.sql} (100%) rename sql/migrations/{20170521001337_roundsRewrite.sql => 20170807301337_roundsRewrite.sql} (100%) diff --git a/sql/migrations/20170507001337_roundsRewards.sql b/sql/migrations/20170807001337_roundsRewards.sql similarity index 100% rename from sql/migrations/20170507001337_roundsRewards.sql rename to sql/migrations/20170807001337_roundsRewards.sql diff --git a/sql/migrations/20170508001337_validateMemTables.sql b/sql/migrations/20170807101337_validateMemTables.sql similarity index 100% rename from sql/migrations/20170508001337_validateMemTables.sql rename to sql/migrations/20170807101337_validateMemTables.sql diff --git a/sql/migrations/20170508101337_generateDelegatesList.sql b/sql/migrations/20170807201337_generateDelegatesList.sql similarity index 100% rename from sql/migrations/20170508101337_generateDelegatesList.sql rename to sql/migrations/20170807201337_generateDelegatesList.sql diff --git a/sql/migrations/20170521001337_roundsRewrite.sql b/sql/migrations/20170807301337_roundsRewrite.sql similarity index 100% rename from sql/migrations/20170521001337_roundsRewrite.sql rename to sql/migrations/20170807301337_roundsRewrite.sql From ab3e08a3c6bfb86baa54fb2e44f5d5efde405f8e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 17:40:28 +0200 Subject: [PATCH 81/88] Improve descriptions for tests --- test/unit/sql/rounds.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 3dc324b0609..30b19bcae13 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -434,7 +434,7 @@ describe('Rounds-related SQL triggers', function () { }) }); - it('transactions of genesis block should be applied to mem_accounts (native)', function () { + it('should apply genesis block transactions to mem_accounts (native)', function () { // Wait 10 seconds for proper initialisation return Promise.delay(10000).then(function () { return getMemAccounts(); @@ -723,7 +723,7 @@ describe('Rounds-related SQL triggers', function () { }); }); - it('delegates list should be different than one generated at the beginning of round 1', function () { + it('should generate a different delegate list than one generated at the beginning of round 1', function () { var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); expect(tmpDelegatesList).to.not.deep.equal(delegatesList); }); @@ -776,7 +776,7 @@ describe('Rounds-related SQL triggers', function () { return deleteLastBlockPromise(); }); - it('we should be on height 99 after delete one more block', function () { + it('last block height should be at height 99 after deleting one more block', function () { var last_block = library.modules.blocks.lastBlock.get(); expect(last_block.height).to.equal(99); }); @@ -810,7 +810,7 @@ describe('Rounds-related SQL triggers', function () { }); }); - it('finish round, delegates list should be different than one generated at the beginning of round 1', function () { + it('after finishing round, delegates list should be different than one generated at the beginning of round 1', function () { var transactions = []; return tickAndValidate(transactions) @@ -830,7 +830,7 @@ describe('Rounds-related SQL triggers', function () { }); }); - it('delete last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { + it('after deleting last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { return deleteLastBlockPromise().delay(20) .then(function () { var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); @@ -877,12 +877,12 @@ describe('Rounds-related SQL triggers', function () { }); }); - it('we should be on height 100', function () { + it('last block height should be at height 100', function () { var last_block = library.modules.blocks.lastBlock.get(); expect(last_block.height).to.equal(100); }); - it('finish round, should unvote expected forger of last block of round and vote new delegate', function () { + it('after finishing round, should unvote expected forger of last block of round and vote new delegate', function () { var transactions = []; var tx = node.lisk.vote.createVote( node.gAccount.password, @@ -936,7 +936,7 @@ describe('Rounds-related SQL triggers', function () { }); }); - it('delete last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { + it('after deleting last block of round, delegates list should be equal to one generated at the beginning of round 1', function () { return deleteLastBlockPromise().delay(20) .then(function () { var tmpDelegatesList = rewiredModules.delegates.__get__('__private.delegatesList'); @@ -974,7 +974,7 @@ describe('Rounds-related SQL triggers', function () { // Get height of last block var current_height = library.modules.blocks.lastBlock.get().height; // Calculate how many block to forge before rewards start - var blocks_to_forge = constants.rewards.offset - current_height - 1; // 1 before rewards start, co we can check + var blocks_to_forge = constants.rewards.offset - current_height - 1; // 1 block before rewards start, so we can check var blocks_processed = 0; async.doUntil(function (untilCb) { @@ -987,7 +987,7 @@ describe('Rounds-related SQL triggers', function () { }, done); }); - it('block just before rewards start should have reward 0', function () { + it('block just before rewards start should have 0 reward', function () { var last_block = library.modules.blocks.lastBlock.get(); expect(last_block.reward).to.equal(0); }); From 45e6ec8a240419ae5518cb4a1acca0c9ae27ee8e Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 17:42:35 +0200 Subject: [PATCH 82/88] Better structure for delegatesList SQL tests --- test/unit/sql/delegatesList.js | 654 +++++++++++++++++---------------- 1 file changed, 331 insertions(+), 323 deletions(-) diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index 77b7bcf5880..de5c4b9699e 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -33,7 +33,7 @@ function generateDelegatesList (round, delegates) { return list; }; -describe('DelegatesListSQL', function () { +describe('Delegate list SQL functions', function () { before(function (done) { modulesLoader.getDbConnection(function (err, db_handle) { @@ -78,352 +78,360 @@ describe('DelegatesListSQL', function () { }); } - describe('checking SQL function generateDelegatesList()', function () { - it('SQL generateDelegatesList() results should be equal to generateDelegatesList() - fake 101 delegates', function (done) { - var round = '26381'; - var delegates = []; - for (var i = 1; i <= 101; i++) { - delegates.push(i.toString()); - } - var expectedDelegates = generateDelegatesList(round, delegates); - - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - expect(rows).to.be.an('array').and.lengthOf(1); - expect(rows[0]).to.be.an('object'); - expect(rows[0].delegates).to.be.an('array').and.lengthOf(delegates.length); - for (var i = rows[0].delegates.length - 1; i >= 0; i--) { - expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); - } - done(); - }).catch(done); - }); + describe('generateDelegatesList()', function () { - it('SQL generateDelegatesList() results should be equal to generateDelegatesList() - real 101 delegates, exact order', function (done) { - var round = '26381'; - var delegates = [ - 'ec111c8ad482445cfe83d811a7edd1f1d2765079c99d7d958cca1354740b7614', - 'b002f58531c074c7190714523eec08c48db8c7cfc0c943097db1a2e82ed87f84', - '1a99630b0ca1642b232888a3119e68b000b6194eced51e7fe3231bbe476f7c10', - '677c79b243ed96a8439e8bd193d6ab966ce43c9aa18830d2b9eb8974455d79f8', - '25e961fa459d202816776c8736560d493a94fdd7381971f63fb9b70479487598', - '32f20bee855238630b0f791560c02cf93014977b4b25c19ef93cd92220390276', - '00de7d28ec3f55f42329667f08352d0d90faa3d2d4e62c883d86d1d7b083dd7c', - 'ad936990fb57f7e686763c293e9ca773d1d921888f5235189945a10029cd95b0', - '253e674789632f72c98d47a650f1ca5ece0dbb82f591080471129d57ed88fb8a', - 'd12a6aef4b165b0197adb82d0d544202897b95300ff1fff93c339cf866defb0d', - '76ceefed8f29dd48664b07d207f4bf202122f2ffed6dcefa802d7fe348203b88', - '326bff18531703385d4037e5585b001e732c4a68afb8f82efe2b46c27dcf05aa', - '2493d52fc34ecaaa4a7d0d76e6de9bda24f1b5e11e3363c30a13d59e9c345f82', - 'c4d96fbfe80102f01579945fe0c5fe2a1874a7ffeca6bacef39140f9358e3db6', - '393f73238941510379d930e674e21ca4c00ba30c0877cd3728b5bd5874588671', - 'eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7', - '9771b09041466268948626830cbfea5a043527f39174d70e11a80011f386bb57', - 'b3953cb16e2457b9be78ad8c8a2985435dedaed5f0dd63443bdfbccc92d09f2d', - 'f147c1cba67acad603309d5004f25d9ab41ae073b318f4c6f972f96106c9b527', - 'ac09bc40c889f688f9158cca1fcfcdf6320f501242e0f7088d52a5077084ccba', - '93ec60444c28e8b8c0f1a613ed41f518b637280c454188e5500ea4e54e1a2f12', - '247bf1854471c1a97ccc363c63e876bb6b9a7f06038486048a17196a8a5493dc', - '1b774b27f49f4fe43cc8218e230bc39d0b16d0ee68abe828585ef87d316493ac', - '1305f8955a240a464393f52867d17ba271454fa2a6f2249fb5901b86e7c7334e', - 'adbe299504da4e6cf9d7eb481bdf72f23e6a0332df8049b4a018b99604e394da', - '41bb70d08312d9c17ec89a4577d30da77d5b936594fc06ccb0646602bed6ad40', - '5c4a92f575822b2d2deaa4bc0985ec9a57a17719bd5427af634ec1b4bf9c045b', - '186ffbe710bc27690934ef6cd64aeda2afdd634cbbaf6d23310ca7a31ab96e60', - '2d59fbcce531fb9661cdfa8371c49b6898ce0895fe71da88ffec851c7ed60782', - '484beb54e2990e17c18119b6065d00c8a65954039ec2d40a9e4ac41862dc561e', - '1681920f9cb83ff2590a8e5c502a7015d4834f5365cf5ed17392c9c78147f94d', - '7e838ec9b59a50d2c3333f079b0489871f12c1726eff483c3a88a287dbe36713', - '130649e3d8d34eb59197c00bcf6f199bc4ec06ba0968f1d473b010384569e7f0', - '9172179a88f8cfeeb81518ad31da4397555273b8658eb3ea2d1eca7965d8e615', - 'aad413159fe85e4f4d1941166ddcc97850f5964ee2ef8bda95519d019af8d488', - '6a01c4b86f4519ec9fa5c3288ae20e2e7a58822ebe891fb81e839588b95b242a', - 'feac3a6303ac41ebb9561d52fe2f3b4271fe846d2d2ffae722f18b6f04fc4ce9', - 'c7a0f96797a9dc3085534463650a09e1f160fecb6c0ec6c21e74ef2a222b73a4', - 'e36f75a27598512c2f4ad06fffeeffa37d3aad81c3a4edb77d871ec0ee933471', - 'eaa5ccb65e635e9ad3ebd98c2b7402b3a7c048fcd300c2d8aed8864f621ee6b2', - '2cb967f6c73d9b6b8604d7b199271fed3183ff18ae0bd9cde6d6ef6072f83c05', - '76c321881c08b0c2f538abf753044603ab3081f5441fe069c125a6e2803015da', - 'f8fa9e01047c19102133d2af06aab6cc377d5665bede412f04f81bcdc368d00e', - 'f91766de68f3a8859a3634c3a0fdde38ebd82dd91fc37b67ac6cf010800a3e6e', - 'b70f1d97cd254e93e2dd7b24567b3dbe06a60b5cbabe3443463c61cb87879b47', - 'f88b86d0a104bda71b2ff4d8234fef4e184ee771a9c2d3a298280790c185231b', - 'b6de69ebd1ba0bfe2d37ea6733c64b7e3eb262bee6c9cee05034b0b4465e2678', - 'b73fa499a7794c111fcd011cdc7dcc426341a28c6c2d6a32b8d7d028dcb8493f', - '0a13f5d075186bc99b9ec5b7bd3fbaeee0ab68a9314ac8d12a1f562e82d5e1c5', - 'e0f1c6cca365cd61bbb01cfb454828a698fa4b7170e85a597dde510567f9dda5', - '5386c93dbc76fce1e3a5ae5436ba98bb39e6a0929d038ee2118af54afd45614a', - 'e5c785871ac07632b42bc3862e7035330ff44fb0314e2253d1d7c0a35f3866f9', - 'c88af4585b4fabba89e8015bcf180c38a8027a8057dcf575977875a361282d7b', - 'a0f768d6476a9cfec1a64a895064fe114b26bd3fb6aeda397ccce7ef7f3f98ef', - '6cb825715058d2e821aa4af75fbd0da52181910d9fda90fabe73cd533eeb6acb', - 'a2c3a994fdf110802d5856ff18f306e7a3731452ed7a0fed8aac48e58fd729aa', - 'fbac76743fad9448ed0b45fb4c97a62f81a358908aa14f6a2c76d2a8dc207141', - 'de918e28b554600a81cbf119abf5414648b58a8efafbc3b0481df0242684dc1b', - '90ad9bfed339af2d6b4b3b7f7cdf25d927b255f9f25dbbc892ee9ca57ef67807', - 'a40c3e1549a9bbea71606ef05b793629923bdb151390145e3730dfe2b28b9217', - 'e7ac617b33d0f019d9d030c2e34870767d8994680e7b10ebdaf2af0e59332524', - 'b851863cf6b4769df5cecca718463173485bb9fe21e20f7cfb0802f5ab5973c2', - '33e8874f91f2b1295a2218e8d9f83761827a8d326fbc23e26b52a527714e75f0', - 'faf9f863e704f9cf560bc7a5718a25d851666d38195cba3cacd360cd5fa96fd3', - 'a2fc2420262f081d0f6426364301ef40597756e163f6b1fd813eff9b03594125', - '8966b54a95b327651e3103d8adb69579ff50bf22a004a65731a41f7caca2859f', - 'db4b4db208667f9266e8a4d7fad9d8b2e711891175a21ee5f5f2cd088d1d8083', - '6971dc02efc00140fbfcb262dd6f84d2dee533b258427de7017528b2e10ac2b1', - '613e4178a65c1194192eaa29910f0ecca3737f92587dd05d58c6435da41220f6', - 'e4717693ad6a02a4e6615e1ad4070fdf24d6a628a9d19a8396e4c91018a11307', - 'a81d59b68ba8942d60c74d10bc6488adec2ae1fa9b564a22447289076fe7b1e4', - 'c119a622b3ea85727b236574c43e83350252973ae765bb2061623a13c4f3d431', - 'ca1285393e1848ee41ba0c5e47789e5e0c570a7b51d8e2f7f6db37417b892cf9', - 'b690204a2a4a39431b8aaa4bb9af4e53aead93d2d46c5042edada9f5d43d6cd3', - '9c99976107b5d98e5669452392d912edf33c968e5832b52f2eedcd044b5cc2f2', - '942972349c8f2afe93ad874b3f19de05de7e34c120b23803438c7eeb8e6113b7', - '7fba92f4a2a510ae7301dddddf136e1f8673b54fd0ff0d92ec63f59b68bf4a8f', - 'abe994f962c34d7997506a657beee403f6b807eb2b2605dc6c3b93bb67b839eb', - '0fec636f5866c66f63a0d3db9b00e3cd5ba1b3a0324712c7935ae845dbfcf58a', - '63db2063e7760b241b0fe69436834fa2b759746b8237e1aafd2e099a38fc64d6', - '72f0cd8486d8627b5bd4f10c2e592a4512ac58e572edb3e37c0448b3ac7dd405', - '3345cae6361e329bc931fda1245e263617c797c8b21b7abfb7914fcda1a7833b', - '77c59f444c8a49bcd354759cc912166fe6eaa603a5f9d4a9525405b30a52ac10', - '45ab8f54edff6b802335dc3ea5cd5bc5324e4031c0598a2cdcae79402e4941f8', - '465085ba003a03d1fed0cfd35b7f3c07927c9db41d32194d273f8fe2fa238faa', - 'b68f666f1ede5615bf382958a815988a42aea8e4e03fbf0470a57bceac7714db', - 'c58078a7d12d81190ef0c5deb7611f97fc923d064647b66b9b25512029a13daf', - 'd9299750eeb71720dda470bccb8fafa57cf13f4939749615642c75a191481dea', - '2f58f5b6b1e2e91a9634dfadd1d6726a5aed2875f33260b6753cb9ec7da72917', - '0c0c8f58e7feeaa687d7dc9a5146ea14afe1bc647f518990b197b9f55728effa', - '968ba2fa993ea9dc27ed740da0daf49eddd740dbd7cb1cb4fc5db3a20baf341b', - '47226b469031d48f215973a11876c3f03a6d74360b40a55192b2ba9e5a74ede5', - '88260051bbe6634431f8a2f3ac66680d1ee9ef1087222e6823d9b4d81170edc7', - '619a3113c6cb1d3db7ef9731e6e06b618296815b3cfe7ca8d23f3767198b00ea', - '7ac9d4b708fb19eaa200eb883be56601ddceed96290a3a033114750b7fda9d0b', - 'fd039dd8caa03d58c0ecbaa09069403e7faff864dccd5933da50a41973292fa1', - 'b7633636a88ba1ce8acd98aa58b4a9618650c8ab860c167be6f8d78404265bae', - 'f495866ce86de18d8d4e746fca6a3a130608e5882875b88908b6551104f28e6a', - '31e1174043091ab0feb8d1e2ada4041a0ff54d0ced1e809890940bd706ffc201', - 'e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f', - 'f54ce2a222ab3513c49e586464d89a2a7d9959ecce60729289ec0bb6106bd4ce' - ]; - var expectedDelegates = generateDelegatesList(round, delegates); - - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - expect(rows).to.be.an('array').and.lengthOf(1); - expect(rows[0]).to.be.an('object'); - expect(rows[0].delegates).to.be.an('array').and.lengthOf(delegates.length); - for (var i = rows[0].delegates.length - 1; i >= 0; i--) { - expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); - } - expect(rows[0].delegates[0]).to.equal('b7633636a88ba1ce8acd98aa58b4a9618650c8ab860c167be6f8d78404265bae'); - expect(rows[0].delegates[1]).to.equal('1681920f9cb83ff2590a8e5c502a7015d4834f5365cf5ed17392c9c78147f94d'); - expect(rows[0].delegates[2]).to.equal('186ffbe710bc27690934ef6cd64aeda2afdd634cbbaf6d23310ca7a31ab96e60'); - expect(rows[0].delegates[3]).to.equal('0a13f5d075186bc99b9ec5b7bd3fbaeee0ab68a9314ac8d12a1f562e82d5e1c5'); - expect(rows[0].delegates[4]).to.equal('25e961fa459d202816776c8736560d493a94fdd7381971f63fb9b70479487598'); - expect(rows[0].delegates[5]).to.equal('3345cae6361e329bc931fda1245e263617c797c8b21b7abfb7914fcda1a7833b'); - expect(rows[0].delegates[6]).to.equal('a2c3a994fdf110802d5856ff18f306e7a3731452ed7a0fed8aac48e58fd729aa'); - expect(rows[0].delegates[7]).to.equal('f495866ce86de18d8d4e746fca6a3a130608e5882875b88908b6551104f28e6a'); - expect(rows[0].delegates[8]).to.equal('e36f75a27598512c2f4ad06fffeeffa37d3aad81c3a4edb77d871ec0ee933471'); - expect(rows[0].delegates[9]).to.equal('d12a6aef4b165b0197adb82d0d544202897b95300ff1fff93c339cf866defb0d'); - expect(rows[0].delegates[10]).to.equal('d9299750eeb71720dda470bccb8fafa57cf13f4939749615642c75a191481dea'); - expect(rows[0].delegates[11]).to.equal('77c59f444c8a49bcd354759cc912166fe6eaa603a5f9d4a9525405b30a52ac10'); - expect(rows[0].delegates[12]).to.equal('b73fa499a7794c111fcd011cdc7dcc426341a28c6c2d6a32b8d7d028dcb8493f'); - expect(rows[0].delegates[13]).to.equal('47226b469031d48f215973a11876c3f03a6d74360b40a55192b2ba9e5a74ede5'); - expect(rows[0].delegates[14]).to.equal('393f73238941510379d930e674e21ca4c00ba30c0877cd3728b5bd5874588671'); - expect(rows[0].delegates[15]).to.equal('abe994f962c34d7997506a657beee403f6b807eb2b2605dc6c3b93bb67b839eb'); - expect(rows[0].delegates[16]).to.equal('e4717693ad6a02a4e6615e1ad4070fdf24d6a628a9d19a8396e4c91018a11307'); - expect(rows[0].delegates[17]).to.equal('f91766de68f3a8859a3634c3a0fdde38ebd82dd91fc37b67ac6cf010800a3e6e'); - expect(rows[0].delegates[18]).to.equal('7e838ec9b59a50d2c3333f079b0489871f12c1726eff483c3a88a287dbe36713'); - expect(rows[0].delegates[19]).to.equal('942972349c8f2afe93ad874b3f19de05de7e34c120b23803438c7eeb8e6113b7'); - expect(rows[0].delegates[20]).to.equal('33e8874f91f2b1295a2218e8d9f83761827a8d326fbc23e26b52a527714e75f0'); - expect(rows[0].delegates[21]).to.equal('7ac9d4b708fb19eaa200eb883be56601ddceed96290a3a033114750b7fda9d0b'); - expect(rows[0].delegates[22]).to.equal('90ad9bfed339af2d6b4b3b7f7cdf25d927b255f9f25dbbc892ee9ca57ef67807'); - expect(rows[0].delegates[23]).to.equal('968ba2fa993ea9dc27ed740da0daf49eddd740dbd7cb1cb4fc5db3a20baf341b'); - expect(rows[0].delegates[24]).to.equal('adbe299504da4e6cf9d7eb481bdf72f23e6a0332df8049b4a018b99604e394da'); - expect(rows[0].delegates[25]).to.equal('e7ac617b33d0f019d9d030c2e34870767d8994680e7b10ebdaf2af0e59332524'); - expect(rows[0].delegates[26]).to.equal('b002f58531c074c7190714523eec08c48db8c7cfc0c943097db1a2e82ed87f84'); - expect(rows[0].delegates[27]).to.equal('ec111c8ad482445cfe83d811a7edd1f1d2765079c99d7d958cca1354740b7614'); - expect(rows[0].delegates[28]).to.equal('c88af4585b4fabba89e8015bcf180c38a8027a8057dcf575977875a361282d7b'); - expect(rows[0].delegates[29]).to.equal('484beb54e2990e17c18119b6065d00c8a65954039ec2d40a9e4ac41862dc561e'); - expect(rows[0].delegates[30]).to.equal('c119a622b3ea85727b236574c43e83350252973ae765bb2061623a13c4f3d431'); - expect(rows[0].delegates[31]).to.equal('feac3a6303ac41ebb9561d52fe2f3b4271fe846d2d2ffae722f18b6f04fc4ce9'); - expect(rows[0].delegates[32]).to.equal('c7a0f96797a9dc3085534463650a09e1f160fecb6c0ec6c21e74ef2a222b73a4'); - expect(rows[0].delegates[33]).to.equal('e5c785871ac07632b42bc3862e7035330ff44fb0314e2253d1d7c0a35f3866f9'); - expect(rows[0].delegates[34]).to.equal('253e674789632f72c98d47a650f1ca5ece0dbb82f591080471129d57ed88fb8a'); - expect(rows[0].delegates[35]).to.equal('db4b4db208667f9266e8a4d7fad9d8b2e711891175a21ee5f5f2cd088d1d8083'); - expect(rows[0].delegates[36]).to.equal('b6de69ebd1ba0bfe2d37ea6733c64b7e3eb262bee6c9cee05034b0b4465e2678'); - expect(rows[0].delegates[37]).to.equal('6a01c4b86f4519ec9fa5c3288ae20e2e7a58822ebe891fb81e839588b95b242a'); - expect(rows[0].delegates[38]).to.equal('aad413159fe85e4f4d1941166ddcc97850f5964ee2ef8bda95519d019af8d488'); - expect(rows[0].delegates[39]).to.equal('b690204a2a4a39431b8aaa4bb9af4e53aead93d2d46c5042edada9f5d43d6cd3'); - expect(rows[0].delegates[40]).to.equal('eaa5ccb65e635e9ad3ebd98c2b7402b3a7c048fcd300c2d8aed8864f621ee6b2'); - expect(rows[0].delegates[41]).to.equal('1305f8955a240a464393f52867d17ba271454fa2a6f2249fb5901b86e7c7334e'); - expect(rows[0].delegates[42]).to.equal('1a99630b0ca1642b232888a3119e68b000b6194eced51e7fe3231bbe476f7c10'); - expect(rows[0].delegates[43]).to.equal('31e1174043091ab0feb8d1e2ada4041a0ff54d0ced1e809890940bd706ffc201'); - expect(rows[0].delegates[44]).to.equal('ad936990fb57f7e686763c293e9ca773d1d921888f5235189945a10029cd95b0'); - expect(rows[0].delegates[45]).to.equal('faf9f863e704f9cf560bc7a5718a25d851666d38195cba3cacd360cd5fa96fd3'); - expect(rows[0].delegates[46]).to.equal('a81d59b68ba8942d60c74d10bc6488adec2ae1fa9b564a22447289076fe7b1e4'); - expect(rows[0].delegates[47]).to.equal('1b774b27f49f4fe43cc8218e230bc39d0b16d0ee68abe828585ef87d316493ac'); - expect(rows[0].delegates[48]).to.equal('00de7d28ec3f55f42329667f08352d0d90faa3d2d4e62c883d86d1d7b083dd7c'); - expect(rows[0].delegates[49]).to.equal('e0f1c6cca365cd61bbb01cfb454828a698fa4b7170e85a597dde510567f9dda5'); - expect(rows[0].delegates[50]).to.equal('5386c93dbc76fce1e3a5ae5436ba98bb39e6a0929d038ee2118af54afd45614a'); - expect(rows[0].delegates[51]).to.equal('f54ce2a222ab3513c49e586464d89a2a7d9959ecce60729289ec0bb6106bd4ce'); - expect(rows[0].delegates[52]).to.equal('b70f1d97cd254e93e2dd7b24567b3dbe06a60b5cbabe3443463c61cb87879b47'); - expect(rows[0].delegates[53]).to.equal('613e4178a65c1194192eaa29910f0ecca3737f92587dd05d58c6435da41220f6'); - expect(rows[0].delegates[54]).to.equal('6cb825715058d2e821aa4af75fbd0da52181910d9fda90fabe73cd533eeb6acb'); - expect(rows[0].delegates[55]).to.equal('76c321881c08b0c2f538abf753044603ab3081f5441fe069c125a6e2803015da'); - expect(rows[0].delegates[56]).to.equal('9172179a88f8cfeeb81518ad31da4397555273b8658eb3ea2d1eca7965d8e615'); - expect(rows[0].delegates[57]).to.equal('41bb70d08312d9c17ec89a4577d30da77d5b936594fc06ccb0646602bed6ad40'); - expect(rows[0].delegates[58]).to.equal('fd039dd8caa03d58c0ecbaa09069403e7faff864dccd5933da50a41973292fa1'); - expect(rows[0].delegates[59]).to.equal('a40c3e1549a9bbea71606ef05b793629923bdb151390145e3730dfe2b28b9217'); - expect(rows[0].delegates[60]).to.equal('f88b86d0a104bda71b2ff4d8234fef4e184ee771a9c2d3a298280790c185231b'); - expect(rows[0].delegates[61]).to.equal('88260051bbe6634431f8a2f3ac66680d1ee9ef1087222e6823d9b4d81170edc7'); - expect(rows[0].delegates[62]).to.equal('0fec636f5866c66f63a0d3db9b00e3cd5ba1b3a0324712c7935ae845dbfcf58a'); - expect(rows[0].delegates[63]).to.equal('677c79b243ed96a8439e8bd193d6ab966ce43c9aa18830d2b9eb8974455d79f8'); - expect(rows[0].delegates[64]).to.equal('a2fc2420262f081d0f6426364301ef40597756e163f6b1fd813eff9b03594125'); - expect(rows[0].delegates[65]).to.equal('de918e28b554600a81cbf119abf5414648b58a8efafbc3b0481df0242684dc1b'); - expect(rows[0].delegates[66]).to.equal('c58078a7d12d81190ef0c5deb7611f97fc923d064647b66b9b25512029a13daf'); - expect(rows[0].delegates[67]).to.equal('619a3113c6cb1d3db7ef9731e6e06b618296815b3cfe7ca8d23f3767198b00ea'); - expect(rows[0].delegates[68]).to.equal('c4d96fbfe80102f01579945fe0c5fe2a1874a7ffeca6bacef39140f9358e3db6'); - expect(rows[0].delegates[69]).to.equal('b851863cf6b4769df5cecca718463173485bb9fe21e20f7cfb0802f5ab5973c2'); - expect(rows[0].delegates[70]).to.equal('a0f768d6476a9cfec1a64a895064fe114b26bd3fb6aeda397ccce7ef7f3f98ef'); - expect(rows[0].delegates[71]).to.equal('5c4a92f575822b2d2deaa4bc0985ec9a57a17719bd5427af634ec1b4bf9c045b'); - expect(rows[0].delegates[72]).to.equal('2493d52fc34ecaaa4a7d0d76e6de9bda24f1b5e11e3363c30a13d59e9c345f82'); - expect(rows[0].delegates[73]).to.equal('2cb967f6c73d9b6b8604d7b199271fed3183ff18ae0bd9cde6d6ef6072f83c05'); - expect(rows[0].delegates[74]).to.equal('9c99976107b5d98e5669452392d912edf33c968e5832b52f2eedcd044b5cc2f2'); - expect(rows[0].delegates[75]).to.equal('ac09bc40c889f688f9158cca1fcfcdf6320f501242e0f7088d52a5077084ccba'); - expect(rows[0].delegates[76]).to.equal('ca1285393e1848ee41ba0c5e47789e5e0c570a7b51d8e2f7f6db37417b892cf9'); - expect(rows[0].delegates[77]).to.equal('6971dc02efc00140fbfcb262dd6f84d2dee533b258427de7017528b2e10ac2b1'); - expect(rows[0].delegates[78]).to.equal('b3953cb16e2457b9be78ad8c8a2985435dedaed5f0dd63443bdfbccc92d09f2d'); - expect(rows[0].delegates[79]).to.equal('63db2063e7760b241b0fe69436834fa2b759746b8237e1aafd2e099a38fc64d6'); - expect(rows[0].delegates[80]).to.equal('f8fa9e01047c19102133d2af06aab6cc377d5665bede412f04f81bcdc368d00e'); - expect(rows[0].delegates[81]).to.equal('93ec60444c28e8b8c0f1a613ed41f518b637280c454188e5500ea4e54e1a2f12'); - expect(rows[0].delegates[82]).to.equal('130649e3d8d34eb59197c00bcf6f199bc4ec06ba0968f1d473b010384569e7f0'); - expect(rows[0].delegates[83]).to.equal('0c0c8f58e7feeaa687d7dc9a5146ea14afe1bc647f518990b197b9f55728effa'); - expect(rows[0].delegates[84]).to.equal('b68f666f1ede5615bf382958a815988a42aea8e4e03fbf0470a57bceac7714db'); - expect(rows[0].delegates[85]).to.equal('32f20bee855238630b0f791560c02cf93014977b4b25c19ef93cd92220390276'); - expect(rows[0].delegates[86]).to.equal('45ab8f54edff6b802335dc3ea5cd5bc5324e4031c0598a2cdcae79402e4941f8'); - expect(rows[0].delegates[87]).to.equal('f147c1cba67acad603309d5004f25d9ab41ae073b318f4c6f972f96106c9b527'); - expect(rows[0].delegates[88]).to.equal('72f0cd8486d8627b5bd4f10c2e592a4512ac58e572edb3e37c0448b3ac7dd405'); - expect(rows[0].delegates[89]).to.equal('326bff18531703385d4037e5585b001e732c4a68afb8f82efe2b46c27dcf05aa'); - expect(rows[0].delegates[90]).to.equal('fbac76743fad9448ed0b45fb4c97a62f81a358908aa14f6a2c76d2a8dc207141'); - expect(rows[0].delegates[91]).to.equal('2f58f5b6b1e2e91a9634dfadd1d6726a5aed2875f33260b6753cb9ec7da72917'); - expect(rows[0].delegates[92]).to.equal('9771b09041466268948626830cbfea5a043527f39174d70e11a80011f386bb57'); - expect(rows[0].delegates[93]).to.equal('76ceefed8f29dd48664b07d207f4bf202122f2ffed6dcefa802d7fe348203b88'); - expect(rows[0].delegates[94]).to.equal('247bf1854471c1a97ccc363c63e876bb6b9a7f06038486048a17196a8a5493dc'); - expect(rows[0].delegates[95]).to.equal('8966b54a95b327651e3103d8adb69579ff50bf22a004a65731a41f7caca2859f'); - expect(rows[0].delegates[96]).to.equal('7fba92f4a2a510ae7301dddddf136e1f8673b54fd0ff0d92ec63f59b68bf4a8f'); - expect(rows[0].delegates[97]).to.equal('2d59fbcce531fb9661cdfa8371c49b6898ce0895fe71da88ffec851c7ed60782'); - expect(rows[0].delegates[98]).to.equal('465085ba003a03d1fed0cfd35b7f3c07927c9db41d32194d273f8fe2fa238faa'); - expect(rows[0].delegates[99]).to.equal('e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f'); - expect(rows[0].delegates[100]).to.equal('eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7'); - done(); - }).catch(done); - }); + describe('results', function () { - it('SQL generateDelegatesList() should raise exception for round 0', function (done) { - var round = '0'; - var delegates = ['1']; - - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); + it('SQL results should be equal to native - fake 101 delegates', function (done) { + var round = '26381'; + var delegates = []; + for (var i = 1; i <= 101; i++) { + delegates.push(i.toString()); + } + var expectedDelegates = generateDelegatesList(round, delegates); + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + expect(rows).to.be.an('array').and.lengthOf(1); + expect(rows[0]).to.be.an('object'); + expect(rows[0].delegates).to.be.an('array').and.lengthOf(delegates.length); + for (var i = rows[0].delegates.length - 1; i >= 0; i--) { + expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); + } + done(); + }).catch(done); }); - }); - it('SQL generateDelegatesList() should raise exception for undefined round', function (done) { - var round = undefined; - var delegates = ['1']; - - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); + it('SQL results should be equal to native - real 101 delegates, exact order', function (done) { + var round = '26381'; + var delegates = [ + 'ec111c8ad482445cfe83d811a7edd1f1d2765079c99d7d958cca1354740b7614', + 'b002f58531c074c7190714523eec08c48db8c7cfc0c943097db1a2e82ed87f84', + '1a99630b0ca1642b232888a3119e68b000b6194eced51e7fe3231bbe476f7c10', + '677c79b243ed96a8439e8bd193d6ab966ce43c9aa18830d2b9eb8974455d79f8', + '25e961fa459d202816776c8736560d493a94fdd7381971f63fb9b70479487598', + '32f20bee855238630b0f791560c02cf93014977b4b25c19ef93cd92220390276', + '00de7d28ec3f55f42329667f08352d0d90faa3d2d4e62c883d86d1d7b083dd7c', + 'ad936990fb57f7e686763c293e9ca773d1d921888f5235189945a10029cd95b0', + '253e674789632f72c98d47a650f1ca5ece0dbb82f591080471129d57ed88fb8a', + 'd12a6aef4b165b0197adb82d0d544202897b95300ff1fff93c339cf866defb0d', + '76ceefed8f29dd48664b07d207f4bf202122f2ffed6dcefa802d7fe348203b88', + '326bff18531703385d4037e5585b001e732c4a68afb8f82efe2b46c27dcf05aa', + '2493d52fc34ecaaa4a7d0d76e6de9bda24f1b5e11e3363c30a13d59e9c345f82', + 'c4d96fbfe80102f01579945fe0c5fe2a1874a7ffeca6bacef39140f9358e3db6', + '393f73238941510379d930e674e21ca4c00ba30c0877cd3728b5bd5874588671', + 'eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7', + '9771b09041466268948626830cbfea5a043527f39174d70e11a80011f386bb57', + 'b3953cb16e2457b9be78ad8c8a2985435dedaed5f0dd63443bdfbccc92d09f2d', + 'f147c1cba67acad603309d5004f25d9ab41ae073b318f4c6f972f96106c9b527', + 'ac09bc40c889f688f9158cca1fcfcdf6320f501242e0f7088d52a5077084ccba', + '93ec60444c28e8b8c0f1a613ed41f518b637280c454188e5500ea4e54e1a2f12', + '247bf1854471c1a97ccc363c63e876bb6b9a7f06038486048a17196a8a5493dc', + '1b774b27f49f4fe43cc8218e230bc39d0b16d0ee68abe828585ef87d316493ac', + '1305f8955a240a464393f52867d17ba271454fa2a6f2249fb5901b86e7c7334e', + 'adbe299504da4e6cf9d7eb481bdf72f23e6a0332df8049b4a018b99604e394da', + '41bb70d08312d9c17ec89a4577d30da77d5b936594fc06ccb0646602bed6ad40', + '5c4a92f575822b2d2deaa4bc0985ec9a57a17719bd5427af634ec1b4bf9c045b', + '186ffbe710bc27690934ef6cd64aeda2afdd634cbbaf6d23310ca7a31ab96e60', + '2d59fbcce531fb9661cdfa8371c49b6898ce0895fe71da88ffec851c7ed60782', + '484beb54e2990e17c18119b6065d00c8a65954039ec2d40a9e4ac41862dc561e', + '1681920f9cb83ff2590a8e5c502a7015d4834f5365cf5ed17392c9c78147f94d', + '7e838ec9b59a50d2c3333f079b0489871f12c1726eff483c3a88a287dbe36713', + '130649e3d8d34eb59197c00bcf6f199bc4ec06ba0968f1d473b010384569e7f0', + '9172179a88f8cfeeb81518ad31da4397555273b8658eb3ea2d1eca7965d8e615', + 'aad413159fe85e4f4d1941166ddcc97850f5964ee2ef8bda95519d019af8d488', + '6a01c4b86f4519ec9fa5c3288ae20e2e7a58822ebe891fb81e839588b95b242a', + 'feac3a6303ac41ebb9561d52fe2f3b4271fe846d2d2ffae722f18b6f04fc4ce9', + 'c7a0f96797a9dc3085534463650a09e1f160fecb6c0ec6c21e74ef2a222b73a4', + 'e36f75a27598512c2f4ad06fffeeffa37d3aad81c3a4edb77d871ec0ee933471', + 'eaa5ccb65e635e9ad3ebd98c2b7402b3a7c048fcd300c2d8aed8864f621ee6b2', + '2cb967f6c73d9b6b8604d7b199271fed3183ff18ae0bd9cde6d6ef6072f83c05', + '76c321881c08b0c2f538abf753044603ab3081f5441fe069c125a6e2803015da', + 'f8fa9e01047c19102133d2af06aab6cc377d5665bede412f04f81bcdc368d00e', + 'f91766de68f3a8859a3634c3a0fdde38ebd82dd91fc37b67ac6cf010800a3e6e', + 'b70f1d97cd254e93e2dd7b24567b3dbe06a60b5cbabe3443463c61cb87879b47', + 'f88b86d0a104bda71b2ff4d8234fef4e184ee771a9c2d3a298280790c185231b', + 'b6de69ebd1ba0bfe2d37ea6733c64b7e3eb262bee6c9cee05034b0b4465e2678', + 'b73fa499a7794c111fcd011cdc7dcc426341a28c6c2d6a32b8d7d028dcb8493f', + '0a13f5d075186bc99b9ec5b7bd3fbaeee0ab68a9314ac8d12a1f562e82d5e1c5', + 'e0f1c6cca365cd61bbb01cfb454828a698fa4b7170e85a597dde510567f9dda5', + '5386c93dbc76fce1e3a5ae5436ba98bb39e6a0929d038ee2118af54afd45614a', + 'e5c785871ac07632b42bc3862e7035330ff44fb0314e2253d1d7c0a35f3866f9', + 'c88af4585b4fabba89e8015bcf180c38a8027a8057dcf575977875a361282d7b', + 'a0f768d6476a9cfec1a64a895064fe114b26bd3fb6aeda397ccce7ef7f3f98ef', + '6cb825715058d2e821aa4af75fbd0da52181910d9fda90fabe73cd533eeb6acb', + 'a2c3a994fdf110802d5856ff18f306e7a3731452ed7a0fed8aac48e58fd729aa', + 'fbac76743fad9448ed0b45fb4c97a62f81a358908aa14f6a2c76d2a8dc207141', + 'de918e28b554600a81cbf119abf5414648b58a8efafbc3b0481df0242684dc1b', + '90ad9bfed339af2d6b4b3b7f7cdf25d927b255f9f25dbbc892ee9ca57ef67807', + 'a40c3e1549a9bbea71606ef05b793629923bdb151390145e3730dfe2b28b9217', + 'e7ac617b33d0f019d9d030c2e34870767d8994680e7b10ebdaf2af0e59332524', + 'b851863cf6b4769df5cecca718463173485bb9fe21e20f7cfb0802f5ab5973c2', + '33e8874f91f2b1295a2218e8d9f83761827a8d326fbc23e26b52a527714e75f0', + 'faf9f863e704f9cf560bc7a5718a25d851666d38195cba3cacd360cd5fa96fd3', + 'a2fc2420262f081d0f6426364301ef40597756e163f6b1fd813eff9b03594125', + '8966b54a95b327651e3103d8adb69579ff50bf22a004a65731a41f7caca2859f', + 'db4b4db208667f9266e8a4d7fad9d8b2e711891175a21ee5f5f2cd088d1d8083', + '6971dc02efc00140fbfcb262dd6f84d2dee533b258427de7017528b2e10ac2b1', + '613e4178a65c1194192eaa29910f0ecca3737f92587dd05d58c6435da41220f6', + 'e4717693ad6a02a4e6615e1ad4070fdf24d6a628a9d19a8396e4c91018a11307', + 'a81d59b68ba8942d60c74d10bc6488adec2ae1fa9b564a22447289076fe7b1e4', + 'c119a622b3ea85727b236574c43e83350252973ae765bb2061623a13c4f3d431', + 'ca1285393e1848ee41ba0c5e47789e5e0c570a7b51d8e2f7f6db37417b892cf9', + 'b690204a2a4a39431b8aaa4bb9af4e53aead93d2d46c5042edada9f5d43d6cd3', + '9c99976107b5d98e5669452392d912edf33c968e5832b52f2eedcd044b5cc2f2', + '942972349c8f2afe93ad874b3f19de05de7e34c120b23803438c7eeb8e6113b7', + '7fba92f4a2a510ae7301dddddf136e1f8673b54fd0ff0d92ec63f59b68bf4a8f', + 'abe994f962c34d7997506a657beee403f6b807eb2b2605dc6c3b93bb67b839eb', + '0fec636f5866c66f63a0d3db9b00e3cd5ba1b3a0324712c7935ae845dbfcf58a', + '63db2063e7760b241b0fe69436834fa2b759746b8237e1aafd2e099a38fc64d6', + '72f0cd8486d8627b5bd4f10c2e592a4512ac58e572edb3e37c0448b3ac7dd405', + '3345cae6361e329bc931fda1245e263617c797c8b21b7abfb7914fcda1a7833b', + '77c59f444c8a49bcd354759cc912166fe6eaa603a5f9d4a9525405b30a52ac10', + '45ab8f54edff6b802335dc3ea5cd5bc5324e4031c0598a2cdcae79402e4941f8', + '465085ba003a03d1fed0cfd35b7f3c07927c9db41d32194d273f8fe2fa238faa', + 'b68f666f1ede5615bf382958a815988a42aea8e4e03fbf0470a57bceac7714db', + 'c58078a7d12d81190ef0c5deb7611f97fc923d064647b66b9b25512029a13daf', + 'd9299750eeb71720dda470bccb8fafa57cf13f4939749615642c75a191481dea', + '2f58f5b6b1e2e91a9634dfadd1d6726a5aed2875f33260b6753cb9ec7da72917', + '0c0c8f58e7feeaa687d7dc9a5146ea14afe1bc647f518990b197b9f55728effa', + '968ba2fa993ea9dc27ed740da0daf49eddd740dbd7cb1cb4fc5db3a20baf341b', + '47226b469031d48f215973a11876c3f03a6d74360b40a55192b2ba9e5a74ede5', + '88260051bbe6634431f8a2f3ac66680d1ee9ef1087222e6823d9b4d81170edc7', + '619a3113c6cb1d3db7ef9731e6e06b618296815b3cfe7ca8d23f3767198b00ea', + '7ac9d4b708fb19eaa200eb883be56601ddceed96290a3a033114750b7fda9d0b', + 'fd039dd8caa03d58c0ecbaa09069403e7faff864dccd5933da50a41973292fa1', + 'b7633636a88ba1ce8acd98aa58b4a9618650c8ab860c167be6f8d78404265bae', + 'f495866ce86de18d8d4e746fca6a3a130608e5882875b88908b6551104f28e6a', + '31e1174043091ab0feb8d1e2ada4041a0ff54d0ced1e809890940bd706ffc201', + 'e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f', + 'f54ce2a222ab3513c49e586464d89a2a7d9959ecce60729289ec0bb6106bd4ce' + ]; + var expectedDelegates = generateDelegatesList(round, delegates); + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + expect(rows).to.be.an('array').and.lengthOf(1); + expect(rows[0]).to.be.an('object'); + expect(rows[0].delegates).to.be.an('array').and.lengthOf(delegates.length); + for (var i = rows[0].delegates.length - 1; i >= 0; i--) { + expect(rows[0].delegates[i]).to.equal(expectedDelegates[i]); + } + expect(rows[0].delegates[0]).to.equal('b7633636a88ba1ce8acd98aa58b4a9618650c8ab860c167be6f8d78404265bae'); + expect(rows[0].delegates[1]).to.equal('1681920f9cb83ff2590a8e5c502a7015d4834f5365cf5ed17392c9c78147f94d'); + expect(rows[0].delegates[2]).to.equal('186ffbe710bc27690934ef6cd64aeda2afdd634cbbaf6d23310ca7a31ab96e60'); + expect(rows[0].delegates[3]).to.equal('0a13f5d075186bc99b9ec5b7bd3fbaeee0ab68a9314ac8d12a1f562e82d5e1c5'); + expect(rows[0].delegates[4]).to.equal('25e961fa459d202816776c8736560d493a94fdd7381971f63fb9b70479487598'); + expect(rows[0].delegates[5]).to.equal('3345cae6361e329bc931fda1245e263617c797c8b21b7abfb7914fcda1a7833b'); + expect(rows[0].delegates[6]).to.equal('a2c3a994fdf110802d5856ff18f306e7a3731452ed7a0fed8aac48e58fd729aa'); + expect(rows[0].delegates[7]).to.equal('f495866ce86de18d8d4e746fca6a3a130608e5882875b88908b6551104f28e6a'); + expect(rows[0].delegates[8]).to.equal('e36f75a27598512c2f4ad06fffeeffa37d3aad81c3a4edb77d871ec0ee933471'); + expect(rows[0].delegates[9]).to.equal('d12a6aef4b165b0197adb82d0d544202897b95300ff1fff93c339cf866defb0d'); + expect(rows[0].delegates[10]).to.equal('d9299750eeb71720dda470bccb8fafa57cf13f4939749615642c75a191481dea'); + expect(rows[0].delegates[11]).to.equal('77c59f444c8a49bcd354759cc912166fe6eaa603a5f9d4a9525405b30a52ac10'); + expect(rows[0].delegates[12]).to.equal('b73fa499a7794c111fcd011cdc7dcc426341a28c6c2d6a32b8d7d028dcb8493f'); + expect(rows[0].delegates[13]).to.equal('47226b469031d48f215973a11876c3f03a6d74360b40a55192b2ba9e5a74ede5'); + expect(rows[0].delegates[14]).to.equal('393f73238941510379d930e674e21ca4c00ba30c0877cd3728b5bd5874588671'); + expect(rows[0].delegates[15]).to.equal('abe994f962c34d7997506a657beee403f6b807eb2b2605dc6c3b93bb67b839eb'); + expect(rows[0].delegates[16]).to.equal('e4717693ad6a02a4e6615e1ad4070fdf24d6a628a9d19a8396e4c91018a11307'); + expect(rows[0].delegates[17]).to.equal('f91766de68f3a8859a3634c3a0fdde38ebd82dd91fc37b67ac6cf010800a3e6e'); + expect(rows[0].delegates[18]).to.equal('7e838ec9b59a50d2c3333f079b0489871f12c1726eff483c3a88a287dbe36713'); + expect(rows[0].delegates[19]).to.equal('942972349c8f2afe93ad874b3f19de05de7e34c120b23803438c7eeb8e6113b7'); + expect(rows[0].delegates[20]).to.equal('33e8874f91f2b1295a2218e8d9f83761827a8d326fbc23e26b52a527714e75f0'); + expect(rows[0].delegates[21]).to.equal('7ac9d4b708fb19eaa200eb883be56601ddceed96290a3a033114750b7fda9d0b'); + expect(rows[0].delegates[22]).to.equal('90ad9bfed339af2d6b4b3b7f7cdf25d927b255f9f25dbbc892ee9ca57ef67807'); + expect(rows[0].delegates[23]).to.equal('968ba2fa993ea9dc27ed740da0daf49eddd740dbd7cb1cb4fc5db3a20baf341b'); + expect(rows[0].delegates[24]).to.equal('adbe299504da4e6cf9d7eb481bdf72f23e6a0332df8049b4a018b99604e394da'); + expect(rows[0].delegates[25]).to.equal('e7ac617b33d0f019d9d030c2e34870767d8994680e7b10ebdaf2af0e59332524'); + expect(rows[0].delegates[26]).to.equal('b002f58531c074c7190714523eec08c48db8c7cfc0c943097db1a2e82ed87f84'); + expect(rows[0].delegates[27]).to.equal('ec111c8ad482445cfe83d811a7edd1f1d2765079c99d7d958cca1354740b7614'); + expect(rows[0].delegates[28]).to.equal('c88af4585b4fabba89e8015bcf180c38a8027a8057dcf575977875a361282d7b'); + expect(rows[0].delegates[29]).to.equal('484beb54e2990e17c18119b6065d00c8a65954039ec2d40a9e4ac41862dc561e'); + expect(rows[0].delegates[30]).to.equal('c119a622b3ea85727b236574c43e83350252973ae765bb2061623a13c4f3d431'); + expect(rows[0].delegates[31]).to.equal('feac3a6303ac41ebb9561d52fe2f3b4271fe846d2d2ffae722f18b6f04fc4ce9'); + expect(rows[0].delegates[32]).to.equal('c7a0f96797a9dc3085534463650a09e1f160fecb6c0ec6c21e74ef2a222b73a4'); + expect(rows[0].delegates[33]).to.equal('e5c785871ac07632b42bc3862e7035330ff44fb0314e2253d1d7c0a35f3866f9'); + expect(rows[0].delegates[34]).to.equal('253e674789632f72c98d47a650f1ca5ece0dbb82f591080471129d57ed88fb8a'); + expect(rows[0].delegates[35]).to.equal('db4b4db208667f9266e8a4d7fad9d8b2e711891175a21ee5f5f2cd088d1d8083'); + expect(rows[0].delegates[36]).to.equal('b6de69ebd1ba0bfe2d37ea6733c64b7e3eb262bee6c9cee05034b0b4465e2678'); + expect(rows[0].delegates[37]).to.equal('6a01c4b86f4519ec9fa5c3288ae20e2e7a58822ebe891fb81e839588b95b242a'); + expect(rows[0].delegates[38]).to.equal('aad413159fe85e4f4d1941166ddcc97850f5964ee2ef8bda95519d019af8d488'); + expect(rows[0].delegates[39]).to.equal('b690204a2a4a39431b8aaa4bb9af4e53aead93d2d46c5042edada9f5d43d6cd3'); + expect(rows[0].delegates[40]).to.equal('eaa5ccb65e635e9ad3ebd98c2b7402b3a7c048fcd300c2d8aed8864f621ee6b2'); + expect(rows[0].delegates[41]).to.equal('1305f8955a240a464393f52867d17ba271454fa2a6f2249fb5901b86e7c7334e'); + expect(rows[0].delegates[42]).to.equal('1a99630b0ca1642b232888a3119e68b000b6194eced51e7fe3231bbe476f7c10'); + expect(rows[0].delegates[43]).to.equal('31e1174043091ab0feb8d1e2ada4041a0ff54d0ced1e809890940bd706ffc201'); + expect(rows[0].delegates[44]).to.equal('ad936990fb57f7e686763c293e9ca773d1d921888f5235189945a10029cd95b0'); + expect(rows[0].delegates[45]).to.equal('faf9f863e704f9cf560bc7a5718a25d851666d38195cba3cacd360cd5fa96fd3'); + expect(rows[0].delegates[46]).to.equal('a81d59b68ba8942d60c74d10bc6488adec2ae1fa9b564a22447289076fe7b1e4'); + expect(rows[0].delegates[47]).to.equal('1b774b27f49f4fe43cc8218e230bc39d0b16d0ee68abe828585ef87d316493ac'); + expect(rows[0].delegates[48]).to.equal('00de7d28ec3f55f42329667f08352d0d90faa3d2d4e62c883d86d1d7b083dd7c'); + expect(rows[0].delegates[49]).to.equal('e0f1c6cca365cd61bbb01cfb454828a698fa4b7170e85a597dde510567f9dda5'); + expect(rows[0].delegates[50]).to.equal('5386c93dbc76fce1e3a5ae5436ba98bb39e6a0929d038ee2118af54afd45614a'); + expect(rows[0].delegates[51]).to.equal('f54ce2a222ab3513c49e586464d89a2a7d9959ecce60729289ec0bb6106bd4ce'); + expect(rows[0].delegates[52]).to.equal('b70f1d97cd254e93e2dd7b24567b3dbe06a60b5cbabe3443463c61cb87879b47'); + expect(rows[0].delegates[53]).to.equal('613e4178a65c1194192eaa29910f0ecca3737f92587dd05d58c6435da41220f6'); + expect(rows[0].delegates[54]).to.equal('6cb825715058d2e821aa4af75fbd0da52181910d9fda90fabe73cd533eeb6acb'); + expect(rows[0].delegates[55]).to.equal('76c321881c08b0c2f538abf753044603ab3081f5441fe069c125a6e2803015da'); + expect(rows[0].delegates[56]).to.equal('9172179a88f8cfeeb81518ad31da4397555273b8658eb3ea2d1eca7965d8e615'); + expect(rows[0].delegates[57]).to.equal('41bb70d08312d9c17ec89a4577d30da77d5b936594fc06ccb0646602bed6ad40'); + expect(rows[0].delegates[58]).to.equal('fd039dd8caa03d58c0ecbaa09069403e7faff864dccd5933da50a41973292fa1'); + expect(rows[0].delegates[59]).to.equal('a40c3e1549a9bbea71606ef05b793629923bdb151390145e3730dfe2b28b9217'); + expect(rows[0].delegates[60]).to.equal('f88b86d0a104bda71b2ff4d8234fef4e184ee771a9c2d3a298280790c185231b'); + expect(rows[0].delegates[61]).to.equal('88260051bbe6634431f8a2f3ac66680d1ee9ef1087222e6823d9b4d81170edc7'); + expect(rows[0].delegates[62]).to.equal('0fec636f5866c66f63a0d3db9b00e3cd5ba1b3a0324712c7935ae845dbfcf58a'); + expect(rows[0].delegates[63]).to.equal('677c79b243ed96a8439e8bd193d6ab966ce43c9aa18830d2b9eb8974455d79f8'); + expect(rows[0].delegates[64]).to.equal('a2fc2420262f081d0f6426364301ef40597756e163f6b1fd813eff9b03594125'); + expect(rows[0].delegates[65]).to.equal('de918e28b554600a81cbf119abf5414648b58a8efafbc3b0481df0242684dc1b'); + expect(rows[0].delegates[66]).to.equal('c58078a7d12d81190ef0c5deb7611f97fc923d064647b66b9b25512029a13daf'); + expect(rows[0].delegates[67]).to.equal('619a3113c6cb1d3db7ef9731e6e06b618296815b3cfe7ca8d23f3767198b00ea'); + expect(rows[0].delegates[68]).to.equal('c4d96fbfe80102f01579945fe0c5fe2a1874a7ffeca6bacef39140f9358e3db6'); + expect(rows[0].delegates[69]).to.equal('b851863cf6b4769df5cecca718463173485bb9fe21e20f7cfb0802f5ab5973c2'); + expect(rows[0].delegates[70]).to.equal('a0f768d6476a9cfec1a64a895064fe114b26bd3fb6aeda397ccce7ef7f3f98ef'); + expect(rows[0].delegates[71]).to.equal('5c4a92f575822b2d2deaa4bc0985ec9a57a17719bd5427af634ec1b4bf9c045b'); + expect(rows[0].delegates[72]).to.equal('2493d52fc34ecaaa4a7d0d76e6de9bda24f1b5e11e3363c30a13d59e9c345f82'); + expect(rows[0].delegates[73]).to.equal('2cb967f6c73d9b6b8604d7b199271fed3183ff18ae0bd9cde6d6ef6072f83c05'); + expect(rows[0].delegates[74]).to.equal('9c99976107b5d98e5669452392d912edf33c968e5832b52f2eedcd044b5cc2f2'); + expect(rows[0].delegates[75]).to.equal('ac09bc40c889f688f9158cca1fcfcdf6320f501242e0f7088d52a5077084ccba'); + expect(rows[0].delegates[76]).to.equal('ca1285393e1848ee41ba0c5e47789e5e0c570a7b51d8e2f7f6db37417b892cf9'); + expect(rows[0].delegates[77]).to.equal('6971dc02efc00140fbfcb262dd6f84d2dee533b258427de7017528b2e10ac2b1'); + expect(rows[0].delegates[78]).to.equal('b3953cb16e2457b9be78ad8c8a2985435dedaed5f0dd63443bdfbccc92d09f2d'); + expect(rows[0].delegates[79]).to.equal('63db2063e7760b241b0fe69436834fa2b759746b8237e1aafd2e099a38fc64d6'); + expect(rows[0].delegates[80]).to.equal('f8fa9e01047c19102133d2af06aab6cc377d5665bede412f04f81bcdc368d00e'); + expect(rows[0].delegates[81]).to.equal('93ec60444c28e8b8c0f1a613ed41f518b637280c454188e5500ea4e54e1a2f12'); + expect(rows[0].delegates[82]).to.equal('130649e3d8d34eb59197c00bcf6f199bc4ec06ba0968f1d473b010384569e7f0'); + expect(rows[0].delegates[83]).to.equal('0c0c8f58e7feeaa687d7dc9a5146ea14afe1bc647f518990b197b9f55728effa'); + expect(rows[0].delegates[84]).to.equal('b68f666f1ede5615bf382958a815988a42aea8e4e03fbf0470a57bceac7714db'); + expect(rows[0].delegates[85]).to.equal('32f20bee855238630b0f791560c02cf93014977b4b25c19ef93cd92220390276'); + expect(rows[0].delegates[86]).to.equal('45ab8f54edff6b802335dc3ea5cd5bc5324e4031c0598a2cdcae79402e4941f8'); + expect(rows[0].delegates[87]).to.equal('f147c1cba67acad603309d5004f25d9ab41ae073b318f4c6f972f96106c9b527'); + expect(rows[0].delegates[88]).to.equal('72f0cd8486d8627b5bd4f10c2e592a4512ac58e572edb3e37c0448b3ac7dd405'); + expect(rows[0].delegates[89]).to.equal('326bff18531703385d4037e5585b001e732c4a68afb8f82efe2b46c27dcf05aa'); + expect(rows[0].delegates[90]).to.equal('fbac76743fad9448ed0b45fb4c97a62f81a358908aa14f6a2c76d2a8dc207141'); + expect(rows[0].delegates[91]).to.equal('2f58f5b6b1e2e91a9634dfadd1d6726a5aed2875f33260b6753cb9ec7da72917'); + expect(rows[0].delegates[92]).to.equal('9771b09041466268948626830cbfea5a043527f39174d70e11a80011f386bb57'); + expect(rows[0].delegates[93]).to.equal('76ceefed8f29dd48664b07d207f4bf202122f2ffed6dcefa802d7fe348203b88'); + expect(rows[0].delegates[94]).to.equal('247bf1854471c1a97ccc363c63e876bb6b9a7f06038486048a17196a8a5493dc'); + expect(rows[0].delegates[95]).to.equal('8966b54a95b327651e3103d8adb69579ff50bf22a004a65731a41f7caca2859f'); + expect(rows[0].delegates[96]).to.equal('7fba92f4a2a510ae7301dddddf136e1f8673b54fd0ff0d92ec63f59b68bf4a8f'); + expect(rows[0].delegates[97]).to.equal('2d59fbcce531fb9661cdfa8371c49b6898ce0895fe71da88ffec851c7ed60782'); + expect(rows[0].delegates[98]).to.equal('465085ba003a03d1fed0cfd35b7f3c07927c9db41d32194d273f8fe2fa238faa'); + expect(rows[0].delegates[99]).to.equal('e44b43666fc2a9982c6cd9cb617e4685d7b7cf9fc05e16935f41c7052bb3e15f'); + expect(rows[0].delegates[100]).to.equal('eddeb37070a19e1277db5ec34ea12225e84ccece9e6b2bb1bb27c3ba3999dac7'); + done(); + }).catch(done); }); }); - it('SQL generateDelegatesList() should raise exception for null round', function (done) { - var round = null; - var delegates = ['1']; + describe('exceptions', function () { - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); - }); - }); - - it('SQL generateDelegatesList() should raise exception for negative round', function (done) { - var round = -1; - var delegates = ['1']; + it('should raise exception for round 0', function (done) { + var round = '0'; + var delegates = ['1']; - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); }); - }); - it('SQL generateDelegatesList() should raise exception for empty delegates', function (done) { - var round = 1; - var delegates = []; - - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('cannot determine type of empty array'); - done(); + it('should raise exception for undefined round', function (done) { + var round = undefined; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); }); - }); - it('SQL generateDelegatesListCast() should raise exception for empty delegates', function (done) { - var round = 1; - var delegates = []; + it('should raise exception for null round', function (done) { + var round = null; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); - db.query(sql.generateDelegatesListCast, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); + it('should raise exception for negative round', function (done) { + var round = -1; + var delegates = ['1']; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); }); - }); - it('SQL generateDelegatesList() should raise exception for null delegates', function (done) { - var round = 1; - var delegates = null; + it('should raise exception for empty delegates', function (done) { + var round = 1; + var delegates = []; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('cannot determine type of empty array'); + done(); + }); + }); - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); + it('should raise exception for empty delegates', function (done) { + var round = 1; + var delegates = []; + + db.query(sql.generateDelegatesListCast, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); }); - }); - it('SQL generateDelegatesList() should raise exception for null delegates', function (done) { - var round = 1; - var delegates = undefined; + it('should raise exception for null delegates', function (done) { + var round = 1; + var delegates = null; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); + }); - db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { - done('Should not pass'); - }).catch(function (err) { - expect(err).to.be.an('error'); - expect(err.message).to.contain('Invalid parameters supplied'); - done(); + it('should raise exception for null delegates', function (done) { + var round = 1; + var delegates = undefined; + + db.query(sql.generateDelegatesList, {round: round, delegates: delegates}).then(function (rows) { + done('Should not pass'); + }).catch(function (err) { + expect(err).to.be.an('error'); + expect(err.message).to.contain('Invalid parameters supplied'); + done(); + }); }); }); }); - describe('checking SQL function getDelegatesList()', function () { - it('SQL getDelegatesList() results should be equal to generateDelegatesList() - real 101 delegates from current database', function (done) { + describe('getDelegatesList()', function () { + + it('SQL results should be equal to native - real 101 delegates from current database', function (done) { db.task(function (t) { return t.batch([ t.query(sql.getDelegatesList), From dff4b7a98f36e7f0f6b6a8f808cb1464faf350f6 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 18:11:22 +0200 Subject: [PATCH 83/88] Remove unused bindModules call --- test/unit/logic/vote.js | 1 - test/unit/sql/rounds.js | 1 - 2 files changed, 2 deletions(-) diff --git a/test/unit/logic/vote.js b/test/unit/logic/vote.js index cab3c62942d..3ea9bc5db3d 100644 --- a/test/unit/logic/vote.js +++ b/test/unit/logic/vote.js @@ -125,7 +125,6 @@ describe('vote', function () { }, transactionLogic: ['accountLogic', function (result, cb) { modulesLoader.initLogicWithDb(TransactionLogic, function (err, __transaction) { - __transaction.bindModules(result); cb(err, __transaction); }, { ed: require('../../../helpers/ed'), diff --git a/test/unit/sql/rounds.js b/test/unit/sql/rounds.js index 30b19bcae13..258b4b10311 100644 --- a/test/unit/sql/rounds.js +++ b/test/unit/sql/rounds.js @@ -353,7 +353,6 @@ describe('Rounds-related SQL triggers', function () { // Fire onBind event in every module scope.bus.message('bind', scope.modules); - scope.logic.transaction.bindModules(scope.modules); scope.logic.peers.bindModules(scope.modules); cb(); }] From 452f89216d849fa0d8418ef932d18031463f11da Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 18:12:31 +0200 Subject: [PATCH 84/88] Remove unused bindModules call --- test/unit/logic/transaction.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/logic/transaction.js b/test/unit/logic/transaction.js index b1d9efb6f89..1e5a6af8f09 100644 --- a/test/unit/logic/transaction.js +++ b/test/unit/logic/transaction.js @@ -160,7 +160,6 @@ describe('transaction', function () { }] }, function (err, result) { transactionLogic = result.transactionLogic; - transactionLogic.bindModules(result); attachTransferAsset(transactionLogic, result.accountLogic, done); }); }); From 559ab050863c3f903497d74ce60add0eee292af3 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Mon, 7 Aug 2017 18:29:19 +0200 Subject: [PATCH 85/88] Improve description in tests --- test/unit/helpers/pg-notify.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/helpers/pg-notify.js b/test/unit/helpers/pg-notify.js index 3a46c64b9f1..985979fa8c4 100644 --- a/test/unit/helpers/pg-notify.js +++ b/test/unit/helpers/pg-notify.js @@ -211,7 +211,7 @@ describe('helpers/pg-notify', function () { }); }); - it('should reconnect successfully if it\'s possible', function (done) { + it('should reconnect successfully if possible', function (done) { // Spy private functions var setListeners = pg_notify.__get__('setListeners'); var connection = pg_notify.__get__('connection'); From c13d9191b966c93677569714ca28cd57fcec86a6 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 8 Aug 2017 08:31:10 +0200 Subject: [PATCH 86/88] Fix code standards in pg-notify helper --- helpers/pg-notify.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/pg-notify.js b/helpers/pg-notify.js index 43e63b1cd94..dbed633d612 100644 --- a/helpers/pg-notify.js +++ b/helpers/pg-notify.js @@ -26,7 +26,7 @@ function onNotification (data) { logger.info('pg-notify: Round closed'); try { data.payload = JSON.parse(data.payload); - } catch(e) { + } catch (e) { logger.error('pg-notify: Unable to parse JSON', {err: e, data: data}); return; } @@ -34,7 +34,7 @@ function onNotification (data) { logger.warn('pg-notify: Round reopened'); try { data.payload = JSON.parse(data.payload); - } catch(e) { + } catch (e) { logger.error('pg-notify: Unable to parse JSON', {err: e, data: data}); return; } From 82e03cb8b1e024bbbfff51db48812c6ba771a1c3 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 8 Aug 2017 08:32:08 +0200 Subject: [PATCH 87/88] Added TODO/FIXME comments for things that need to be done --- modules/blocks/process.js | 2 ++ modules/loader.js | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/blocks/process.js b/modules/blocks/process.js index f06d8469984..75fe51c65af 100644 --- a/modules/blocks/process.js +++ b/modules/blocks/process.js @@ -125,6 +125,8 @@ Process.prototype.getCommonBlock = function (peer, height, cb) { }); }; +// FIXME: That function no longer works because rounds rewards are applied by triggers +// TODO: Remove that function as part of #544 /** * Loads full blocks from database, used when rebuilding blockchain, snapshotting diff --git a/modules/loader.js b/modules/loader.js index 69db34af0fa..aa1746c6820 100644 --- a/modules/loader.js +++ b/modules/loader.js @@ -405,6 +405,7 @@ __private.loadBlockChain = function () { } } + // TODO: Remove snapshot verification as part of #544 function verifySnapshot (count, round) { if (library.config.loading.snapshot !== undefined || library.config.loading.snapshot > 0) { library.logger.info('Snapshot mode enabled'); @@ -416,7 +417,8 @@ __private.loadBlockChain = function () { library.config.loading.snapshot = (round > 1) ? (round - 1) : 1; } - //modules.rounds.setSnapshotRounds(library.config.loading.snapshot); + // FIXME: Because round module is removed - that no longer works + // modules.rounds.setSnapshotRounds(library.config.loading.snapshot); } library.logger.info('Snapshotting to end of round: ' + library.config.loading.snapshot); @@ -452,6 +454,9 @@ __private.loadBlockChain = function () { var missed = !(countMemAccounts.count); + // FIXME: That check is not passing because dependant field in mem_accounts is not always updated + // TODO: Remove countMemAccounts check as part of #544 + // if (missed) { // return reload(count, 'Detected missed blocks in mem_accounts'); // } From 0b2365acc9deed1cd40f000f975851a6ec7934e0 Mon Sep 17 00:00:00 2001 From: Mariusz Serek Date: Tue, 8 Aug 2017 08:55:00 +0200 Subject: [PATCH 88/88] Remove unused rounds module references in tests --- modules/delegates.js | 1 - test/unit/modules/accounts.js | 4 ---- test/unit/modules/transactions.js | 14 +++++++------- test/unit/sql/delegatesList.js | 13 ++++++++----- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/modules/delegates.js b/modules/delegates.js index eb9da794fd5..d02d13422b7 100644 --- a/modules/delegates.js +++ b/modules/delegates.js @@ -563,7 +563,6 @@ Delegates.prototype.onBind = function (scope) { delegates: scope.delegates, loader: scope.loader, peers: scope.peers, - rounds: scope.rounds, transactions: scope.transactions, transport: scope.transport }; diff --git a/test/unit/modules/accounts.js b/test/unit/modules/accounts.js index 78cfc3636e8..24ad507474e 100644 --- a/test/unit/modules/accounts.js +++ b/test/unit/modules/accounts.js @@ -13,7 +13,6 @@ var expect = require('chai').expect; var constants = require('../../../helpers/constants.js'); var ws = require('../../common/wsCommunication.js'); -var Rounds = require('../../../modules/rounds.js'); var AccountLogic = require('../../../logic/account.js'); var AccountModule = require('../../../modules/accounts.js'); var TransactionLogic = require('../../../logic/transaction.js'); @@ -62,9 +61,6 @@ describe('account', function () { before(function (done) { async.auto({ - rounds: function (cb) { - modulesLoader.initModule(Rounds, modulesLoader.scope,cb); - }, accountLogic: function (cb) { modulesLoader.initLogicWithDb(AccountLogic, cb); }, diff --git a/test/unit/modules/transactions.js b/test/unit/modules/transactions.js index 8eb75a79bce..46d8d86ac0e 100644 --- a/test/unit/modules/transactions.js +++ b/test/unit/modules/transactions.js @@ -53,13 +53,13 @@ describe('transactions', function () { var dappAccount = node.randomAccount(); - function attachAllAssets (transactionLogic, delegatesModule, roundsModule, accountsModule) { + function attachAllAssets (transactionLogic, delegatesModule, accountsModule) { var sendLogic = transactionLogic.attachAssetType(transactionTypes.SEND, new TransferLogic()); - sendLogic.bind(accountsModule, /* rounds */ null); + sendLogic.bind(accountsModule); expect(sendLogic).to.be.an.instanceof(TransferLogic); var voteLogic = transactionLogic.attachAssetType(transactionTypes.VOTE, new VoteLogic(modulesLoader.logger, modulesLoader.scope.schema)); - voteLogic.bind(delegatesModule, /* rounds */ null); + voteLogic.bind(delegatesModule); expect(voteLogic).to.be.an.instanceof(VoteLogic); var delegateLogic = transactionLogic.attachAssetType(transactionTypes.DELEGATE, new DelegateLogic(modulesLoader.scope.schema)); @@ -71,18 +71,18 @@ describe('transactions', function () { expect(signatureLogic).to.be.an.instanceof(SignatureLogic); var multiLogic = transactionLogic.attachAssetType(transactionTypes.MULTI, new MultisignatureLogic(modulesLoader.scope.schema, modulesLoader.scope.network, transactionLogic, modulesLoader.logger)); - multiLogic.bind(/* rounds */ null, delegatesModule); + multiLogic.bind(accountsModule); expect(multiLogic).to.be.an.instanceof(MultisignatureLogic); var dappLogic = transactionLogic.attachAssetType(transactionTypes.DAPP, new DappLogic(modulesLoader.db, modulesLoader.logger, modulesLoader.scope.schema, modulesLoader.scope.network)); expect(dappLogic).to.be.an.instanceof(DappLogic); var inTransferLogic = transactionLogic.attachAssetType(transactionTypes.IN_TRANSFER, new InTransferLogic(modulesLoader.db, modulesLoader.scope.schema)); - inTransferLogic.bind(accountsModule, /* rounds */ null, /* sharedApi */ null); + inTransferLogic.bind(accountsModule, /* sharedApi */ null); expect(inTransferLogic).to.be.an.instanceof(InTransferLogic); var outTransfer = transactionLogic.attachAssetType(transactionTypes.OUT_TRANSFER, new OutTransferLogic(modulesLoader.db, modulesLoader.scope.schema, modulesLoader.logger)); - outTransfer.bind(accountsModule, /* rounds */ null, /* sharedApi */ null); + outTransfer.bind(accountsModule, /* sharedApi */ null); expect(outTransfer).to.be.an.instanceof(OutTransferLogic); return transactionLogic; } @@ -161,7 +161,7 @@ describe('transactions', function () { loader: result.loaderModule }); - attachAllAssets(result.transactionLogic, result.delegateModule, /*rounds*/null, result.accountsModule); + attachAllAssets(result.transactionLogic, result.delegateModule, result.accountsModule); done(); }); }); diff --git a/test/unit/sql/delegatesList.js b/test/unit/sql/delegatesList.js index de5c4b9699e..fb181336bb0 100644 --- a/test/unit/sql/delegatesList.js +++ b/test/unit/sql/delegatesList.js @@ -1,12 +1,12 @@ 'use strict'; -var chai = require('chai'); +var chai = require('chai'); +var crypto = require('crypto'); var expect = require('chai').expect; -var crypto = require('crypto'); -var sql = require('../../sql/delegatesList.js'); var modulesLoader = require('../../common/initModule').modulesLoader; -var db; +var slots = require('../../../helpers/slots.js'); +var sql = require('../../sql/delegatesList.js'); function generateDelegatesList (round, delegates) { var i, x, n, old, len; @@ -34,6 +34,7 @@ function generateDelegatesList (round, delegates) { }; describe('Delegate list SQL functions', function () { + var db; before(function (done) { modulesLoader.getDbConnection(function (err, db_handle) { @@ -45,6 +46,7 @@ describe('Delegate list SQL functions', function () { }); }); + // Old logic - not used, for reference only function getKeysSortByVote (cb) { library.db.query(sql.delegateList).then(function (rows) { return setImmediate(cb, null, rows.map(function (el) { @@ -55,13 +57,14 @@ describe('Delegate list SQL functions', function () { }); }; + // Old logic - not used, for reference only function generateDelegateList (height, cb) { getKeysSortByVote(function (err, truncDelegateList) { if (err) { return setImmediate(cb, err); } - var seedSource = modules.rounds.calc(height).toString(); + var seedSource = slots.calcRound(height).toString(); var currentSeed = crypto.createHash('sha256').update(seedSource, 'utf8').digest(); for (var i = 0, delCount = truncDelegateList.length; i < delCount; i++) {