Merge branch 'development' into 2404-allow_fuzzy_search_on_data_filter

.npmignore

@@ -6,6 +6,7 @@ pids/*
 release/
 scripts/**/*
 !scripts/update_config.js
+!scripts/generate_config.js
 tasks/
 test/
 Grunt*

.npmrc

@@ -1 +1,2 @@
 save-prefix = "="
+engine-strict = true

Dockerfile

@@ -18,6 +18,7 @@ RUN npm install
 FROM node:8
 
 ENV NODE_ENV=production
+ENV WFI_COMMIT=e34c502a3efe0e8b8166ea6148d55b73da5c8401
 ENV WFI_SHA=0f75de5c9d9c37a933bb9744ffd710750d5773892930cfe40509fa505788835c
 
 RUN echo "deb http://ftp.debian.org/debian jessie-backports main" >/etc/apt/sources.list.d/backports.list && \
@@ -30,10 +31,10 @@ RUN groupadd --gid 1100 lisk && \
 COPY --from=builder --chown=lisk:lisk /home/lisk/lisk/ /home/lisk/lisk/
 # git repository needed for build; cannot be added to .dockerignore
 RUN rm -rf /home/lisk/lisk/.git && \
-    mkdir /home/lisk/lisk/logs && \
+    mkdir -p /home/lisk/lisk/logs && \
     chown lisk:lisk /home/lisk/lisk/logs
 
-ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh /home/lisk/wait-for-it.sh
+ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/${WFI_COMMIT}/wait-for-it.sh /home/lisk/wait-for-it.sh
 RUN if [ x"$( sha256sum /home/lisk/wait-for-it.sh |awk '{print $1}' )" = x"${WFI_SHA}" ]; then \
       chmod 0755 /home/lisk/wait-for-it.sh; \
     else \

Jenkinsfile

@@ -32,8 +32,7 @@ def initializeNode() {
 def buildDependencies() {
 	try {
 		sh '''
-		rsync -axl -e "ssh -oUser=jenkins" master-01:/var/lib/jenkins/lisk/node_modules/ "$WORKSPACE/node_modules/"
-		npm install
+		npm ci
 		'''
 	} catch (err) {
 		echo "Error: ${err}"
@@ -170,13 +169,6 @@ lock(resource: "Lisk-Core-Nodes-v8", inversePrecedence: true) {
 					try {
 						deleteDir()
 						checkout scm
-						sh """
-						if [ ${params.JENKINS_PROFILE} = "jenkins-extensive" ]; then
-							rm -Rf "${env.WORKSPACE}/node_modules/"
-							npm install
-							rsync -axl --delete "${env.WORKSPACE}/node_modules/" /var/lib/jenkins/lisk/node_modules/
-						fi
-						"""
 					} catch (err) {
 						echo "Error: ${err}"
 						currentBuild.result = 'FAILURE'

README.md

@@ -355,12 +355,16 @@ Each of that option can be appended on command line. There are also few `ENV` va
 | <pre nowrap>--log<br> -l</pre>       | LISK_FILE_LOG_LEVEL    | fileLogLevel             | Log level for file output                                                                                                                                                         |
 |                                      | LISK_CONSOLE_LOG_LEVEL | consoleLogLevel          | Log level for console output                                                                                                                                                      |
 |                                      | LISK_CACHE_ENABLED     | cacheEnabled             | Enable or disable cache. Must be set to true/false                                                                                                                                |
-| <pre nowrap>--database<br> -d</pre>  | LISK_DB_NAME           | db.database              | PostgreSQL database name to connect                                                                                                                                               |
+| <pre nowrap>--database<br> -d</pre>  | LISK_DB_NAME           | db.database              | PostgreSQL database name to connect to                                                                                                                                            |
 |                                      | LISK_DB_HOST           | db.host                  | PostgreSQL database host name                                                                                                                                                     |
 |                                      | LISK_DB_PORT           | db.port                  | PostgreSQL database port                                                                                                                                                          |
-|                                      | LISK_DB_USER           | db.user                  | PostgreSQL database username to connect                                                                                                                                           |
-|                                      | LISK_DB_PASSWORD       | db.password              | PostgreSQL database password to connect                                                                                                                                           |
-| <pre nowrap>--peers<br> -p</pre>     | LISK_PEERS             | peers.list               | Comma separated list of peers to connect in the format `192.168.99.100:5000,172.169.99.77:5000`                                                                                   |
+|                                      | LISK_DB_USER           | db.user                  | PostgreSQL database username to connect to                                                                                                                                        |
+|                                      | LISK_DB_PASSWORD       | db.password              | PostgreSQL database password to connect to                                                                                                                                        |
+| <pre nowrap>--redis<br> -r</pre>     | LISK_REDIS_HOST        | redis.host               | Redis host name                                                                                                                                                                   |
+|                                      | LISK_REDIS_PORT        | redis.port               | Redis port                                                                                                                                                                        |
+|                                      | LISK_REDIS_DB_NAME     | redis.db                 | Redis database name to connect to                                                                                                                                                 |
+|                                      | LISK_REDIS_DB_PASSWORD | redis.password           | Redis database password to connect to                                                                                                                                             |
+| <pre nowrap>--peers<br> -p</pre>     | LISK_PEERS             | peers.list               | Comma separated list of peers to connect to in the format `192.168.99.100:5000,172.169.99.77:5000`                                                                                |
 |                                      | LISK_API_PUBLIC        | api.access.public        | Enable or disable public access of http API. Must be set to true/false                                                                                                            |
 |                                      | LISK_API_WHITELIST     | api.access.whiteList     | Comma separated list of IPs to enable API access. Format `192.168.99.100,172.169.99.77`                                                                                           |
 |                                      | LISK_FORGING_DELEGATES | forging.delegates        | Comma separated list of delegates to load in the format _publicKey&#x7c;encryptedPassphrase,publicKey2&#x7c;encryptedPassphrase2_                                                 |
@@ -435,12 +439,14 @@ There are couple of command line scripts that facilitate users of lisk to perfor
 This script will help you to generate unified version of configuration file for any network. Here is the usage of the script:
 
 ```
-Usage: generate_config [options] <network>
+Usage: generate_config [options]
 
 Options:
 
-  -h, --help     output usage information
-  -V, --version  output the version number
+-h, --help               output usage information
+-V, --version            output the version number
+-c, --config [config]    custom config file
+-n, --network [network]  specify the network or use LISK_NETWORK
 ```
 
 Argument `network` is required and can by `devnet`, `testnet`, `mainnet` or any other network folder available under `./config` directory.
@@ -454,13 +460,13 @@ Usage: update_config [options] <input_file> <from_version> [to_version]
 
 Options:
 
--h, --help     output usage information
--V, --version  output the version number
---output       Output file path
---diff         Show only difference from default config file.
+-h, --help               output usage information
+-V, --version            output the version number
+-n, --network [network]  specify the network or use LISK_NETWORK
+-o, --output [output]    output file path
 ```
 
-As you can see from the usage guide, `input_file` and `from_version` are required. So if you have a config file, you must be aware to which version of Lisk this belongs. You can skip the `to_version` argument and it will apply changes up-to latest version of Lisk. If you don't specify `--output` path the final config json will be printed to console. Option `--diff` is useful if you just want to know what are the changes in your version compared to default config located in `./config/default/config.json`.
+As you can see from the usage guide, `input_file` and `from_version` are required. If you skip `to_version` argument changes in config.json will be applied up to the latest version of Lisk Core. If you do not specify `--output` path the final config.json will be printed to stdout. If you do not specify `--network` argument you will have to load it from `LISK_NETWORK` env variable.
 
 #### Console
 

docker/Makefile

@@ -5,18 +5,27 @@ lisk_net := $(shell grep ^ENV_LISK_NETWORK= .env |tail -n 1 |awk -F = '{ gsub("n
 ifeq ($(lisk_net),)
 $(error .env file does not exist or ENV_LISK_NETWORK variable is not set.)
 endif
+lisk_db := $(shell grep ^ENV_LISK_DB_DATABASE= .env |tail -n 1 |awk -F = '{ gsub("net$$", ""); print $$2 }')
+ifeq ($(lisk_net),)
+$(error .env file does not exist or ENV_LISK_DB_DATABASE variable is not set.)
+endif
 
 %blockchain.db.gz:
+ifeq ($(lisk_net),$(filter $(lisk_net),main test))
 	curl --output $(lisk_net)_blockchain.db.gz https://downloads.lisk.io/lisk/$(lisk_net)/blockchain.db.gz
+else
+	touch $(lisk_net)_blockchain.db
+	gzip $(lisk_net)_blockchain.db
+endif
 
 up:
 	docker-compose up -d
 
 coldstart: $(lisk_net)_blockchain.db.gz up
 	docker-compose stop lisk
 	docker-compose start db
-	docker-compose run --rm task dropdb --if-exists lisk
-	docker-compose run --rm task createdb lisk
+	docker-compose run --rm task dropdb --if-exists $(lisk_db)
+	docker-compose run --rm task createdb $(lisk_db)
 	@echo Restoring database snapshot. This can take a few minutes.
 	gzip --decompress --to-stdout $(lisk_net)_blockchain.db.gz |docker-compose run --rm task psql >/dev/null
 	docker-compose start lisk

docker/docker-compose.yml

@@ -20,18 +20,26 @@ services:
       - LISK_DB_USER=${ENV_LISK_DB_USER}
       - LISK_DB_PASSWORD=${ENV_LISK_DB_PASSWORD}
       - LISK_DB_NAME=${ENV_LISK_DB_DATABASE}
+# To enable caching:
+#  - set LISK_CACHE_ENABLED to true,
+#  - uncomment the next two lines and
+#  - uncomment the "redis" block below
       - LISK_CACHE_ENABLED=false
-# uncomment the following line if you want to see debug messages in the docker logs
+      #- LISK_REDIS_HOST=redis
+      #- LISK_REDIS_PORT=6379
+# Uncomment the following line if you want to see debug messages in the docker logs
       #- LISK_CONSOLE_LOG_LEVEL=debug
-# uncomment the following line if you want the API to be public
+# Uncomment the following line if you want the API to be public
       #- LISK_API_PUBLIC=true
-# uncomment and change the following line if you want to forge:
-# you will need to use your own encrypted key (use lisk-commander to encrypt it)
-      #- LISK_FORGING_DELEGATES=salt=d25cb9fd7c0b9935f4abc59388aceb74&cipherText=122096545efdd048eda72552883db00ff62ca3925a8d870a5c6380a043b45372adea1698755b73b7ef8c2f2269907b04c77aac38f7094f08c147eccbc6a3bb62c374252e39acf4519b8d&iv=03781f7ba5ba16ffd8d93edb&tag=55e7d26c9dbd35d011938d38fea13602&version=1:eadd8a8bb1a7283147aa7006c8b5f9f2b871d789f36207dfd44039ed44c1d572
-# uncomment the following lines to allow access to the API
-# change 'YOUR_IP' to you actual IP address
-      #- LISK_API_WHITELIST=YOUR_IP
-      #- LISK_FORGING_WHITELIST=YOUR_IP
+# Uncomment and change the following line if you want to forge.
+# You will need to use your own encrypted key (use lisk-commander to encrypt it)
+      #- LISK_FORGING_DELEGATES=publicKey1|encryptedPassphrase1,publicKey2|encryptedPassphrase2
+# Uncomment the following lines to allow access to the API
+# Change 'YOUR_IP' to you actual IP address
+      #- LISK_API_WHITELIST=IP1,IP2
+      #- LISK_FORGING_WHITELIST=IP1,IP2
+# Uncomment the following to set your own peer:
+      #- LISK_PEERS=PEER_IP_1:PORT_1,PEER_IP_2:PORT_2
 
   db:
     image: postgres:10-alpine
@@ -56,6 +64,14 @@ services:
       - PGHOST=db
     command: /bin/true
 
+# Uncomment if you enabled caching above
+#  redis:
+#    image: redis:alpine
+#    networks:
+#      - lisk
+#    restart: on-failure
+#    command: --maxclients 500 --tcp-backlog 128 --maxmemory-policy allkeys-lru --save ""
+
 networks:
   lisk:
 

helpers/config.js

@@ -51,6 +51,7 @@ function Config(packageJson, parseCommandLineOptions = true) {
 		.option('-p, --port <port>', 'listening port number')
 		.option('-h, --http-port <httpPort>', 'listening HTTP port number')
 		.option('-d, --database <database>', 'database name')
+		.option('-r, --redis <redis host>', 'Redis host name')
 		.option('-a, --address <ip>', 'listening host name or ip')
 		.option('-x, --peers [peers...]', 'peers list')
 		.option('-l, --log <level>', 'log level')
@@ -106,6 +107,12 @@ function Config(packageJson, parseCommandLineOptions = true) {
 			user: getenv(process.env.LISK_DB_USER),
 			password: getenv(process.env.LISK_DB_PASSWORD),
 		},
+		redis: {
+			host: program.redis || getenv(process.env.LISK_REDIS_HOST),
+			port: parseInt(getenv(process.env.LISK_REDIS_PORT)) || null,
+			database: getenv(process.env.LISK_REDIS_DB_NAME),
+			password: getenv(process.env.LISK_REDIS_DB_PASSWORD),
+		},
 		api: {
 			access: {
 				public: getenv(process.env.LISK_API_PUBLIC, null, true),

helpers/json_history.js

@@ -137,9 +137,9 @@ function JSONHistory(title, logger) {
 
 		// Get the version from which to start the migration
 		// Skip the matched version, as json is already in that particular version
-		let startFromVersion = versions.findIndex(version =>
-			semver.satisfies(fromVersion, version)
-		);
+		// Used ltr to to match the range versions e.g. 1.1.x
+		let startFromVersion =
+			versions.findIndex(version => semver.ltr(fromVersion, version)) - 1;
 
 		if (startFromVersion === -1) {
 			throw new Error(
@@ -150,9 +150,14 @@ function JSONHistory(title, logger) {
 		startFromVersion += includeStart ? 0 : 1;
 
 		// Apply changes till that version
-		const tillVersion =
-			versions.findIndex(version => semver.satisfies(toVersion, version)) ||
-			versions.length - 1;
+		// Used ltr to to match the range versions e.g. 1.1.x
+		let tillVersion =
+			versions.findIndex(version => semver.ltr(toVersion, version)) - 1;
+
+		// If no matching version found then migrate till last version in the list
+		if (tillVersion < 0) {
+			tillVersion = versions.length - 1;
+		}
 
 		// Clone the provided json to avoid changes into source
 		let compiledJson = Object.assign({}, json);

logic/broadcaster.js

@@ -225,10 +225,32 @@ __private.filterQueue = function(cb) {
 			if (broadcast.options.immediate) {
 				return setImmediate(filterCb, null, false);
 			} else if (broadcast.options.data) {
-				const transaction =
-					broadcast.options.data.transaction ||
-					broadcast.options.data.signature;
-				return __private.filterTransaction(transaction, filterCb);
+				let transactionId;
+				if (broadcast.options.data.transaction) {
+					// Look for a transaction of a given "id" when broadcasting transactions
+					transactionId = broadcast.options.data.transaction.id;
+				} else if (broadcast.options.data.signature) {
+					// Look for a corresponding "transactionId" of a given signature when broadcasting signatures
+					transactionId = broadcast.options.data.signature.transactionId;
+				}
+				if (!transactionId) {
+					return setImmediate(filterCb, null, false);
+				}
+				// Broadcast if transaction is in transaction pool
+				if (modules.transactions.transactionInPool(transactionId)) {
+					return setImmediate(filterCb, null, true);
+				}
+				// Don't broadcast if transaction is already confirmed
+				return library.logic.transaction.checkConfirmed(
+					{ id: transactionId },
+					// In case of SQL error:
+					// err = true, isConfirmed = false => return false
+					// In case transaction exists in "trs" table:
+					// err = null, isConfirmed = true => return false
+					// In case transaction doesn't exists in "trs" table:
+					// err = null, isConfirmed = false => return true
+					(err, isConfirmed) => filterCb(null, !err && !isConfirmed)
+				);
 			}
 			return setImmediate(filterCb, null, true);
 		},
@@ -241,28 +263,6 @@ __private.filterQueue = function(cb) {
 	);
 };
 
-/**
- * Checks if transaction is in pool or confirm it.
- *
- * @private
- * @param {transaction} transaction
- * @param {function} cb
- * @returns {SetImmediate} null, boolean
- * @todo Add description for the params
- */
-__private.filterTransaction = function(transaction, cb) {
-	if (transaction !== undefined) {
-		if (modules.transactions.transactionInPool(transaction.id)) {
-			return setImmediate(cb, null, true);
-		}
-		return library.logic.transaction.checkConfirmed(
-			transaction,
-			(err, isConfirmed) => setImmediate(cb, null, !err && !isConfirmed)
-		);
-	}
-	return setImmediate(cb, null, false);
-};
-
 /**
  * Groups broadcasts by api.
  *

logic/transaction.js

@@ -279,7 +279,8 @@ class Transaction {
 	/**
 	 * Returns true if a transaction was confirmed.
 	 *
-	 * @param {transaction} transaction
+	 * @param {Object} transaction
+	 * @param {string} transaction.id - only transaction id is necessary here
 	 * @param {function} cb
 	 * @returns {SetImmediate} error, isConfirmed
 	 * @todo Add description for the params