Add ability to save inclusion proofs at every new block (#9197)

* ♻️ Add ability to save inclusion proofs at every new block
- Add config to take keepInclusionProofsForHeights and inclusionProofKeys
- Add DB update to get/set/delete inclusionProofs
- Add handler in new block event to save inclusion proofs
- Add endpoint for chain to get inclusion proofs by height

* ♻️ Move inclusion proof collection to consensus

* ✅ Fix tests

* ✅ Add tests for get/set inclusion proofs

* ✅ Update snapshot

* ♻️ Save inclusionProofs on both block receive and generate

* ✅ Add unit tests for saving inclusion proofs

elements/lisk-chain/src/chain.ts

@@ -18,6 +18,7 @@ import * as createDebug from 'debug';
 import { regularMerkleTree } from '@liskhq/lisk-tree';
 import {
 	DEFAULT_KEEP_EVENTS_FOR_HEIGHTS,
+	DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS,
 	DEFAULT_MAX_BLOCK_HEADER_CACHE,
 	DEFAULT_MIN_BLOCK_HEADER_CACHE,
 	GENESIS_BLOCK_VERSION,
@@ -40,6 +41,7 @@ interface ChainConstructor {
 	readonly maxTransactionsSize: number;
 	readonly minBlockHeaderCache?: number;
 	readonly maxBlockHeaderCache?: number;
+	readonly keepInclusionProofsForHeights: number;
 }
 
 interface ChainInitArgs {
@@ -61,6 +63,7 @@ export class Chain {
 		readonly minBlockHeaderCache: number;
 		readonly maxBlockHeaderCache: number;
 		readonly keepEventsForHeights: number;
+		readonly keepInclusionProofsForHeights: number;
 	};
 
 	private _lastBlock?: Block;
@@ -74,6 +77,7 @@ export class Chain {
 		keepEventsForHeights = DEFAULT_KEEP_EVENTS_FOR_HEIGHTS,
 		minBlockHeaderCache = DEFAULT_MIN_BLOCK_HEADER_CACHE,
 		maxBlockHeaderCache = DEFAULT_MAX_BLOCK_HEADER_CACHE,
+		keepInclusionProofsForHeights = DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS,
 	}: ChainConstructor) {
 		// Register codec schema
 		codec.addSchema(blockSchema);
@@ -86,6 +90,7 @@ export class Chain {
 			maxBlockHeaderCache,
 			minBlockHeaderCache,
 			keepEventsForHeights,
+			keepInclusionProofsForHeights,
 		};
 	}
 
@@ -118,6 +123,7 @@ export class Chain {
 			minBlockHeaderCache: this.constants.minBlockHeaderCache,
 			maxBlockHeaderCache: this.constants.maxBlockHeaderCache,
 			keepEventsForHeights: this.constants.keepEventsForHeights,
+			keepInclusionProofsForHeights: this.constants.keepInclusionProofsForHeights,
 		});
 	}
 

elements/lisk-chain/src/constants.ts

@@ -15,6 +15,7 @@
 import { utils } from '@liskhq/lisk-cryptography';
 
 export const DEFAULT_KEEP_EVENTS_FOR_HEIGHTS = 300;
+export const DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS = 300;
 export const DEFAULT_MIN_BLOCK_HEADER_CACHE = 309;
 export const DEFAULT_MAX_BLOCK_HEADER_CACHE = 515;
 

elements/lisk-chain/src/data_access/data_access.ts

@@ -12,7 +12,7 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 
-import { Database, NotFoundError } from '@liskhq/lisk-db';
+import { Database, NotFoundError, Proof } from '@liskhq/lisk-db';
 import { Transaction } from '../transaction';
 import { RawBlock } from '../types';
 import { BlockHeader } from '../block_header';
@@ -29,6 +29,7 @@ interface DAConstructor {
 	readonly minBlockHeaderCache: number;
 	readonly maxBlockHeaderCache: number;
 	readonly keepEventsForHeights: number;
+	readonly keepInclusionProofsForHeights: number;
 }
 
 export class DataAccess {
@@ -40,8 +41,9 @@ export class DataAccess {
 		minBlockHeaderCache,
 		maxBlockHeaderCache,
 		keepEventsForHeights,
+		keepInclusionProofsForHeights,
 	}: DAConstructor) {
-		this._storage = new StorageAccess(db, { keepEventsForHeights });
+		this._storage = new StorageAccess(db, { keepEventsForHeights, keepInclusionProofsForHeights });
 		this._blocksCache = new BlockCache(minBlockHeaderCache, maxBlockHeaderCache);
 	}
 
@@ -243,6 +245,16 @@ export class DataAccess {
 		return events;
 	}
 
+	public async getInclusionProofs(height: number): Promise<Proof> {
+		const proofs = await this._storage.getInclusionProofs(height);
+
+		return proofs;
+	}
+
+	public async setInclusionProofs(proof: Proof, height: number): Promise<void> {
+		await this._storage.setInclusionProofs(proof, height);
+	}
+
 	public async isBlockPersisted(blockId: Buffer): Promise<boolean> {
 		const isPersisted = await this._storage.isBlockPersisted(blockId);
 

elements/lisk-chain/src/data_access/storage.ts

@@ -11,7 +11,7 @@
  *
  * Removal or modification of this copyright notice is prohibited.
  */
-import { Batch, Database, NotFoundError } from '@liskhq/lisk-db';
+import { Batch, Database, NotFoundError, Proof } from '@liskhq/lisk-db';
 import { codec } from '@liskhq/lisk-codec';
 import { utils } from '@liskhq/lisk-cryptography';
 import { RawBlock, StateDiff } from '../types';
@@ -26,11 +26,16 @@ import {
 	DB_KEY_FINALIZED_HEIGHT,
 	DB_KEY_BLOCK_ASSETS_BLOCK_ID,
 	DB_KEY_BLOCK_EVENTS,
+	DB_KEY_INCLUSION_PROOFS,
 } from '../db_keys';
 import { concatDBKeys, uint32BE } from '../utils';
-import { stateDiffSchema } from '../schema';
+import { inclusionProofSchema, stateDiffSchema } from '../schema';
 import { CurrentState } from '../state_store';
-import { DEFAULT_KEEP_EVENTS_FOR_HEIGHTS, MAX_UINT32 } from '../constants';
+import {
+	DEFAULT_KEEP_EVENTS_FOR_HEIGHTS,
+	DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS,
+	MAX_UINT32,
+} from '../constants';
 
 const bytesArraySchema = {
 	$id: '/liskChain/bytesarray',
@@ -61,15 +66,19 @@ export const encodeByteArray = (val: Buffer[]): Buffer =>
 
 interface StorageOption {
 	keepEventsForHeights?: number;
+	keepInclusionProofsForHeights?: number;
 }
 
 export class Storage {
 	private readonly _db: Database;
 	private readonly _keepEventsForHeights: number;
+	private readonly _keepInclusionProofsForHeights: number;
 
 	public constructor(db: Database, options?: StorageOption) {
 		this._db = db;
 		this._keepEventsForHeights = options?.keepEventsForHeights ?? DEFAULT_KEEP_EVENTS_FOR_HEIGHTS;
+		this._keepInclusionProofsForHeights =
+			options?.keepInclusionProofsForHeights ?? DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS;
 	}
 
 	/*
@@ -257,6 +266,29 @@ export class Storage {
 		}
 	}
 
+	public async setInclusionProofs(proof: Proof, height: number): Promise<void> {
+		const proofBytes = codec.encode(inclusionProofSchema, proof);
+		await this._db.set(concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(height)), proofBytes);
+	}
+
+	public async getInclusionProofs(height: number): Promise<Proof> {
+		try {
+			const proofBytes = await this._db.get(
+				concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(height)),
+			);
+
+			return codec.decode(inclusionProofSchema, proofBytes);
+		} catch (error) {
+			if (!(error instanceof NotFoundError)) {
+				throw error;
+			}
+			return {
+				queries: [],
+				siblingHashes: [],
+			};
+		}
+	}
+
 	public async getTempBlocks(): Promise<Buffer[]> {
 		const stream = this._db.createReadStream({
 			gte: concatDBKeys(DB_KEY_TEMPBLOCKS_HEIGHT, uint32BE(0)),
@@ -419,6 +451,7 @@ export class Storage {
 			batch.del(concatDBKeys(DB_KEY_TRANSACTIONS_BLOCK_ID, id));
 		}
 		batch.del(concatDBKeys(DB_KEY_BLOCK_EVENTS, heightBuf));
+		batch.del(concatDBKeys(DB_KEY_INCLUSION_PROOFS, heightBuf));
 		if (assets.length > 0) {
 			batch.del(concatDBKeys(DB_KEY_BLOCK_ASSETS_BLOCK_ID, id));
 		}
@@ -486,6 +519,22 @@ export class Storage {
 				);
 			}
 		}
+
+		if (this._keepInclusionProofsForHeights > -1) {
+			// inclusion proofs are removed only if finalized and below height - keepInclusionProofsForHeights
+			const minInclusionProofDeleteHeight = Math.min(
+				finalizedHeight,
+				Math.max(0, currentHeight - this._keepInclusionProofsForHeights),
+			);
+			if (minInclusionProofDeleteHeight > 0) {
+				const endHeight = Buffer.alloc(4);
+				endHeight.writeUInt32BE(minInclusionProofDeleteHeight - 1, 0);
+				await this._clear(
+					Buffer.concat([DB_KEY_INCLUSION_PROOFS, Buffer.alloc(4, 0)]),
+					Buffer.concat([DB_KEY_INCLUSION_PROOFS, endHeight]),
+				);
+			}
+		}
 	}
 
 	private async _getBlockAssets(blockID: Buffer): Promise<Buffer[]> {

elements/lisk-chain/src/db_keys.ts

@@ -20,6 +20,7 @@ export const DB_KEY_TRANSACTIONS_ID = Buffer.from([6]);
 export const DB_KEY_TEMPBLOCKS_HEIGHT = Buffer.from([7]);
 export const DB_KEY_BLOCK_ASSETS_BLOCK_ID = Buffer.from([8]);
 export const DB_KEY_BLOCK_EVENTS = Buffer.from([9]);
+export const DB_KEY_INCLUSION_PROOFS = Buffer.from([11]);
 
 export const DB_KEY_STATE_STORE = Buffer.from([10]);
 

elements/lisk-chain/src/schema.ts

@@ -137,6 +137,42 @@ export const blockAssetSchema = {
 	},
 };
 
+export const inclusionProofSchema = {
+	$id: '/storage/inclusionProof',
+	type: 'object',
+	required: ['siblingHashes', 'queries'],
+	properties: {
+		siblingHashes: {
+			type: 'array',
+			fieldNumber: 1,
+			items: {
+				dataType: 'bytes',
+			},
+		},
+		queries: {
+			type: 'array',
+			fieldNumber: 2,
+			items: {
+				type: 'object',
+				properties: {
+					key: {
+						dataType: 'bytes',
+						fieldNumber: 1,
+					},
+					value: {
+						dataType: 'bytes',
+						fieldNumber: 2,
+					},
+					bitmap: {
+						dataType: 'bytes',
+						fieldNumber: 3,
+					},
+				},
+			},
+		},
+	},
+};
+
 export const stateDiffSchema = {
 	$id: '/state/diff',
 	type: 'object',

elements/lisk-chain/src/types.ts

@@ -40,6 +40,11 @@ export interface UpdatedDiff {
 	readonly value: Buffer;
 }
 
+export interface InclusionProofConfig {
+	keysForInclusionProof: Buffer[];
+	keepInclusionProofsForHeights: number;
+}
+
 type Primitive = string | number | bigint | boolean | null | undefined;
 type Replaced<T, TReplace, TWith, TKeep = Primitive> = T extends TReplace | TKeep
 	? T extends TReplace

elements/lisk-chain/test/integration/data_access/blocks.spec.ts

@@ -63,6 +63,7 @@ describe('dataAccess.blocks', () => {
 			minBlockHeaderCache: 3,
 			maxBlockHeaderCache: 5,
 			keepEventsForHeights: -1,
+			keepInclusionProofsForHeights: -1,
 		});
 		// Prepare sample data
 		const block300 = await createValidDefaultBlock({

elements/lisk-chain/test/integration/data_access/transactions.spec.ts

@@ -34,6 +34,7 @@ describe('dataAccess.transactions', () => {
 			minBlockHeaderCache: 3,
 			maxBlockHeaderCache: 5,
 			keepEventsForHeights: -1,
+			keepInclusionProofsForHeights: -1,
 		});
 	});
 

elements/lisk-chain/test/unit/chain.spec.ts

@@ -38,6 +38,7 @@ describe('chain', () => {
 	const constants = {
 		maxTransactionsSize: 15 * 1024,
 		keepEventsForHeights: 300,
+		keepInclusionProofsForHeights: 300,
 	};
 	const emptyEncodedDiff = codec.encode(stateDiffSchema, {
 		created: [],

elements/lisk-chain/test/unit/data_access/data_access.spec.ts

@@ -13,7 +13,8 @@
  */
 import { Readable } from 'stream';
 import { when } from 'jest-when';
-import { NotFoundError, InMemoryDatabase } from '@liskhq/lisk-db';
+import { NotFoundError, InMemoryDatabase, Proof } from '@liskhq/lisk-db';
+import { codec } from '@liskhq/lisk-codec';
 import { utils } from '@liskhq/lisk-cryptography';
 import { DataAccess } from '../../../src/data_access';
 import { createFakeBlockHeader, createValidDefaultBlock } from '../../utils/block';
@@ -24,11 +25,13 @@ import {
 	DB_KEY_BLOCKS_ID,
 	DB_KEY_TRANSACTIONS_ID,
 	DB_KEY_BLOCK_EVENTS,
+	DB_KEY_INCLUSION_PROOFS,
 } from '../../../src/db_keys';
 import { Block } from '../../../src/block';
 import { Event } from '../../../src/event';
 import { BlockAssets, BlockHeader } from '../../../src';
 import { encodeByteArray } from '../../../src/data_access/storage';
+import { inclusionProofSchema } from '../../../src/schema';
 
 jest.mock('@liskhq/lisk-db');
 
@@ -45,6 +48,7 @@ describe('data_access', () => {
 			minBlockHeaderCache: 3,
 			maxBlockHeaderCache: 5,
 			keepEventsForHeights: 1,
+			keepInclusionProofsForHeights: 1,
 		});
 		block = await createValidDefaultBlock({ header: { height: 1 } });
 	});
@@ -400,6 +404,57 @@ describe('data_access', () => {
 		});
 	});
 
+	describe('#getInclusionProofs', () => {
+		it('should get empty array if the inclusionProofs does not exist', async () => {
+			db.get.mockRejectedValue(new NotFoundError());
+
+			const resp = await dataAccess.getInclusionProofs(30);
+			expect(db.get).toHaveBeenCalledWith(concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(30)));
+			expect(resp).toEqual({
+				siblingHashes: [],
+				queries: [],
+			});
+		});
+
+		it('should get the inclusion proofs related to heights', async () => {
+			const original = {
+				siblingHashes: [Buffer.alloc(3)],
+				queries: [
+					{
+						key: Buffer.alloc(2),
+						value: Buffer.alloc(2),
+						bitmap: Buffer.alloc(1),
+					},
+				],
+			};
+			db.get.mockResolvedValue(codec.encode(inclusionProofSchema, original) as never);
+
+			const resp = await dataAccess.getInclusionProofs(30);
+			expect(db.get).toHaveBeenCalledWith(concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(30)));
+			expect(resp).toEqual(original);
+		});
+	});
+
+	describe('#setInclusionProofs', () => {
+		it('should set inclusionProofs for a given height', async () => {
+			const proofs: Proof = {
+				siblingHashes: [Buffer.alloc(3)],
+				queries: [
+					{
+						key: Buffer.alloc(2),
+						value: Buffer.alloc(2),
+						bitmap: Buffer.alloc(1),
+					},
+				],
+			};
+			await dataAccess.setInclusionProofs(proofs, 30);
+			expect(db.set).toHaveBeenCalledWith(
+				concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(30)),
+				codec.encode(inclusionProofSchema, proofs),
+			);
+		});
+	});
+
 	describe('#isBlockPersisted', () => {
 		it('should call check if the id exists in the database', async () => {
 			// Act

examples/interop/pos-mainchain-fast/config/custom_config_node_one.json

@@ -1,7 +1,6 @@
 {
 	"system": {
 		"dataPath": "~/.lisk/pos-mainchain-node-one",
-		"keepEventsForHeights": 300,
 		"logLevel": "info"
 	},
 	"rpc": {