♻️ Add ability to save inclusion proofs at every new block

- Add config to take keepInclusionProofsForHeights and inclusionProofKeys
- Add DB update to get/set/delete inclusionProofs
- Add handler in new block event to save inclusion proofs
- Add endpoint for chain to get inclusion proofs by height

elements/lisk-chain/src/chain.ts

@@ -18,6 +18,7 @@ import * as createDebug from 'debug';
 import { regularMerkleTree } from '@liskhq/lisk-tree';
 import {
 	DEFAULT_KEEP_EVENTS_FOR_HEIGHTS,
+	DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS,
 	DEFAULT_MAX_BLOCK_HEADER_CACHE,
 	DEFAULT_MIN_BLOCK_HEADER_CACHE,
 	GENESIS_BLOCK_VERSION,
@@ -40,6 +41,7 @@ interface ChainConstructor {
 	readonly maxTransactionsSize: number;
 	readonly minBlockHeaderCache?: number;
 	readonly maxBlockHeaderCache?: number;
+	readonly keepInclusionProofsForHeights: number;
 }
 
 interface ChainInitArgs {
@@ -61,6 +63,7 @@ export class Chain {
 		readonly minBlockHeaderCache: number;
 		readonly maxBlockHeaderCache: number;
 		readonly keepEventsForHeights: number;
+		readonly keepInclusionProofsForHeights: number;
 	};
 
 	private _lastBlock?: Block;
@@ -74,6 +77,7 @@ export class Chain {
 		keepEventsForHeights = DEFAULT_KEEP_EVENTS_FOR_HEIGHTS,
 		minBlockHeaderCache = DEFAULT_MIN_BLOCK_HEADER_CACHE,
 		maxBlockHeaderCache = DEFAULT_MAX_BLOCK_HEADER_CACHE,
+		keepInclusionProofsForHeights = DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS,
 	}: ChainConstructor) {
 		// Register codec schema
 		codec.addSchema(blockSchema);
@@ -86,6 +90,7 @@ export class Chain {
 			maxBlockHeaderCache,
 			minBlockHeaderCache,
 			keepEventsForHeights,
+			keepInclusionProofsForHeights,
 		};
 	}
 
@@ -118,6 +123,7 @@ export class Chain {
 			minBlockHeaderCache: this.constants.minBlockHeaderCache,
 			maxBlockHeaderCache: this.constants.maxBlockHeaderCache,
 			keepEventsForHeights: this.constants.keepEventsForHeights,
+			keepInclusionProofsForHeights: this.constants.keepInclusionProofsForHeights,
 		});
 	}
 

elements/lisk-chain/src/constants.ts

@@ -15,6 +15,7 @@
 import { utils } from '@liskhq/lisk-cryptography';
 
 export const DEFAULT_KEEP_EVENTS_FOR_HEIGHTS = 300;
+export const DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS = 300;
 export const DEFAULT_MIN_BLOCK_HEADER_CACHE = 309;
 export const DEFAULT_MAX_BLOCK_HEADER_CACHE = 515;
 

elements/lisk-chain/src/data_access/data_access.ts

@@ -12,7 +12,7 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 
-import { Database, NotFoundError } from '@liskhq/lisk-db';
+import { Database, NotFoundError, Proof } from '@liskhq/lisk-db';
 import { Transaction } from '../transaction';
 import { RawBlock } from '../types';
 import { BlockHeader } from '../block_header';
@@ -29,6 +29,7 @@ interface DAConstructor {
 	readonly minBlockHeaderCache: number;
 	readonly maxBlockHeaderCache: number;
 	readonly keepEventsForHeights: number;
+	readonly keepInclusionProofsForHeights: number;
 }
 
 export class DataAccess {
@@ -40,8 +41,9 @@ export class DataAccess {
 		minBlockHeaderCache,
 		maxBlockHeaderCache,
 		keepEventsForHeights,
+		keepInclusionProofsForHeights,
 	}: DAConstructor) {
-		this._storage = new StorageAccess(db, { keepEventsForHeights });
+		this._storage = new StorageAccess(db, { keepEventsForHeights, keepInclusionProofsForHeights });
 		this._blocksCache = new BlockCache(minBlockHeaderCache, maxBlockHeaderCache);
 	}
 
@@ -243,6 +245,16 @@ export class DataAccess {
 		return events;
 	}
 
+	public async getInclusionProofs(height: number): Promise<Proof> {
+		const proofs = await this._storage.getInclusionProofs(height);
+
+		return proofs;
+	}
+
+	public async setInclusionProofs(proof: Proof, height: number): Promise<void> {
+		await this._storage.setInclusionProofs(proof, height);
+	}
+
 	public async isBlockPersisted(blockId: Buffer): Promise<boolean> {
 		const isPersisted = await this._storage.isBlockPersisted(blockId);
 

elements/lisk-chain/src/data_access/storage.ts

@@ -11,7 +11,7 @@
  *
  * Removal or modification of this copyright notice is prohibited.
  */
-import { Batch, Database, NotFoundError } from '@liskhq/lisk-db';
+import { Batch, Database, NotFoundError, Proof } from '@liskhq/lisk-db';
 import { codec } from '@liskhq/lisk-codec';
 import { utils } from '@liskhq/lisk-cryptography';
 import { RawBlock, StateDiff } from '../types';
@@ -26,11 +26,16 @@ import {
 	DB_KEY_FINALIZED_HEIGHT,
 	DB_KEY_BLOCK_ASSETS_BLOCK_ID,
 	DB_KEY_BLOCK_EVENTS,
+	DB_KEY_INCLUSION_PROOFS,
 } from '../db_keys';
 import { concatDBKeys, uint32BE } from '../utils';
-import { stateDiffSchema } from '../schema';
+import { inclusionProofSchema, stateDiffSchema } from '../schema';
 import { CurrentState } from '../state_store';
-import { DEFAULT_KEEP_EVENTS_FOR_HEIGHTS, MAX_UINT32 } from '../constants';
+import {
+	DEFAULT_KEEP_EVENTS_FOR_HEIGHTS,
+	DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS,
+	MAX_UINT32,
+} from '../constants';
 
 const bytesArraySchema = {
 	$id: '/liskChain/bytesarray',
@@ -61,15 +66,19 @@ export const encodeByteArray = (val: Buffer[]): Buffer =>
 
 interface StorageOption {
 	keepEventsForHeights?: number;
+	keepInclusionProofsForHeights?: number;
 }
 
 export class Storage {
 	private readonly _db: Database;
 	private readonly _keepEventsForHeights: number;
+	private readonly _keepInclusionProofsForHeights: number;
 
 	public constructor(db: Database, options?: StorageOption) {
 		this._db = db;
 		this._keepEventsForHeights = options?.keepEventsForHeights ?? DEFAULT_KEEP_EVENTS_FOR_HEIGHTS;
+		this._keepInclusionProofsForHeights =
+			options?.keepInclusionProofsForHeights ?? DEFAULT_KEEP_INCLUSION_PROOFS_FOR_HEIGHTS;
 	}
 
 	/*
@@ -257,6 +266,29 @@ export class Storage {
 		}
 	}
 
+	public async setInclusionProofs(proof: Proof, height: number): Promise<void> {
+		const proofBytes = codec.encode(inclusionProofSchema, proof);
+		await this._db.set(concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(height)), proofBytes);
+	}
+
+	public async getInclusionProofs(height: number): Promise<Proof> {
+		try {
+			const proofBytes = await this._db.get(
+				concatDBKeys(DB_KEY_INCLUSION_PROOFS, uint32BE(height)),
+			);
+
+			return codec.decode(inclusionProofSchema, proofBytes);
+		} catch (error) {
+			if (!(error instanceof NotFoundError)) {
+				throw error;
+			}
+			return {
+				queries: [],
+				siblingHashes: [],
+			};
+		}
+	}
+
 	public async getTempBlocks(): Promise<Buffer[]> {
 		const stream = this._db.createReadStream({
 			gte: concatDBKeys(DB_KEY_TEMPBLOCKS_HEIGHT, uint32BE(0)),
@@ -419,6 +451,7 @@ export class Storage {
 			batch.del(concatDBKeys(DB_KEY_TRANSACTIONS_BLOCK_ID, id));
 		}
 		batch.del(concatDBKeys(DB_KEY_BLOCK_EVENTS, heightBuf));
+		batch.del(concatDBKeys(DB_KEY_INCLUSION_PROOFS, heightBuf));
 		if (assets.length > 0) {
 			batch.del(concatDBKeys(DB_KEY_BLOCK_ASSETS_BLOCK_ID, id));
 		}
@@ -486,6 +519,22 @@ export class Storage {
 				);
 			}
 		}
+
+		if (this._keepEventsForHeights > -1) {
+			// events are removed only if finalized and below height - keepEventsForHeights
+			const minInclusionProofDeleteHeight = Math.min(
+				finalizedHeight,
+				Math.max(0, currentHeight - this._keepInclusionProofsForHeights),
+			);
+			if (minInclusionProofDeleteHeight > 0) {
+				const endHeight = Buffer.alloc(4);
+				endHeight.writeUInt32BE(minInclusionProofDeleteHeight - 1, 0);
+				await this._clear(
+					Buffer.concat([DB_KEY_INCLUSION_PROOFS, Buffer.alloc(4, 0)]),
+					Buffer.concat([DB_KEY_INCLUSION_PROOFS, endHeight]),
+				);
+			}
+		}
 	}
 
 	private async _getBlockAssets(blockID: Buffer): Promise<Buffer[]> {

elements/lisk-chain/src/db_keys.ts

@@ -20,6 +20,7 @@ export const DB_KEY_TRANSACTIONS_ID = Buffer.from([6]);
 export const DB_KEY_TEMPBLOCKS_HEIGHT = Buffer.from([7]);
 export const DB_KEY_BLOCK_ASSETS_BLOCK_ID = Buffer.from([8]);
 export const DB_KEY_BLOCK_EVENTS = Buffer.from([9]);
+export const DB_KEY_INCLUSION_PROOFS = Buffer.from([11]);
 
 export const DB_KEY_STATE_STORE = Buffer.from([10]);
 

elements/lisk-chain/src/schema.ts

@@ -137,6 +137,42 @@ export const blockAssetSchema = {
 	},
 };
 
+export const inclusionProofSchema = {
+	$id: '/storage/inclusionProof',
+	type: 'object',
+	required: ['siblingHashes', 'queries'],
+	properties: {
+		siblingHashes: {
+			type: 'array',
+			fieldNumber: 1,
+			items: {
+				dataType: 'bytes',
+			},
+		},
+		queries: {
+			type: 'array',
+			fieldNumber: 2,
+			items: {
+				type: 'object',
+				properties: {
+					key: {
+						dataType: 'bytes',
+						fieldNumber: 1,
+					},
+					value: {
+						dataType: 'bytes',
+						fieldNumber: 2,
+					},
+					bitmap: {
+						dataType: 'bytes',
+						fieldNumber: 3,
+					},
+				},
+			},
+		},
+	},
+};
+
 export const stateDiffSchema = {
 	$id: '/state/diff',
 	type: 'object',

elements/lisk-chain/src/types.ts

@@ -40,6 +40,11 @@ export interface UpdatedDiff {
 	readonly value: Buffer;
 }
 
+export interface InclusionProofConfig {
+	keysForInclusionProof: Buffer[];
+	keepInclusionProofsForHeights: number;
+}
+
 type Primitive = string | number | bigint | boolean | null | undefined;
 type Replaced<T, TReplace, TWith, TKeep = Primitive> = T extends TReplace | TKeep
 	? T extends TReplace

examples/interop/pos-mainchain-fast/config/custom_config_node_one.json

@@ -2,7 +2,10 @@
 	"system": {
 		"dataPath": "~/.lisk/pos-mainchain-node-one",
 		"keepEventsForHeights": 300,
-		"logLevel": "info"
+		"logLevel": "info",
+		"inclusionProofKeys": [
+			"83ed0d250000160811fdaf692ba77eabfbfc3a6bb3c4cf6a87beafd28cfe90b5dc64cb20ab46"
+		]
 	},
 	"rpc": {
 		"modes": ["ipc"],

examples/interop/pos-mainchain-fast/config/custom_config_node_two.json

@@ -2,7 +2,10 @@
 	"system": {
 		"dataPath": "~/.lisk/pos-mainchain-node-two",
 		"keepEventsForHeights": 300,
-		"logLevel": "info"
+		"logLevel": "info",
+		"inclusionProofKeys": [
+			"83ed0d250000160811fdaf692ba77eabfbfc3a6bb3c4cf6a87beafd28cfe90b5dc64cb20ab46"
+		]
 	},
 	"rpc": {
 		"modes": ["ipc"],

examples/interop/pos-mainchain-fast/config/default/config.json

@@ -2,6 +2,9 @@
 	"system": {
 		"dataPath": "~/.lisk/pos-mainchain-fast",
 		"keepEventsForHeights": 300,
+		"inclusionProofKeys": [
+			"83ed0d250000160811fdaf692ba77eabfbfc3a6bb3c4cf6a87beafd28cfe90b5dc64cb20ab46"
+		],
 		"logLevel": "info"
 	},
 	"rpc": {

framework/src/engine/endpoint/chain.ts

@@ -198,6 +198,23 @@ export class ChainEndpoint {
 		return events.map(e => e.toJSON());
 	}
 
+	public async getInclusionProofsAtHeight(context: RequestContext): Promise<JSONObject<Proof>> {
+		const { height } = context.params;
+		if (typeof height !== 'number' || height < 0) {
+			throw new Error('Invalid parameters. height must be zero or a positive number.');
+		}
+		const inclusionProof = await this._chain.dataAccess.getInclusionProofs(height);
+
+		return {
+			queries: inclusionProof.queries.map(q => ({
+				bitmap: q.bitmap.toString('hex'),
+				key: q.key.toString('hex'),
+				value: q.value.toString('hex'),
+			})),
+			siblingHashes: inclusionProof.siblingHashes.map(h => h.toString('hex')),
+		};
+	}
+
 	public async proveEvents(context: RequestContext): Promise<JSONObject<Proof>> {
 		validator.validate(proveEventsRequestSchema, context.params);
 

framework/src/engine/engine.ts

@@ -100,10 +100,14 @@ export class Engine {
 	private _blockchainDB!: Database;
 	private _legacyDB!: Database;
 	private _chainID!: Buffer;
+	private readonly _inclusionProofKeys: Buffer[];
 
 	public constructor(abi: ABI, config: EngineConfig) {
 		this._abi = abi;
 		this._config = config;
+		this._inclusionProofKeys = this._config.system.inclusionProofKeys.map(k =>
+			Buffer.from(k, 'hex'),
+		);
 	}
 
 	public async generateBlock(input: BlockGenerateInput): Promise<Block> {
@@ -156,6 +160,7 @@ export class Engine {
 		this._chain = new Chain({
 			maxTransactionsSize: this._config.genesis.maxTransactionsSize,
 			keepEventsForHeights: this._config.system.keepEventsForHeights,
+			keepInclusionProofsForHeights: this._config.system.keepInclusionProofsForHeights,
 		});
 
 		this._bftModule = new BFTModule();
@@ -299,6 +304,19 @@ export class Engine {
 	private _registerEventListeners() {
 		this._consensus.events.on(CONSENSUS_EVENT_BLOCK_NEW, ({ block }: { block: Block }) => {
 			this._generator.onNewBlock(block);
+
+			if (this._config.system.inclusionProofKeys.length > 0) {
+				this._abi
+					.prove({ keys: this._inclusionProofKeys, stateRoot: block.header.stateRoot as Buffer })
+					.then(result => {
+						// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+						this._chain.dataAccess
+							.setInclusionProofs(result.proof, block.header.height)
+							.catch((err: Error) => this._logger.error({ err }, 'Failed to set inclusion proof'));
+					})
+					.catch(err => this._logger.error({ err: err as Error }, 'Failed to query prove'));
+			}
+
 			Promise.all([
 				this._rpcServer.publish(EVENT_CHAIN_BLOCK_NEW, { blockHeader: block.header.toJSON() }),
 				this._rpcServer.publish(EVENT_NETWORK_BLOCK_NEW, { blockHeader: block.header.toJSON() }),