♻️ Handle ccm processing error #9100

- Create Panic function to exit application on unexpected error while CCU execution
- Add events to verifyPartnerChainOutboxRoot on smt and partnerOutbox failure #9111
- Rename executeCommon->beforeCrossChainMessagesExecution
- Add verifyRoutingRules similar to LIP0053
- Move verifyCertificateSignature and verifyPartnerChainOutboxRoot outside beforeCrossChainMessagesExecution and call before it #9112
- Move constant EVENT_TOPIC_CCM_EXECUTION to interoperability module

framework/src/modules/interoperability/base_cross_chain_update_command.ts

@@ -19,7 +19,7 @@ import { CommandExecuteContext, CommandVerifyContext } from '../../state_machine
 import { BaseInteroperabilityCommand } from './base_interoperability_command';
 import { BaseInteroperabilityInternalMethod } from './base_interoperability_internal_methods';
 import { BaseInteroperabilityMethod } from './base_interoperability_method';
-import { CCMStatusCode, EMPTY_BYTES, EmptyCCM } from './constants';
+import { CCMStatusCode, EMPTY_BYTES, EVENT_TOPIC_CCM_EXECUTION, EmptyCCM } from './constants';
 import { CCMProcessedCode, CcmProcessedEvent, CCMProcessedResult } from './events/ccm_processed';
 import { CcmSendSuccessEvent } from './events/ccm_send_success';
 import { ccmSchema, crossChainUpdateTransactionParams } from './schemas';
@@ -34,6 +34,7 @@ import { ChainAccountStore, ChainStatus } from './stores/chain_account';
 import {
 	emptyActiveValidatorsUpdate,
 	getEncodedCCMAndID,
+	getIDFromCCMBytes,
 	getMainchainID,
 	isInboxUpdateEmpty,
 	validateFormat,
@@ -122,68 +123,50 @@ export abstract class BaseCrossChainUpdateCommand<
 		}
 	}
 
-	protected async executeCommon(
+	protected async beforeCrossChainMessagesExecution(
 		context: CommandExecuteContext<CrossChainUpdateTransactionParams>,
 		isMainchain: boolean,
 	): Promise<[CCMsg[], boolean]> {
-		const { params, transaction } = context;
+		const { params } = context;
 		const { inboxUpdate } = params;
 
-		// Verify certificate signature. We do it here because if it fails, the transaction fails rather than being invalid.
-		await this.internalMethod.verifyCertificateSignature(context, params);
-
-		if (!isInboxUpdateEmpty(inboxUpdate)) {
-			// This check is expensive. Therefore, it is done in the execute step instead of the verify
-			// step. Otherwise, a malicious relayer could spam the transaction pool with computationally
-			// costly CCU verifications without paying fees.
-			try {
-				await this.internalMethod.verifyPartnerChainOutboxRoot(context, params);
-			} catch (error) {
-				return [[], false];
-			}
-
-			// Initialize the relayer account for the message fee token.
-			// This is necessary to ensure that the relayer can receive the CCM fees
-			// If the account already exists, nothing is done.
-			const messageFeeTokenID = await this._interopsMethod.getMessageFeeTokenID(
-				context,
-				params.sendingChainID,
-			);
-			await this._tokenMethod.initializeUserAccount(
-				context,
-				transaction.senderAddress,
-				messageFeeTokenID,
-			);
-		}
-
 		const ccms: CCMsg[] = [];
 		let ccm: CCMsg;
 
 		// Process cross-chain messages in inbox update.
 		// First process basic checks for all CCMs.
 		for (const ccmBytes of inboxUpdate.crossChainMessages) {
+			const ccmID = getIDFromCCMBytes(ccmBytes);
+			const ccmContext = {
+				...context,
+				eventQueue: context.eventQueue.getChildQueue(
+					Buffer.concat([EVENT_TOPIC_CCM_EXECUTION, ccmID]),
+				),
+			};
 			try {
 				// Verify general format. Past this point, we can access ccm root properties.
 				ccm = codec.decode<CCMsg>(ccmSchema, ccmBytes);
 			} catch (error) {
-				await this.internalMethod.terminateChainInternal(context, params.sendingChainID);
-				this.events.get(CcmProcessedEvent).log(context, params.sendingChainID, context.chainID, {
-					ccm: EmptyCCM,
-					result: CCMProcessedResult.DISCARDED,
-					code: CCMProcessedCode.INVALID_CCM_DECODING_EXCEPTION,
-				});
+				await this.internalMethod.terminateChainInternal(ccmContext, params.sendingChainID);
+				this.events
+					.get(CcmProcessedEvent)
+					.log(ccmContext, params.sendingChainID, ccmContext.chainID, {
+						ccm: EmptyCCM,
+						result: CCMProcessedResult.DISCARDED,
+						code: CCMProcessedCode.INVALID_CCM_DECODING_EXCEPTION,
+					});
 				// In this case, we do not even update the chain account with the new certificate.
 				return [[], false];
 			}
 
 			try {
 				validateFormat(ccm);
 			} catch (error) {
-				await this.internalMethod.terminateChainInternal(context, params.sendingChainID);
+				await this.internalMethod.terminateChainInternal(ccmContext, params.sendingChainID);
 				ccm = { ...ccm, params: EMPTY_BYTES };
 				this.events
 					.get(CcmProcessedEvent)
-					.log(context, params.sendingChainID, ccm.receivingChainID, {
+					.log(ccmContext, params.sendingChainID, ccm.receivingChainID, {
 						ccm,
 						result: CCMProcessedResult.DISCARDED,
 						code: CCMProcessedCode.INVALID_CCM_VALIDATION_EXCEPTION,
@@ -193,27 +176,13 @@ export abstract class BaseCrossChainUpdateCommand<
 			}
 
 			try {
-				// The CCM must come from the sending chain.
-				if (isMainchain && !ccm.sendingChainID.equals(params.sendingChainID)) {
-					throw new Error('CCM is not from the sending chain.');
-				}
-				// Sending and receiving chains must differ.
-				if (ccm.receivingChainID.equals(ccm.sendingChainID)) {
-					throw new Error('Sending and receiving chains must differ.');
-				}
-				// The CCM must come be directed to the sidechain, unless it was bounced on the mainchain.
-				if (!isMainchain && !context.chainID.equals(ccm.receivingChainID)) {
-					throw new Error('CCM is not directed to the sidechain.');
-				}
-				if (isMainchain && ccm.status === CCMStatusCode.CHANNEL_UNAVAILABLE) {
-					throw new Error('CCM status channel unavailable can only be set on the mainchain.');
-				}
+				this.verifyRoutingRules(ccm, params, ccmContext.chainID, isMainchain);
 				ccms.push(ccm);
 			} catch (error) {
-				await this.internalMethod.terminateChainInternal(context, params.sendingChainID);
+				await this.internalMethod.terminateChainInternal(ccmContext, params.sendingChainID);
 				this.events
 					.get(CcmProcessedEvent)
-					.log(context, params.sendingChainID, ccm.receivingChainID, {
+					.log(ccmContext, params.sendingChainID, ccm.receivingChainID, {
 						ccm,
 						result: CCMProcessedResult.DISCARDED,
 						code: CCMProcessedCode.INVALID_CCM_ROUTING_EXCEPTION,
@@ -226,6 +195,29 @@ export abstract class BaseCrossChainUpdateCommand<
 		return [ccms, true];
 	}
 
+	protected verifyRoutingRules(
+		ccm: CCMsg,
+		ccuParams: CrossChainUpdateTransactionParams,
+		ownChainID: Buffer,
+		isMainchain: boolean,
+	) {
+		// The CCM must come from the sending chain.
+		if (isMainchain && !ccm.sendingChainID.equals(ccuParams.sendingChainID)) {
+			throw new Error('CCM is not from the sending chain.');
+		}
+		// Sending and receiving chains must differ.
+		if (ccm.receivingChainID.equals(ccm.sendingChainID)) {
+			throw new Error('Sending and receiving chains must differ.');
+		}
+		// The CCM must come be directed to the sidechain, unless it was bounced on the mainchain.
+		if (!isMainchain && !ownChainID.equals(ccm.receivingChainID)) {
+			throw new Error('CCM is not directed to the sidechain.');
+		}
+		if (isMainchain && ccm.status === CCMStatusCode.CHANNEL_UNAVAILABLE) {
+			throw new Error('CCM status channel unavailable can only be set on the mainchain.');
+		}
+	}
+
 	protected async afterExecuteCommon(
 		context: CommandExecuteContext<CrossChainUpdateTransactionParams>,
 	) {

framework/src/modules/interoperability/base_interoperability_internal_methods.ts

@@ -48,7 +48,12 @@ import { TerminatedOutboxAccount, TerminatedOutboxStore } from './stores/termina
 import { ChainAccountUpdatedEvent } from './events/chain_account_updated';
 import { TerminatedStateCreatedEvent } from './events/terminated_state_created';
 import { BaseInternalMethod } from '../BaseInternalMethod';
-import { MethodContext, ImmutableMethodContext, NotFoundError } from '../../state_machine';
+import {
+	MethodContext,
+	ImmutableMethodContext,
+	NotFoundError,
+	CommandExecuteContext,
+} from '../../state_machine';
 import { ChainValidatorsStore } from './stores/chain_validators';
 import { certificateSchema } from '../../engine/consensus/certificate_generation/schema';
 import { Certificate } from '../../engine/consensus/certificate_generation/types';
@@ -60,6 +65,8 @@ import { TerminatedOutboxCreatedEvent } from './events/terminated_outbox_created
 import { BaseCCMethod } from './base_cc_method';
 import { verifyAggregateCertificateSignature } from '../../engine/consensus/certificate_generation/utils';
 import { InvalidCertificateSignatureEvent } from './events/invalid_certificate_signature';
+import { InvalidSMTVerification } from './events/invalid_smt_verification';
+import { InvalidOutboxRootverification } from './events/invalid_outbox_root_verification';
 
 export abstract class BaseInteroperabilityInternalMethod extends BaseInternalMethod {
 	protected readonly interoperableModuleMethods = new Map<string, BaseCCMethod>();
@@ -665,7 +672,7 @@ export abstract class BaseInteroperabilityInternalMethod extends BaseInternalMet
 	 * @see https://github.com/LiskHQ/lips/blob/main/proposals/lip-0053.md#verifypartnerchainoutboxroot
 	 */
 	public async verifyPartnerChainOutboxRoot(
-		context: ImmutableMethodContext,
+		context: CommandExecuteContext<CrossChainUpdateTransactionParams>,
 		params: CrossChainUpdateTransactionParams,
 	): Promise<void> {
 		const channel = await this.stores.get(ChannelDataStore).get(context, params.sendingChainID);
@@ -689,6 +696,15 @@ export abstract class BaseInteroperabilityInternalMethod extends BaseInternalMet
 
 		if (params.certificate.length === 0) {
 			if (!newInboxRoot.equals(channel.partnerChainOutboxRoot)) {
+				this.events.get(InvalidOutboxRootverification).error(
+					context,
+					params.sendingChainID,
+					{
+						inboxRoot: newInboxRoot,
+						partnerChainOutboxRoot: channel.partnerChainOutboxRoot,
+					},
+					true,
+				);
 				throw new Error('Inbox root does not match partner chain outbox root.');
 			}
 			return;
@@ -713,6 +729,7 @@ export abstract class BaseInteroperabilityInternalMethod extends BaseInternalMet
 		const smt = new SparseMerkleTree();
 		const valid = await smt.verifyInclusionProof(certificate.stateRoot, [outboxKey], proof);
 		if (!valid) {
+			this.events.get(InvalidSMTVerification).error(context);
 			throw new Error('Invalid inclusion proof for inbox update.');
 		}
 	}

framework/src/modules/interoperability/base_interoperability_module.ts

@@ -53,6 +53,7 @@ import { TerminatedOutboxCreatedEvent } from './events/terminated_outbox_created
 import { TerminatedStateCreatedEvent } from './events/terminated_state_created';
 import { InvalidSMTVerification } from './events/invalid_smt_verification';
 import { InvalidRMTVerification } from './events/invalid_rmt_verification';
+import { InvalidOutboxRootverification } from './events/invalid_outbox_root_verification';
 
 export abstract class BaseInteroperabilityModule extends BaseInteroperableModule {
 	protected interoperableCCCommands = new Map<string, BaseCCCommand[]>();
@@ -88,6 +89,10 @@ export abstract class BaseInteroperabilityModule extends BaseInteroperableModule
 			InvalidCertificateSignatureEvent,
 			new InvalidCertificateSignatureEvent(this.name),
 		);
+		this.events.register(
+			InvalidOutboxRootverification,
+			new InvalidOutboxRootverification(this.name),
+		);
 	}
 
 	// Common name for mainchain/sidechain interoperability module

framework/src/modules/interoperability/constants.ts

@@ -90,8 +90,10 @@ export const EVENT_NAME_CHAIN_ACCOUNT_UPDATED = 'chainAccountUpdated';
 export const EVENT_NAME_CCM_PROCESSED = 'ccmProcessed';
 export const EVENT_NAME_CCM_SEND_SUCCESS = 'ccmSendSucess';
 export const EVENT_NAME_INVALID_CERTIFICATE_SIGNATURE = 'invalidCertificateSignature';
+export const EVENT_NAME_INVALID_OUTBOX_ROOT_VERIFICATION = 'invalidOutboxRootVerification';
 
 export const CONTEXT_STORE_KEY_CCM_PROCESSING = 'CONTEXT_STORE_KEY_CCM_PROCESSING';
+export const EVENT_TOPIC_CCM_EXECUTION = Buffer.from([5]);
 
 // https://github.com/LiskHQ/lips/blob/main/proposals/lip-0045.md#empty-cross-chain-message
 export const EmptyCCM = {

framework/src/modules/interoperability/events/invalid_outbox_root_verification.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright © 2023 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+import { BaseEvent, EventQueuer } from '../../base_event';
+import { HASH_LENGTH } from '../constants';
+
+export interface InvalidOutboxRootVerificationData {
+	inboxRoot: Buffer;
+	partnerChainOutboxRoot: Buffer;
+}
+
+export const invalidOutboxRootVerificationSchema = {
+	$id: '/interoperability/events/invalidOutboxRootVerification',
+	type: 'object',
+	required: ['inboxRoot', 'partnerChainOutboxRoot'],
+	properties: {
+		inboxRoot: {
+			dataType: 'bytes',
+			fieldNumber: 1,
+			minLength: HASH_LENGTH,
+			maxLength: HASH_LENGTH,
+		},
+		partnerChainOutboxRoot: {
+			dataType: 'bytes',
+			fieldNumber: 2,
+			minLength: HASH_LENGTH,
+			maxLength: HASH_LENGTH,
+		},
+	},
+};
+
+export class InvalidOutboxRootverification extends BaseEvent<InvalidOutboxRootVerificationData> {
+	public schema = invalidOutboxRootVerificationSchema;
+
+	public error(
+		ctx: EventQueuer,
+		chainID: Buffer,
+		data: InvalidOutboxRootVerificationData,
+		noRevert: boolean,
+	): void {
+		this.add(ctx, data, [chainID], noRevert);
+	}
+}

framework/src/modules/interoperability/mainchain/commands/recover_message.ts

@@ -31,7 +31,11 @@ import {
 	getEncodedCCMAndID,
 	getDecodedCCMAndID,
 } from '../../utils';
-import { CCMStatusCode, CONTEXT_STORE_KEY_CCM_PROCESSING } from '../../constants';
+import {
+	CCMStatusCode,
+	CONTEXT_STORE_KEY_CCM_PROCESSING,
+	EVENT_TOPIC_CCM_EXECUTION,
+} from '../../constants';
 import { ccmSchema, messageRecoveryParamsSchema } from '../../schemas';
 import { TerminatedOutboxAccount, TerminatedOutboxStore } from '../../stores/terminated_outbox';
 import {
@@ -40,7 +44,6 @@ import {
 	CCMProcessedResult,
 } from '../../events/ccm_processed';
 import { InvalidRMTVerification } from '../../events/invalid_rmt_verification';
-import { EVENT_TOPIC_CCM_EXECUTION } from '../../../../state_machine/constants';
 
 // https://github.com/LiskHQ/lips/blob/main/proposals/lip-0054.md#message-recovery-command
 export class RecoverMessageCommand extends BaseInteroperabilityCommand<MainchainInteroperabilityInternalMethod> {

framework/src/modules/interoperability/mainchain/commands/submit_mainchain_cross_chain_update.ts

@@ -25,6 +25,7 @@ import {
 	CONTEXT_STORE_KEY_CCM_PROCESSING,
 	CROSS_CHAIN_COMMAND_SIDECHAIN_TERMINATED,
 	EMPTY_FEE_ADDRESS,
+	EVENT_TOPIC_CCM_EXECUTION,
 	MODULE_NAME_INTEROPERABILITY,
 } from '../../constants';
 import {
@@ -47,7 +48,7 @@ import {
 	getIDFromCCMBytes,
 } from '../../utils';
 import { MainchainInteroperabilityInternalMethod } from '../internal_method';
-import { EVENT_TOPIC_CCM_EXECUTION } from '../../../../state_machine/constants';
+import { panic } from '../../../../utils/panic';
 
 // https://github.com/LiskHQ/lips/blob/main/proposals/lip-0053.md#commands
 export class SubmitMainchainCrossChainUpdateCommand extends BaseCrossChainUpdateCommand<MainchainInteroperabilityInternalMethod> {
@@ -82,11 +83,35 @@ export class SubmitMainchainCrossChainUpdateCommand extends BaseCrossChainUpdate
 	public async execute(
 		context: CommandExecuteContext<CrossChainUpdateTransactionParams>,
 	): Promise<void> {
-		const [decodedCCMs, ok] = await this.executeCommon(context, true);
+		const { params, transaction } = context;
+		const { inboxUpdate } = params;
+		// Verify certificate signature. We do it here because if it fails, the transaction fails rather than being invalid.
+		await this.internalMethod.verifyCertificateSignature(context, params);
+
+		if (!isInboxUpdateEmpty(inboxUpdate)) {
+			// This check is expensive. Therefore, it is done in the execute step instead of the verify
+			// step. Otherwise, a malicious relayer could spam the transaction pool with computationally
+			// costly CCU verifications without paying fees.
+			await this.internalMethod.verifyPartnerChainOutboxRoot(context, params);
+
+			// Initialize the relayer account for the message fee token.
+			// This is necessary to ensure that the relayer can receive the CCM fees
+			// If the account already exists, nothing is done.
+			const messageFeeTokenID = await this._interopsMethod.getMessageFeeTokenID(
+				context,
+				params.sendingChainID,
+			);
+			await this._tokenMethod.initializeUserAccount(
+				context,
+				transaction.senderAddress,
+				messageFeeTokenID,
+			);
+		}
+
+		const [decodedCCMs, ok] = await this.beforeCrossChainMessagesExecution(context, true);
 		if (!ok) {
 			return;
 		}
-		const { params } = context;
 
 		try {
 			// Update the context to indicate that now we start the CCM processing.
@@ -117,6 +142,8 @@ export class SubmitMainchainCrossChainUpdateCommand extends BaseCrossChainUpdate
 				// would refer to an inbox where the message has not been appended yet).
 				await this.internalMethod.appendToInboxTree(context, params.sendingChainID, ccmBytes);
 			}
+		} catch (error) {
+			panic(context.logger, error as Error);
 		} finally {
 			// Update the context to indicate that now we stop the CCM processing.
 			context.contextStore.delete(CONTEXT_STORE_KEY_CCM_PROCESSING);