From 24b8eaf9c95d102ca53c203244fe1ff5fd1298e7 Mon Sep 17 00:00:00 2001 From: Yogesh01000100 Date: Thu, 29 Aug 2024 21:29:02 +0000 Subject: [PATCH] feat(recovery): add rollback implementations Signed-off-by: Yogesh01000100 --- .../typescript/core/recovery/crash-manager.ts | 70 +++++++++++-------- .../rollback/rollback-strategy-factory.ts | 9 +-- .../rollback/stage0-rollback-strategy.ts | 45 ++++++------ .../rollback/stage1-rollback-strategy.ts | 26 +++---- .../rollback/stage2-rollback-strategy.ts | 32 +++++++-- .../rollback/stage3-rollback-strategy.ts | 33 +++++++-- 6 files changed, 134 insertions(+), 81 deletions(-) diff --git a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/crash-manager.ts b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/crash-manager.ts index 5ea2841b7b0..1186c50056b 100644 --- a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/crash-manager.ts +++ b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/crash-manager.ts @@ -149,41 +149,51 @@ export class CrashRecoveryManager { this.log.info(`${fnTag} Checking crash status for session ${sessionId}`); try { - const crashStatus = await this.checkCrash(sessionId); - if (crashStatus === CrashStatus.IN_RECOVERY) { - // create new occurrence - const crashOccurrence = new CrashOccurrence( - CrashStatus.IN_RECOVERY, - new Date(), - new Date(), - ); - this.log.debug(crashOccurrence); - // todo manage occurrence - // call corresponding services via handler for crash recovery - const sessionData = sessionId.hasClientSessionData() - ? sessionId.getClientSessionData() - : sessionId.getServerSessionData(); - - const lastSequenceNumber = BigInt(sessionData.lastSequenceNumber); - const maxRetries = BigInt(sessionData.maxRetries); - - if (lastSequenceNumber > maxRetries) { - this.log.warn( - `${fnTag} Max retries exceeded for session ${sessionId}. Initiating rollback...`, - ); - await this.initiateRollback(sessionId, true); - } else { + const sessionData = sessionId.hasClientSessionData() + ? sessionId.getClientSessionData() + : sessionId.getServerSessionData(); + + if (!sessionData) { + throw new Error(`${fnTag}, session data is not correctly initialized`); + } + + let attempts = 0; + let crashOccurrence: CrashOccurrence | undefined; + + while (attempts < BigInt(sessionData.maxRetries)) { + const crashStatus = await this.checkCrash(sessionId); + + if (crashStatus === CrashStatus.IN_RECOVERY) { this.log.info( - `${fnTag} Attempting recovery for session ${sessionId}`, + `${fnTag} Crash detected. Attempting recovery for session ${sessionId}`, ); + + if (!crashOccurrence) { + crashOccurrence = new CrashOccurrence( + CrashStatus.IN_RECOVERY, + new Date(), + new Date(), + ); + } else { + crashOccurrence.lastUpdate = new Date(); + } + await this.handleRecovery(sessionId); + this.log.info(`${fnTag} Recovery successful.`); + + crashOccurrence.status = CrashStatus.RECOVERED; + + return true; } - crashOccurrence.status = CrashStatus.RECOVERED; - return true; - } else { - this.log.info(`${fnTag} No crash detected. No action required.`); - return false; + attempts++; + this.log.info( + `${fnTag} Retry attempt ${attempts} for session ${sessionId}`, + ); } + + this.log.warn(`${fnTag} All retries exhausted. Initiating rollback.`); + await this.initiateRollback(sessionId, true); + return false; } catch (error) { this.log.error(`${fnTag} Error during crash resolution: ${error}`); return false; diff --git a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/rollback-strategy-factory.ts b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/rollback-strategy-factory.ts index 67d1d24d32d..901d2194bea 100644 --- a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/rollback-strategy-factory.ts +++ b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/rollback-strategy-factory.ts @@ -37,25 +37,26 @@ export class RollbackStrategyFactory { const sessionData = session.hasClientSessionData() ? session.getClientSessionData()! : session.getServerSessionData()!; + const rollbackLogEntry = new RollbackLogEntry(); if (!sessionData.hashes) { this.log.debug(`${fnTag} Creating Stage0RollbackStrategy`); - return new Stage0RollbackStrategy(); + return new Stage0RollbackStrategy(rollbackLogEntry); } else if ( !sessionData.hashes.stage2 || Object.keys(sessionData.hashes.stage2).length === 0 ) { this.log.debug(`${fnTag} Creating Stage1RollbackStrategy`); - return new Stage1RollbackStrategy(this.bridgeManager); + return new Stage1RollbackStrategy(this.bridgeManager, rollbackLogEntry); } else if ( !sessionData.hashes.stage3 || Object.keys(sessionData.hashes.stage3).length === 0 ) { this.log.debug(`${fnTag} Creating Stage2RollbackStrategy`); - return new Stage2RollbackStrategy(); + return new Stage2RollbackStrategy(this.bridgeManager, rollbackLogEntry); } else { this.log.debug(`${fnTag} Creating Stage3RollbackStrategy`); - return new Stage3RollbackStrategy(); + return new Stage3RollbackStrategy(this.bridgeManager, rollbackLogEntry); } } } diff --git a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage0-rollback-strategy.ts b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage0-rollback-strategy.ts index 323d247f0a8..fd2b5ec709c 100644 --- a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage0-rollback-strategy.ts +++ b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage0-rollback-strategy.ts @@ -5,9 +5,11 @@ import { RollbackLogEntry } from "../../../generated/proto/cacti/satp/v02/crash_ export class Stage0RollbackStrategy implements RollbackStrategy { private log: Logger; + private rollbackLogEntry: RollbackLogEntry; - constructor() { + constructor(logEntry: RollbackLogEntry) { this.log = LoggerProvider.getOrCreate({ label: "Stage0RollbackStrategy" }); + this.rollbackLogEntry = logEntry; } // return a rollback state in all strategies @@ -17,46 +19,43 @@ export class Stage0RollbackStrategy implements RollbackStrategy { // check session exists if (!session) { - this.log.error(`${fnTag} Session not found`); - return false; + throw new Error(`${fnTag}, session data is not correctly initialized`); } try { // TODO record the rollback on the log. Implement RollbackLogEntry this.log.debug("Persisting rollback log entry"); - const rollbackLogEntry: RollbackLogEntry = new RollbackLogEntry(); - rollbackLogEntry.sessionId = session.getSessionId(); - rollbackLogEntry.stage = "Stage0"; - rollbackLogEntry.timestamp = Date.now().toString(); - rollbackLogEntry.action = "Rollback Initiated"; - rollbackLogEntry.status = "SUCCESS"; - rollbackLogEntry.details = "Stage 0 rollback executed without issues."; - - await this.localLogs.saveRollbackLog(rollbackLogEntry); // log for the rollbackentry + this.rollbackLogEntry.sessionId = session.getSessionId(); + this.rollbackLogEntry.stage = "Stage0"; + this.rollbackLogEntry.timestamp = Date.now().toString(); + this.rollbackLogEntry.action = ""; + this.rollbackLogEntry.status = "SUCCESS"; + this.rollbackLogEntry.details = ""; this.log.info(`Successfully rolled back Stage 0`); const state: RollbackState = { currentStage: "Stage0", - rollbackLogEntry: rollbackLogEntry, + rollbackLogEntry: this.rollbackLogEntry, }; + await this.rollbackLogs.create(state); // todo: log for the rollbackentry return state; } catch (error) { this.log.error(`Failed to rollback Stage 0: ${error}`); - const rollbackLogEntry: RollbackLogEntry = new RollbackLogEntry(); - rollbackLogEntry.sessionId = session.getSessionId(); // session Id from satp session data - rollbackLogEntry.stage = "Stage0"; - rollbackLogEntry.timestamp = Date.now().toString(); - rollbackLogEntry.action = "Rollback Failed"; - rollbackLogEntry.status = "FAILURE"; - rollbackLogEntry.details = "Stage 0 rollback executed without issues."; - await this.localLogs.saveRollbackLog(rollbackLogEntry); // todo: implement the correct log support + this.rollbackLogEntry.sessionId = session.getSessionId(); + this.rollbackLogEntry.stage = "Stage0"; + this.rollbackLogEntry.timestamp = Date.now().toString(); + this.rollbackLogEntry.action = ""; + this.rollbackLogEntry.status = "FAILURE"; + this.rollbackLogEntry.details = ""; + const state: RollbackState = { currentStage: "Stage0", - rollbackLogEntry: rollbackLogEntry, + rollbackLogEntry: this.rollbackLogEntry, }; + await this.rollbackLogs.create(state); // todo: implement the correct log support return state; } } @@ -64,10 +63,8 @@ export class Stage0RollbackStrategy implements RollbackStrategy { async cleanup(session: SATPSession): Promise { const fnTag = "Stage0RollbackStrategy#cleanup"; // for stage 0, do nothing - const rollbackLogEntry: RollbackLogEntry = new RollbackLogEntry(); const state: RollbackState = { currentStage: "Stage0", - rollbackLogEntry: rollbackLogEntry, }; if (!session) { this.log.error(`${fnTag} Session not found`); diff --git a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage1-rollback-strategy.ts b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage1-rollback-strategy.ts index 8d75943181e..bd50d4ca1e8 100644 --- a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage1-rollback-strategy.ts +++ b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage1-rollback-strategy.ts @@ -7,10 +7,12 @@ import { SATPBridgeManager } from "../../stage-services/satp-bridge/satp-bridge- export class Stage1RollbackStrategy implements RollbackStrategy { private log: Logger; private bridgeManager: SATPBridgeManager; + private rollbackLogEntry: RollbackLogEntry; - constructor(bridgeManager: SATPBridgeManager) { + constructor(bridgeManager: SATPBridgeManager, logEntry: RollbackLogEntry) { this.log = LoggerProvider.getOrCreate({ label: "Stage1RollbackStrategy" }); this.bridgeManager = bridgeManager; + this.rollbackLogEntry = logEntry; } async execute(session: SATPSession): Promise { @@ -18,8 +20,7 @@ export class Stage1RollbackStrategy implements RollbackStrategy { this.log.info(`${fnTag} Executing rollback for Stage 1`); if (!session) { - this.log.error(`${fnTag} Session not found`); - return false; + throw new Error(`${fnTag}, session data is not correctly initialized`); } try { @@ -27,18 +28,16 @@ export class Stage1RollbackStrategy implements RollbackStrategy { // TODO: Record the rollback on the log. Implement RollbackLogEntry - const receipt = await this.bridgeManager.unwrapAsset("assetId"); // unwrap + const receipt = await this.bridgeManager.unwrapAsset("assetId"); this.log.info(`${fnTag}, Asset unlocked: ${receipt}`); - const rollbackLogEntry: RollbackLogEntry = { - sessionId: session.getSessionId(), - stage: "Stage1", - timestamp: new Date().toString(), - action: "Unwrap Asset", - status: "SUCCESS", - details: `Asset unwrap receipt: ${receipt}`, - }; + this.rollbackLogEntry.sessionId = session.getSessionId(); + this.rollbackLogEntry.stage = "Stage1"; + this.rollbackLogEntry.timestamp = Date.now().toString(); + this.rollbackLogEntry.action = "UNWRAP"; + this.rollbackLogEntry.status = "SUCCESS"; + this.rollbackLogEntry.details = ""; this.log.debug("Persisting rollback log entry"); @@ -46,9 +45,10 @@ export class Stage1RollbackStrategy implements RollbackStrategy { const state: RollbackState = { currentStage: "Stage1", - rollbackLogEntry: rollbackLogEntry, + rollbackLogEntry: this.rollbackLogEntry, }; + await this.rollbackLogs.create(state); // todo: log support return state; } catch (error) { this.log.error(`Failed to rollback Stage 1: ${error}`); diff --git a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage2-rollback-strategy.ts b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage2-rollback-strategy.ts index b984d767b6a..261e13384c6 100644 --- a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage2-rollback-strategy.ts +++ b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage2-rollback-strategy.ts @@ -1,12 +1,18 @@ import { Logger, LoggerProvider } from "@hyperledger/cactus-common"; -import { SATPSession } from "./satp-session"; +import { SATPSession } from "../../satp-session"; import { RollbackState, RollbackStrategy } from "./rollback-strategy-factory"; +import { RollbackLogEntry } from "../../../generated/proto/cacti/satp/v02/crash_recovery_pb"; +import { SATPBridgeManager } from "../../stage-services/satp-bridge/satp-bridge-manager"; export class Stage2RollbackStrategy implements RollbackStrategy { private log: Logger; + private bridgeManager: SATPBridgeManager; + private rollbackLogEntry: RollbackLogEntry; - constructor() { + constructor(bridgeManager: SATPBridgeManager, logEntry: RollbackLogEntry) { this.log = LoggerProvider.getOrCreate({ label: "Stage2RollbackStrategy" }); + this.bridgeManager = bridgeManager; + this.rollbackLogEntry = logEntry; } async execute(session: SATPSession): Promise { @@ -14,8 +20,7 @@ export class Stage2RollbackStrategy implements RollbackStrategy { this.log.info(`${fnTag} Executing rollback for Stage 2`); if (!session) { - this.log.error(`${fnTag} Session not found`); - return false; + throw new Error(`${fnTag}, session data is not correctly initialized`); } try { @@ -24,8 +29,25 @@ export class Stage2RollbackStrategy implements RollbackStrategy { // TODO: Record the rollback on the log. Implement RollbackLogEntry this.log.debug("Persisting rollback log entry"); + const receipt = await this.bridgeManager.unlockAsset("assetId", Number()); + + this.log.info(`${fnTag}, Asset unlocked: ${receipt}`); + this.rollbackLogEntry.sessionId = session.getSessionId(); + this.rollbackLogEntry.stage = "Stage2"; + this.rollbackLogEntry.timestamp = Date.now().toString(); + this.rollbackLogEntry.action = "UNLOCK"; + this.rollbackLogEntry.status = "SUCCESS"; + this.rollbackLogEntry.details = ""; + this.log.info(`Successfully rolled back Stage 2`); - return true; + + const state: RollbackState = { + currentStage: "Stage2", + rollbackLogEntry: this.rollbackLogEntry, + }; + + await this.rollbackLogs.create(state); // todo: log support + return state; } catch (error) { this.log.error(`Failed to rollback Stage 2: ${error}`); return false; diff --git a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage3-rollback-strategy.ts b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage3-rollback-strategy.ts index 498649cb698..1bf35232c4d 100644 --- a/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage3-rollback-strategy.ts +++ b/packages/cactus-plugin-satp-hermes/src/main/typescript/core/recovery/rollback/stage3-rollback-strategy.ts @@ -1,12 +1,18 @@ import { Logger, LoggerProvider } from "@hyperledger/cactus-common"; -import { SATPSession } from "./satp-session"; +import { SATPSession } from "../../satp-session"; import { RollbackState, RollbackStrategy } from "./rollback-strategy-factory"; +import { RollbackLogEntry } from "../../../generated/proto/cacti/satp/v02/crash_recovery_pb"; +import { SATPBridgeManager } from "../../stage-services/satp-bridge/satp-bridge-manager"; export class Stage3RollbackStrategy implements RollbackStrategy { private log: Logger; + private bridgeManager: SATPBridgeManager; + private rollbackLogEntry: RollbackLogEntry; - constructor() { + constructor(bridgeManager: SATPBridgeManager, logEntry: RollbackLogEntry) { this.log = LoggerProvider.getOrCreate({ label: "Stage3RollbackStrategy" }); + this.bridgeManager = bridgeManager; + this.rollbackLogEntry = logEntry; } async execute(session: SATPSession): Promise { @@ -14,18 +20,35 @@ export class Stage3RollbackStrategy implements RollbackStrategy { this.log.info(`${fnTag} Executing rollback for Stage 3`); if (!session) { - this.log.error(`${fnTag} Session not found`); - return false; + throw new Error(`${fnTag}, session data is not correctly initialized`); } try { // TODO: Implement Stage 3 specific rollback logic // TODO: Record the rollback on the log. Implement RollbackLogEntry + + const receipt = await this.bridgeManager.burnAsset("assetId", Number()); + + this.log.info(`${fnTag}, Asset unlocked: ${receipt}`); + + this.rollbackLogEntry.sessionId = session.getSessionId(); + this.rollbackLogEntry.stage = "Stage3"; + this.rollbackLogEntry.timestamp = Date.now().toString(); + this.rollbackLogEntry.action = "BURN"; + this.rollbackLogEntry.status = "SUCCESS"; + this.rollbackLogEntry.details = ""; + this.log.debug("Persisting rollback log entry"); this.log.info(`Successfully rolled back Stage 3`); - return true; + const state: RollbackState = { + currentStage: "Stage3", + rollbackLogEntry: this.rollbackLogEntry, + }; + + await this.rollbackLogs.create(state); // todo: log support + return state; } catch (error) { this.log.error(`Failed to rollback Stage 3: ${error}`); return false;