Stale redemptions monitoring (#682)

Refs: keep-network/keep-core#3664
Depends on: #678

This changeset adds one new system event to the monitoring system:

### Stale redemption

A **warning system event** indicating that a redemption request became
stale, i.e. was not handled within the expected time. This event is sent
to Sentry hub and should get team’s attention. The default action is
investigating the cause of the extended processing time as this alert
may be an early sign of a malfunctioning wallet or may indicate a
problem with the maintainer bot.

monitoring/docs/monitoring-and-telemetry.adoc

@@ -118,6 +118,7 @@ events produced by the monitoring component are:
 * DKG result challenged,
 * large deposit revealed,
 * large redemption requested,
+* stale redemption,
 * optimistic minting canceled,
 * optimistic minting requested too early,
 * optimistic minting requested for undetermined Bitcoin transaction,
@@ -180,6 +181,14 @@ team’s attention. The default action is making sure that the redemption is
 not a result of a malicious action, and if not, that the redemption is
 handled correctly by the system.
 
+=== Stale redemption
+
+A *warning system event* indicating that a redemption request became stale, i.e.
+was not handled within the expected time. This event is sent to Sentry hub and
+should get team’s attention. The default action is investigating the cause
+of the extended processing time as this alert may be an early sign of
+a malfunctioning wallet or may indicate a problem with the maintainer bot.
+
 ==== Optimistic minting cancelled
 
 A *warning system event* indicating that an optimistic minting request was

monitoring/src/contracts.ts

@@ -39,6 +39,11 @@ const resolve = () => {
     return block.number
   }
 
+  const blockTimestamp = async (blockNumber: number): Promise<number> => {
+    const block = await provider.getBlock(blockNumber)
+    return block.timestamp
+  }
+
   // eslint-disable-next-line @typescript-eslint/no-var-requires,global-require,import/no-dynamic-require
   const bridgeArtifact = require(`${packageName}/artifacts/Bridge.json`)
   const bridge: Bridge = new EthereumBridge({
@@ -63,7 +68,7 @@ const resolve = () => {
     deployedAtBlockNumber: tbtcTokenArtifact.receipt.blockNumber,
   })
 
-  return { bridge, tbtcVault, tbtcToken, latestBlock }
+  return { bridge, tbtcVault, tbtcToken, latestBlock, blockTimestamp }
 }
 
 export const contracts = resolve()

monitoring/src/redemption-monitor.ts

@@ -1,13 +1,21 @@
 import { BigNumber } from "ethers"
+import { Hex } from "@keep-network/tbtc-v2.ts"
 
 import { context } from "./context"
 import { SystemEventType } from "./system-event"
 import { satsToRoundedBTC } from "./deposit-monitor"
 import { createEthTxUrl } from "./block-explorer"
+import { contracts } from "./contracts"
 
+import type { Monitor as SystemEventMonitor, SystemEvent } from "./system-event"
 import type { RedemptionRequestedEvent as RedemptionRequestedChainEvent } from "@keep-network/tbtc-v2.ts/dist/src/redemption"
 import type { Bridge } from "@keep-network/tbtc-v2.ts/dist/src/chain"
-import type { Monitor as SystemEventMonitor, SystemEvent } from "./system-event"
+
+// The time after which a pending redemption request is considered stale.
+// Typically, a redemption request processing time should not exceed 5 hours.
+// A redemption request pending for 8 hours indicates that something is wrong.
+// This value is expressed in blocks, assuming 12 seconds per block.
+const redemptionRequestStaleBlocks = (8 * 60 * 60) / 12
 
 const RedemptionRequested = (
   chainEvent: RedemptionRequestedChainEvent
@@ -47,6 +55,25 @@ const LargeRedemptionRequested = (
   }
 }
 
+const StaleRedemption = (
+  chainEvent: RedemptionRequestedChainEvent
+): SystemEvent => {
+  const ethRequestTxHashURL = createEthTxUrl(chainEvent.transactionHash)
+
+  return {
+    title: "Stale redemption",
+    type: SystemEventType.Warning,
+    data: {
+      walletPublicKeyHash: chainEvent.walletPublicKeyHash,
+      redeemerOutputScript: chainEvent.redeemerOutputScript,
+      requestedAmountBTC: satsToRoundedBTC(chainEvent.requestedAmount),
+      ethRequestTxHash: chainEvent.transactionHash.toPrefixedString(),
+      ethRequestTxHashURL,
+    },
+    block: chainEvent.blockNumber,
+  }
+}
+
 export class RedemptionMonitor implements SystemEventMonitor {
   private bridge: Bridge
 
@@ -58,6 +85,22 @@ export class RedemptionMonitor implements SystemEventMonitor {
     // eslint-disable-next-line no-console
     console.log("running redemption monitor check")
 
+    const systemEvents: SystemEvent[] = []
+    systemEvents.push(
+      ...(await this.checkRequestedRedemptions(fromBlock, toBlock))
+    )
+    systemEvents.push(...(await this.checkStaleRedemptions(fromBlock, toBlock)))
+
+    // eslint-disable-next-line no-console
+    console.log("completed redemption monitor check")
+
+    return systemEvents
+  }
+
+  private async checkRequestedRedemptions(
+    fromBlock: number,
+    toBlock: number
+  ): Promise<SystemEvent[]> {
     const chainEvents = await this.bridge.getRedemptionRequestedEvents({
       fromBlock,
       toBlock,
@@ -80,9 +123,73 @@ export class RedemptionMonitor implements SystemEventMonitor {
       }
     }
 
-    // eslint-disable-next-line no-console
-    console.log("completed redemption monitor check")
-
     return systemEvents
   }
+
+  private async checkStaleRedemptions(
+    fromBlock: number,
+    toBlock: number
+  ): Promise<SystemEvent[]> {
+    const rewindBlock = (block: number, shift: number) =>
+      block - shift > 0 ? block - shift : 0
+
+    // We need to rewind the block window by the minting request timeout.
+    // This way, we are looking for past deposits whose time for creating
+    // the minting request was already elapsed.
+    const chainEvents = await this.bridge.getRedemptionRequestedEvents({
+      fromBlock: rewindBlock(fromBlock, redemptionRequestStaleBlocks),
+      toBlock: rewindBlock(toBlock, redemptionRequestStaleBlocks),
+    })
+
+    const chainEventsTimestamps = await Promise.all(
+      chainEvents.map((ce) => contracts.blockTimestamp(ce.blockNumber))
+    )
+
+    // To fetch pending redemptions requests, we need to know the plain-text
+    // public keys of the wallets used by the given chain events. In order to
+    // achieve that, we build a cache where the key is the wallet public key
+    // hash and the value is the wallet plain text public key.
+    //
+    // TODO: This can be optimized by refactoring the Bridge.pendingRedemptions
+    //       method to accept wallet public key hashes directly.
+    const walletCache = new Map<string, string>()
+    // eslint-disable-next-line no-plusplus
+    for (let i = 0; i < chainEvents.length; i++) {
+      const { walletPublicKeyHash } = chainEvents[i]
+
+      if (!walletCache.has(walletPublicKeyHash)) {
+        // eslint-disable-next-line no-await-in-loop
+        const wallet = await this.bridge.wallets(Hex.from(walletPublicKeyHash))
+        walletCache.set(walletPublicKeyHash, wallet.walletPublicKey.toString())
+      }
+    }
+
+    const pendingRedemptionsRequests = await Promise.all(
+      chainEvents.map((ce) =>
+        this.bridge.pendingRedemptions(
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          walletCache.get(ce.walletPublicKeyHash)!,
+          ce.redeemerOutputScript
+        )
+      )
+    )
+
+    return chainEvents
+      .filter((ce, index) => {
+        const pendingRedemptionRequest = pendingRedemptionsRequests[index]
+        const chainEventTimestamp = chainEventsTimestamps[index]
+
+        // To consider a redemption as stale, the redemption request must be
+        // still pending after redemptionRequestStaleBlocks. As there can
+        // be multiple redemption requests for the given wallet and
+        // redeemer output script pair, we need to make sure the pending
+        // redemption request timestamp matches the timestamp of the given
+        // chain event emission block.
+        return (
+          pendingRedemptionRequest.requestedAt !== 0 &&
+          pendingRedemptionRequest.requestedAt === chainEventTimestamp
+        )
+      })
+      .map(StaleRedemption)
+  }
 }