Add bouncer tests involving the full deposit monitor pipeline.

.github/workflows/upgrade-test.yml

@@ -255,6 +255,12 @@ jobs:
         run: |
           cat /tmp/chainflip/chainflip-lp-api.*log
 
+      - name: Print chainflip-deposit-monitor logs 🔬
+        if: always()
+        continue-on-error: true
+        run: |
+          cat /tmp/chainflip/chainflip-deposit-monitor.*log
+
       - name: Print localnet init debug logs 🕵️‍♂️
         if: always()
         continue-on-error: true

bouncer/shared/upgrade_network.ts

@@ -10,7 +10,6 @@ import { compileBinaries } from './utils/compile_binaries';
 import { submitRuntimeUpgradeWithRestrictions } from './submit_runtime_upgrade';
 import { execWithLog } from './utils/exec_with_log';
 import { submitGovernanceExtrinsic } from './cf_governance';
-import { setupLpAccount } from './setup_lp_account';
 
 async function readPackageTomlVersion(projectRoot: string): Promise<string> {
   const data = await fs.readFile(path.join(projectRoot, '/state-chain/runtime/Cargo.toml'), 'utf8');
@@ -339,17 +338,16 @@ export async function upgradeNetworkPrebuilt(
     );
   }
 
-  // Temp: until localnet/bouncer initialises to a version where the LP_API is funded already.
-  if (cleanOldVersion.startsWith('1.6')) {
-    console.log('Setting up LP account and adding liquidity for the LP-API.');
-    // Liquidity is provided as part of the LP-API test setup.
-    await setupLpAccount('//LP_API');
-    // Write LP_API key to keys/ so that the LP-API can use it - when upgrading the old version, which the upgrade-test is
-    // started from doesn't yet have this key.
-    await fs.writeFile(
-      `${localnetInitPath}/keys/LP_API`,
-      '8e1866e65039304e4142f09452a8305acd28d0ae0b833cd268b21a57d68782c1',
-    );
+  // Temp: until localnet/bouncer initialises to a version where the deposit-monitor is started already.
+  if (cleanOldVersion.startsWith('1.7')) {
+    console.log('Starting up deposit-monitor.');
+
+    execWithLog(`${localnetInitPath}/scripts/start-deposit-monitor.sh`, 'start-deposit-monitor', {
+      LOCALNET_INIT_DIR: `${localnetInitPath}`,
+      DEPOSIT_MONITOR_CONTAINER: 'deposit-monitor',
+      DOCKER_COMPOSE_CMD: 'docker compose',
+      additional_docker_compose_up_args: '--quiet-pull',
+    });
   }
 
   if (cleanOldVersion === nodeVersion) {

bouncer/tests/all_concurrent_tests.ts

@@ -23,8 +23,6 @@ async function runAllConcurrentTests() {
   const match = process.argv[2] ? process.argv[2].match(/\d+/) : null;
   const givenNumberOfNodes = match ? parseInt(match[0]) : null;
   const numberOfNodes = givenNumberOfNodes ?? 1;
-  // If the third argument is not explicitly false, we assume it's true and we are in a localnet environment.
-  const addConcurrentLocalnetTests = process.argv[3] !== 'false';
 
   const broadcastAborted = observeBadEvent(':BroadcastAborted', {
     label: 'Concurrent broadcast aborted',
@@ -48,6 +46,7 @@ async function runAllConcurrentTests() {
     testCancelOrdersBatch.run(),
     depositChannelCreation.run(),
     testBtcVaultSwap.run(),
+    testBrokerLevelScreening.run(),
   ];
 
   // Tests that only work if there is more than one node
@@ -57,12 +56,6 @@ async function runAllConcurrentTests() {
     tests.push(...multiNodeTests);
   }
 
-  // Tests that only work with localnet but can be run concurrent.
-  if (addConcurrentLocalnetTests) {
-    const localnetTests = [testBrokerLevelScreening.run()];
-    tests.push(...localnetTests);
-  }
-
   await Promise.all(tests);
 
   await Promise.all([broadcastAborted.stop(), feeDeficitRefused.stop()]);

bouncer/tests/broker_level_screening.ts

@@ -1,14 +1,14 @@
+import axios from 'axios';
 import { randomBytes } from 'crypto';
-import { execSync } from 'child_process';
 import { InternalAsset } from '@chainflip/cli';
 import { ExecutableTest } from '../shared/executable_test';
 import { sendBtc } from '../shared/send_btc';
 import {
-  hexStringToBytesArray,
   newAddress,
   sleep,
   handleSubstrateError,
   brokerMutex,
+  hexStringToBytesArray,
 } from '../shared/utils';
 import { getChainflipApi, observeEvent } from '../shared/utils/substrate';
 import Keyring from '../polkadot/keyring';
@@ -56,35 +56,89 @@ async function newAssetAddress(asset: InternalAsset, seed = null): Promise<strin
 /**
  * Submits a transaction as tainted to the extrinsic on the state chain.
  *
- * @param txId - The txId to submit as tainted as byte array in the order it is on the Bitcoin chain - which
- * is reverse of how it's normally displayed in block explorers.
+ * @param txId - The txId to submit as tainted, in its typical representation in bitcoin explorers,
+ * i.e., reverse of its memory representation.
  */
-async function submitTxAsTainted(txId: number[]) {
+async function submitTxAsTainted(txId: string) {
+  // The engine uses the memory representation everywhere, so we convert the txId here.
+  const memoryRepresentationTxId = hexStringToBytesArray(txId).reverse();
   await using chainflip = await getChainflipApi();
   return brokerMutex.runExclusive(async () =>
     chainflip.tx.bitcoinIngressEgress
-      .markTransactionAsTainted(txId)
+      .markTransactionAsTainted(memoryRepresentationTxId)
       .signAndSend(broker, { nonce: -1 }, handleSubstrateError(chainflip)),
   );
 }
 
 /**
- * Pauses or resumes the bitcoin block production. We send a command to the docker container to start or stop mining blocks.
+ * Submit a post request to the deposit-monitor, with error handling.
+ * @param portAndRoute Where we want to submit the request to.
+ * @param body The request body, is serialized as JSON.
+ */
+async function postToDepositMonitor(portAndRoute: string, body: string | object) {
+  return axios
+    .post('http://127.0.0.1' + portAndRoute, JSON.stringify(body), {
+      headers: {
+        'Content-Type': 'application/json',
+        Accept: 'application/json',
+      },
+      timeout: 5000,
+    })
+    .then((res) => res.data)
+    .catch((error) => {
+      let message;
+      if (error.response) {
+        message = error.response.data + ' (' + error.response.status + ')';
+      } else {
+        message = error;
+      }
+      throw new Error('Request to deposit monitor (' + portAndRoute + ') failed: ' + message);
+    });
+}
+
+/**
+ * Typescript representation of the allowed parameters to `setMockmode`. The JSON encoding of these
+ * is what the deposit-monitor expects.
+ */
+type Mockmode =
+  | 'Manual'
+  | { Deterministic: { score: number; incomplete_probability: number } }
+  | { Random: { min_score: number; max_score: number; incomplete_probability: number } };
+
+/**
+ * Set the mockmode of the deposit monitor, controlling how it analyses incoming transactions.
+ *
+ * @param mode Object describing the mockmode we want to set the deposit-monitor to,
+ */
+async function setMockmode(mode: Mockmode) {
+  return postToDepositMonitor(':6070/mockmode', mode);
+}
+
+/**
+ * Call the deposit-monitor to set risk score of given transaction in mock analysis provider.
  *
- * @param pause - Whether to pause or resume the block production.
- * @returns - Whether the command was successful.
+ * @param txid Hash of the transaction we want to report.
+ * @param score Risk score for this transaction. Can be in range [0.0, 10.0].
  */
-function pauseBtcBlockProduction(pause: boolean): boolean {
-  try {
-    execSync(
-      pause
-        ? 'docker exec bitcoin rm /root/mine_blocks'
-        : 'docker exec bitcoin touch /root/mine_blocks',
+async function setTxRiskScore(txid: string, score: number) {
+  await postToDepositMonitor(':6070/riskscore', [
+    txid,
+    {
+      risk_score: { Score: score },
+      unknown_contribution_percentage: 0.0,
+    },
+  ]);
+}
+
+/**
+ * Checks that the deposit monitor has started up successfully and is healthy.
+ */
+async function ensureHealth() {
+  const response = await postToDepositMonitor(':6060/health', {});
+  if (response.starting === true || response.all_processors === false) {
+    throw new Error(
+      "Deposit monitor is running, but not healthy. It's response was: " + JSON.stringify(response),
     );
-    return true;
-  } catch (error) {
-    console.error(error);
-    return false;
   }
 }
 
@@ -94,16 +148,13 @@ function pauseBtcBlockProduction(pause: boolean): boolean {
  * @param amount - The deposit amount.
  * @param doBoost - Whether to boost the deposit.
  * @param refundAddress - The address to refund to.
- * @param stopBlockProductionFor - The number of blocks to stop block production for. We need this to ensure that the tainted tx is on chain before the deposit is witnessed/prewitnessed.
- * @param waitBeforeReport - The number of milliseconds to wait before reporting the tx as tainted.
  * @returns - The the channel id of the deposit channel.
  */
 async function brokerLevelScreeningTestScenario(
   amount: string,
   doBoost: boolean,
   refundAddress: string,
-  stopBlockProductionFor = 0,
-  waitBeforeReport = 0,
+  reportFunction: (txId: string) => Promise<void>,
 ): Promise<string> {
   const destinationAddressForUsdc = await newAssetAddress('Usdc');
   const refundParameters: FillOrKillParamsX128 = {
@@ -122,20 +173,9 @@ async function brokerLevelScreeningTestScenario(
     doBoost ? 100 : 0,
     refundParameters,
   );
-  const txId = await sendBtc(swapParams.depositAddress, amount);
-  if (stopBlockProductionFor > 0) {
-    pauseBtcBlockProduction(true);
-  }
-  await sleep(waitBeforeReport);
-  // Note: The bitcoin core js lib returns the txId in reverse order.
-  // On chain we expect the txId to be in the correct order (like the Bitcoin internal representation).
-  // Because of this we need to reverse the txId before submitting it as tainted.
-  await submitTxAsTainted(hexStringToBytesArray(txId).reverse());
-  await sleep(stopBlockProductionFor);
-  if (stopBlockProductionFor > 0) {
-    pauseBtcBlockProduction(false);
-  }
-  return Promise.resolve(swapParams.channelId.toString());
+  const txId = await sendBtc(swapParams.depositAddress, amount, 0);
+  await reportFunction(txId);
+  return swapParams.channelId.toString();
 }
 
 // -- Test suite for broker level screening --
@@ -147,17 +187,17 @@ async function brokerLevelScreeningTestScenario(
 // 3. Boost and late tx report -> Tainted tx is reported late and the swap is not refunded.
 async function main() {
   const MILLI_SECS_PER_BLOCK = 6000;
-  const BLOCKS_TO_WAIT = 2;
+
+  // 0. -- Ensure that deposit monitor is running with manual mocking mode --
+  await ensureHealth();
+  const previousMockmode = (await setMockmode('Manual')).previous;
 
   // 1. -- Test no boost and early tx report --
   testBrokerLevelScreening.log('Testing broker level screening with no boost...');
   let btcRefundAddress = await newAssetAddress('Btc');
 
-  await brokerLevelScreeningTestScenario(
-    '0.2',
-    false,
-    btcRefundAddress,
-    MILLI_SECS_PER_BLOCK * BLOCKS_TO_WAIT,
+  await brokerLevelScreeningTestScenario('0.2', false, btcRefundAddress, async (txId) =>
+    setTxRiskScore(txId, 9.0),
   );
 
   await observeEvent('bitcoinIngressEgress:TaintedTransactionRejected').event;
@@ -173,11 +213,8 @@ async function main() {
   );
   btcRefundAddress = await newAssetAddress('Btc');
 
-  await brokerLevelScreeningTestScenario(
-    '0.2',
-    true,
-    btcRefundAddress,
-    MILLI_SECS_PER_BLOCK * BLOCKS_TO_WAIT,
+  await brokerLevelScreeningTestScenario('0.2', true, btcRefundAddress, async (txId) =>
+    setTxRiskScore(txId, 9.0),
   );
   await observeEvent('bitcoinIngressEgress:TaintedTransactionRejected').event;
 
@@ -195,13 +232,21 @@ async function main() {
     '0.2',
     true,
     btcRefundAddress,
-    0,
-    MILLI_SECS_PER_BLOCK * BLOCKS_TO_WAIT,
+    // We wait 12 seconds (2 localnet btc blocks) before we submit the tx.
+    // We submit the extrinsic manually in order to ensure that even though it definitely arrives,
+    // the transaction is refunded because the extrinsic is submitted too late.
+    async (txId) => {
+      await sleep(MILLI_SECS_PER_BLOCK * 2);
+      await submitTxAsTainted(txId);
+    },
   );
 
   await observeEvent('bitcoinIngressEgress:DepositFinalised', {
     test: (event) => event.data.channelId === channelId,
   }).event;
 
   testBrokerLevelScreening.log(`Swap was executed and tainted transaction was not refunded 👍.`);
+
+  // 4. -- Restore mockmode --
+  await setMockmode(previousMockmode);
 }

localnet/common.sh

@@ -4,6 +4,7 @@ export GENESIS_NODES=("bashful" "doc" "dopey")
 export REQUIRED_BINARIES="engine-runner chainflip-node chainflip-broker-api chainflip-lp-api"
 export INIT_CONTAINERS="eth-init solana-init"
 export CORE_CONTAINERS="bitcoin geth polkadot redis"
+export DEPOSIT_MONITOR_CONTAINER="deposit-monitor"
 export ARB_CONTAINERS="sequencer staker-unsafe poster"
 export SOLANA_BASE_PATH="/tmp/solana"
 export CHAINFLIP_BASE_PATH="/tmp/chainflip"
@@ -172,6 +173,13 @@ build-localnet() {
   echo "🤑 Starting LP API ..."
   KEYS_DIR=$KEYS_DIR ./$LOCALNET_INIT_DIR/scripts/start-lp-api.sh $BINARY_ROOT_PATH
 
+  echo "🔬 Starting Deposit Monitor ..."
+  LOCALNET_INIT_DIR=$LOCALNET_INIT_DIR \
+  DOCKER_COMPOSE_CMD=$DOCKER_COMPOSE_CMD \
+  DEPOSIT_MONITOR_CONTAINER=$DEPOSIT_MONITOR_CONTAINER \
+  additional_docker_compose_up_args=$additional_docker_compose_up_args \
+  ./$LOCALNET_INIT_DIR/scripts/start-deposit-monitor.sh
+
   if [[ $START_TRACKER == "y" ]]; then
     echo "👁 Starting Ingress-Egress-tracker ..."
     KEYS_DIR=$KEYS_DIR ./$LOCALNET_INIT_DIR/scripts/start-ingress-egress-tracker.sh $BINARY_ROOT_PATH

localnet/docker-compose.yml

@@ -159,3 +159,35 @@ services:
       timeout: 5s
       retries: 5
       start_period: 10s
+
+  deposit-monitor:
+    image: ghcr.io/chainflip-io/chainflip-deposit-monitor/chainflip-deposit-monitor:main-fb5b372059be6819454e03f0f1913fb77d999fcc
+    platform: linux/amd64
+    container_name: deposit-monitor
+    restart: unless-stopped
+    pull_policy: always
+    ports:
+      - 6060:6060
+      - 6070:6070
+    stop_signal: SIGINT
+    stop_grace_period: 5s
+    volumes:
+      - "/tmp/chainflip/data/deposit-monitor:/persistent_state"
+    environment:
+      - CFDM_RUNNING_IN_MOCK_ENVIRONMENT=true
+      - CFDM_BITCOIN_NETWORK=regtest
+      - CFDM_PERSISTENT_STATE_DIR=/persistent_state
+      - CFDM_CRITICAL_ELLIPTIC_RISK_SCORE=8.0
+      - CFDM_BROKER_API_URL=${CFDM_BROKER_API_URL}
+      - CFDM_BTC_RPC_HTTP_ENDPOINT=http://bitcoin:8332
+      - CFDM_BTC_RPC_BASIC_AUTH_USER=flip
+      - CFDM_BTC_RPC_BASIC_AUTH_PASSWORD=flip
+      - CFDM_OUTPUT_PROCESSOR_PORT=6060
+      - CFDM_MOCK_ANALYSIS_PROVIDER_PORT=6070
+      - CFDM_ANALYSIS_PROVIDER_SELECTION=Mock
+      - CFDM_REFUND_PROVIDER_SELECTION=BrokerApi
+      - CFDM_STATECHAIN_DEPOSIT_CHANNELS_POLLING_INTERVAL_MILLIS=500
+      - CFDM_BTC_MEMPOOL_POLLING_INTERVAL_MILLIS=500
+      - CFDM_BTC_CHAIN_POLLING_INTERVAL_MILLIS=2000
+      - CFDM_TRANSACTION_PROCESSOR_INCOMPLETE_ANALYSIS_TIMEOUT_SECS=120
+

localnet/init/scripts/start-deposit-monitor.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+DATETIME=$(date '+%Y-%m-%d_%H-%M-%S')
+
+source $LOCALNET_INIT_DIR/../helper.sh
+
+# On some machines (e.g. MacOS), 172.17.0.1 is not accessible from inside the container, so we need to use host.docker.internal
+if [[ $CI == true ]]; then
+  export CFDM_BROKER_API_URL='ws://172.17.0.1:10997'
+else
+  export CFDM_BROKER_API_URL='ws://host.docker.internal:10997'
+fi
+$DOCKER_COMPOSE_CMD -f $LOCALNET_INIT_DIR/../docker-compose.yml -p "chainflip-localnet" up $DEPOSIT_MONITOR_CONTAINER $additional_docker_compose_up_args -d \
+  > /tmp/chainflip/chainflip-deposit-monitor.$DATETIME.log 2>&1
+
+while true; do
+  echo "🩺 Checking deposit-monitor's health ..."
+  REPLY=$(check_endpoint_health 'http://localhost:6060/health')
+  starting=$(echo $REPLY | jq .starting)
+  all_healthy=$(echo $REPLY | jq .all_processors)
+  if test "$starting" == "false" && test "$all_healthy" == "true" ; then
+    echo "💚 deposit-monitor is running!"
+    break
+  fi
+  sleep 1
+done