feat: automate incompatible CFE upgrade testing (#4196)

* refactor: bump specVersionAgainstNetwork

* chore: remove publish = false

* chore: fix comment

* feat: test incompatible CFE upgrades

* doc: add cargo-workspaces comment

bouncer/commands/simple_runtime_upgrade.ts

@@ -7,7 +7,7 @@
 // Optional args:
 // -test: Run the swap tests after the upgrade.
 //
-// For example ./commands/noop_runtime_upgrade.ts
+// For example ./commands/simple_runtime_upgrade.ts
 // NB: It *must* be run from the bouncer directory.
 
 import path from 'path';

bouncer/commands/submit_runtime_upgrade.ts

@@ -9,7 +9,7 @@
 //
 // For example: ./commands/submit_runtime_upgrade.ts /path/to/state_chain_runtime.compact.compressed.wasm '{"major": 1, "minor": 2, "patch": 3}' 50
 
-import { submitRuntimeUpgrade } from '../shared/submit_runtime_upgrade';
+import { submitRuntimeUpgradeWithRestrictions } from '../shared/submit_runtime_upgrade';
 import { runWithTimeout } from '../shared/utils';
 
 async function main() {
@@ -21,7 +21,7 @@ async function main() {
   const arg4 = process.argv[4].trim();
   const percentNodesUpgraded = arg4 ? Number(arg4) : undefined;
 
-  await submitRuntimeUpgrade(wasmPath, semverRestriction, percentNodesUpgraded);
+  await submitRuntimeUpgradeWithRestrictions(wasmPath, semverRestriction, percentNodesUpgraded);
   process.exit(0);
 }
 

bouncer/commands/upgrade_network.ts

@@ -3,30 +3,44 @@
 // Upgrades a localnet network to a new version.
 // Start a network with the version you want to upgrade from. Then run this command, providing the git reference (commit, branch, tag) you wish to upgrade to.
 //
+// PRE-REQUISITES:
+// - cargo workspaces must be installed - `cargo install cargo-workspaces`
+//
 // Optional args:
-// patch/minor/major: If the version of the commit we're upgrading to is the same as the version of the commit we're upgrading from, we bump the version by the specified level.
+// --git <git ref>: The git reference (commit, branch, tag) you wish to upgrade to.
+// --bump <patch/minor/major>: If the version of the commit we're upgrading to is the same as the version of the commit we're upgrading from, we bump the version by the specified level.
+// --nodes <1 or 3>: The number of nodes running on your localnet. Defaults to 1.
 //
 // For example: ./commands/upgrade_network.ts v0.10.1
-// or: ./commands/upgrade_network.ts v0.10.1 patch
+// or: ./commands/upgrade_network.ts --git 0.10.1 --bump major --nodes 3
+
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
 
 import { upgradeNetwork } from '../shared/upgrade_network';
 import { runWithTimeout } from '../shared/utils';
 
 async function main(): Promise<void> {
-  const upgradeTo = process.argv[2]?.trim();
+  const argv = yargs(hideBin(process.argv)).argv;
+
+  const upgradeTo = argv.git;
 
   if (!upgradeTo) {
     console.error('Please provide a git reference to upgrade to.');
     process.exit(-1);
   }
 
-  const optBumptTo: string = process.argv[3]?.trim().toLowerCase();
-  if (optBumptTo === 'patch' || optBumptTo === 'minor' || optBumptTo === 'major') {
-    await upgradeNetwork(upgradeTo, optBumptTo);
-  } else {
-    await upgradeNetwork(upgradeTo);
+  const optBumptTo: string = argv.bump ? argv.bump.toString().toLowerCase() : 'patch';
+  if (optBumptTo !== 'patch' && optBumptTo !== 'minor' && optBumptTo !== 'major') {
+    console.error('Please provide a valid bump level: patch, minor, or major.');
+    process.exit(-1);
   }
 
+  let numberOfNodes = argv.nodes ? argv.nodes : 1;
+  numberOfNodes = numberOfNodes === 1 || numberOfNodes === 3 ? numberOfNodes : 1;
+
+  await upgradeNetwork(upgradeTo, optBumptTo, numberOfNodes);
+
   process.exit(0);
 }
 

bouncer/package.json

@@ -11,6 +11,7 @@
     "@polkadot/keyring": "12.2.1",
     "@polkadot/util": "12.2.1",
     "@polkadot/util-crypto": "12.2.1",
+    "@types/yargs": "^17.0.29",
     "async-mutex": "^0.4.0",
     "axios": "^1.3.5",
     "bitcoin-core": "^4.1.0",
@@ -22,7 +23,8 @@
     "minimist": "^1.2.8",
     "tiny-secp256k1": "^2.2.1",
     "toml": "^3.0.0",
-    "web3": "^1.9.0"
+    "web3": "^1.9.0",
+    "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.2",

bouncer/shared/simple_runtime_upgrade.ts

@@ -1,29 +1,14 @@
 import { submitRuntimeUpgrade } from './submit_runtime_upgrade';
-import { jsonRpc } from './json_rpc';
-import { getChainflipApi, observeEvent } from './utils';
-import { bumpSpecVersion } from './utils/bump_spec_version';
+import { bumpSpecVersionAgainstNetwork, getCurrentSpecVersion } from './utils/bump_spec_version';
 import { compileBinaries } from './utils/compile_binaries';
 
-async function getCurrentSpecVersion(): Promise<number> {
-  return Number((await jsonRpc('state_getRuntimeVersion', [], 9944)).specVersion);
-}
-
 // Do a runtime upgrade using the code in the projectRoot directory.
 export async function simpleRuntimeUpgrade(projectRoot: string): Promise<void> {
-  const chainflip = await getChainflipApi();
-  const currentSpecVersion = await getCurrentSpecVersion();
-  console.log('Current spec_version: ' + currentSpecVersion);
-  const nextSpecVersion = currentSpecVersion + 1;
-  bumpSpecVersion(`${projectRoot}/state-chain/runtime/src/lib.rs`, nextSpecVersion);
+  const nextSpecVersion = await bumpSpecVersionAgainstNetwork(projectRoot);
 
   await compileBinaries('runtime', projectRoot);
 
-  console.log('Applying runtime upgrade.');
-  await submitRuntimeUpgrade(
-    `${projectRoot}/target/release/wbuild/state-chain-runtime/state_chain_runtime.compact.compressed.wasm`,
-  );
-
-  await observeEvent('system:CodeUpdated', chainflip);
+  await submitRuntimeUpgrade(projectRoot);
 
   const newSpecVersion = await getCurrentSpecVersion();
   console.log('New spec_version: ' + newSpecVersion);

bouncer/shared/submit_runtime_upgrade.ts

@@ -1,14 +1,14 @@
 import { compactAddLength } from '@polkadot/util';
 import { promises as fs } from 'fs';
 import { submitGovernanceExtrinsic } from './cf_governance';
-import { getChainflipApi } from '../shared/utils';
+import { getChainflipApi, observeEvent } from '../shared/utils';
 
 async function readRuntimeWasmFromFile(filePath: string): Promise<Uint8Array> {
   return compactAddLength(new Uint8Array(await fs.readFile(filePath)));
 }
 
 // By default we don't want to restrict that any of the nodes need to be upgraded.
-export async function submitRuntimeUpgrade(
+export async function submitRuntimeUpgradeWithRestrictions(
   wasmPath: string,
   semverRestriction?: Record<string, number>,
   percentNodesUpgraded = 0,
@@ -29,5 +29,17 @@ export async function submitRuntimeUpgrade(
     chainflip.tx.governance.chainflipRuntimeUpgrade(versionPercentRestriction, runtimeWasm),
   );
 
+  // TODO: Check if there were any errors in the submission, like `UpgradeConditionsNotMet` and `NotEnoughAuthoritiesCfesAtTargetVersion`.
+  // and exit with error.
+
+  await observeEvent('system:CodeUpdated', chainflip);
+
   console.log('Runtime upgrade completed.');
 }
+
+// Restrictions not provided.
+export async function submitRuntimeUpgrade(projectRoot: string) {
+  await submitRuntimeUpgradeWithRestrictions(
+    `${projectRoot}/target/release/wbuild/state-chain-runtime/state_chain_runtime.compact.compressed.wasm`,
+  );
+}

bouncer/shared/upgrade_network.ts

@@ -4,7 +4,10 @@ import * as toml from 'toml';
 import path from 'path';
 import { SemVerLevel, bumpReleaseVersion } from './bump_release_version';
 import { simpleRuntimeUpgrade } from './simple_runtime_upgrade';
-import { compareSemVer } from './utils';
+import { compareSemVer, sleep } from './utils';
+import { bumpSpecVersionAgainstNetwork } from './utils/bump_spec_version';
+import { compileBinaries } from './utils/compile_binaries';
+import { submitRuntimeUpgrade } from './submit_runtime_upgrade';
 
 async function readPackageTomlVersion(projectRoot: string): Promise<string> {
   const data = await fs.readFile(path.join(projectRoot, '/state-chain/runtime/Cargo.toml'), 'utf8');
@@ -23,63 +26,117 @@ function isCompatibleWith(semVer1: string, semVer2: string) {
 
 // Create a git workspace in the tmp/ directory and check out the specified commit.
 // Remember to delete it when you're done!
-function createGitWorkspaceAt(absoluteWorkspacePath: string, toGitRef: string) {
+function createGitWorkspaceAt(nextVersionWorkspacePath: string, toGitRef: string) {
   try {
     // Create a directory for the new workspace
-    execSync(`mkdir -p ${absoluteWorkspacePath}`);
+    execSync(`mkdir -p ${nextVersionWorkspacePath}`);
 
     // Create a new workspace using git worktree.
-    execSync(`git worktree add ${absoluteWorkspacePath}`);
+    execSync(`git worktree add ${nextVersionWorkspacePath}`);
 
     // Navigate to the new workspace and checkout the specific commit
-    execSync(`cd ${absoluteWorkspacePath} && git checkout ${toGitRef}`);
+    execSync(`cd ${nextVersionWorkspacePath} && git checkout ${toGitRef}`);
 
     console.log('Commit checked out successfully in new workspace.');
   } catch (error) {
     console.error(`Error: ${error}`);
   }
 }
 
+async function incompatibleUpgrade(
+  currentVersionWorkspacePath: string,
+  nextVersionWorkspacePath: string,
+  numberOfNodes: 1 | 3,
+) {
+  await bumpSpecVersionAgainstNetwork(nextVersionWorkspacePath);
+
+  await compileBinaries('all', nextVersionWorkspacePath);
+
+  let selectedNodes;
+  if (numberOfNodes === 1) {
+    selectedNodes = ['bashful'];
+  } else if (numberOfNodes === 3) {
+    selectedNodes = ['bashful', 'doc', 'dopey'];
+  } else {
+    throw new Error('Invalid number of nodes');
+  }
+
+  console.log('Starting all the engines');
+
+  const nodeCount = numberOfNodes + '-node';
+  execSync(
+    `LOG_SUFFIX="-upgrade" NODE_COUNT=${nodeCount} SELECTED_NODES="${selectedNodes.join(
+      ' ',
+    )}" LOCALNET_INIT_DIR=${currentVersionWorkspacePath}/localnet/init BINARY_ROOT_PATH=${nextVersionWorkspacePath}/target/release ${currentVersionWorkspacePath}/localnet/init/scripts/start-all-engines.sh`,
+  );
+
+  // let the engines do what they gotta do
+  sleep(6000);
+
+  console.log('Engines started');
+
+  await submitRuntimeUpgrade(nextVersionWorkspacePath);
+
+  console.log(
+    'Check that the old engine has now shut down, and that the new engine is now running.',
+  );
+}
+
 // Upgrades a bouncer network from the commit currently running on localnet to the provided git reference (commit, branch, tag).
 // If the version of the commit we're upgrading to is the same as the version of the commit we're upgrading from, we bump the version by the specified level.
-export async function upgradeNetwork(toGitRef: string, bumpByIfEqual: SemVerLevel = 'patch') {
-  const fromTomlVersion = await readPackageTomlVersion(path.dirname(process.cwd()));
+// Only the incompatible upgrade requires the number of nodes.
+export async function upgradeNetwork(
+  toGitRef: string,
+  bumpByIfEqual: SemVerLevel = 'patch',
+  numberOfNodes: 1 | 3 = 1,
+) {
+  const currentVersionWorkspacePath = path.dirname(process.cwd());
+
+  const fromTomlVersion = await readPackageTomlVersion(currentVersionWorkspacePath);
   console.log("Version we're upgrading from: " + fromTomlVersion);
 
   // tmp/ is ignored in the bouncer .gitignore file.
-  const absoluteWorkspacePath = path.join(process.cwd(), 'tmp/upgrade-network');
+  const nextVersionWorkspacePath = path.join(process.cwd(), 'tmp/upgrade-network');
 
-  console.log('Creating a new git workspace at: ' + absoluteWorkspacePath);
+  console.log('Creating a new git workspace at: ' + nextVersionWorkspacePath);
+  createGitWorkspaceAt(nextVersionWorkspacePath, toGitRef);
 
-  createGitWorkspaceAt(absoluteWorkspacePath, toGitRef);
-
-  const toTomlVersion = await readPackageTomlVersion(`${absoluteWorkspacePath}`);
-  console.log("Version we're upgrading to: " + toTomlVersion);
+  const toTomlVersion = await readPackageTomlVersion(`${nextVersionWorkspacePath}`);
+  console.log("Version of commit we're upgrading to: " + toTomlVersion);
 
   if (compareSemVer(fromTomlVersion, toTomlVersion) === 'greater') {
     throw new Error(
       "The version we're upgrading to is older than the version we're upgrading from. Ensure you selected the correct commits.",
     );
   }
 
+  // Now we need to bump the TOML versions if required, to ensure the `CurrentReleaseVersion` in the environment pallet is correct.
   if (fromTomlVersion === toTomlVersion) {
-    await bumpReleaseVersion(bumpByIfEqual, absoluteWorkspacePath);
+    console.log('Versions are equal, bumping by: ' + bumpByIfEqual);
+    await bumpReleaseVersion(bumpByIfEqual, nextVersionWorkspacePath);
+  } else {
+    console.log('Versions are not equal, no need to bump.');
   }
 
-  const newToTomlVersion = await readPackageTomlVersion(path.join(absoluteWorkspacePath));
+  const newToTomlVersion = await readPackageTomlVersion(path.join(nextVersionWorkspacePath));
+  console.log("Version we're upgrading to: " + newToTomlVersion);
+
   const isCompatible = isCompatibleWith(fromTomlVersion, newToTomlVersion);
+  console.log('Is compatible: ' + isCompatible);
 
+  // For some reason shit isn't working here. Keeps thinking it's compatible or something.
   if (isCompatible) {
     // The CFE could be upgraded too. But an incompatible CFE upgrade would mean it's... incompatible, so covered in the other path.
     console.log('The versions are compatible.');
 
-    // Runtime upgrade using the *new* version.
-    await simpleRuntimeUpgrade(absoluteWorkspacePath);
+    await simpleRuntimeUpgrade(nextVersionWorkspacePath);
     console.log('Upgrade complete.');
   } else if (!isCompatible) {
-    // Incompatible upgrades requires running two versions of the CFEs side by side.
-    console.log('Incompatible CFE upgrades are not yet supported :(');
+    console.log('The versions are incompatible.');
+    await incompatibleUpgrade(currentVersionWorkspacePath, nextVersionWorkspacePath, numberOfNodes);
   }
 
-  execSync(`cd ${absoluteWorkspacePath} && git worktree remove . --force`);
+  console.log('Cleaning up...');
+  execSync(`cd ${nextVersionWorkspacePath} && git worktree remove . --force`);
+  console.log('Done.');
 }

bouncer/shared/utils/bump_spec_version.ts

@@ -1,7 +1,11 @@
 import fs from 'fs';
+import { jsonRpc } from '../json_rpc';
+
+export async function getCurrentSpecVersion(): Promise<number> {
+  return Number((await jsonRpc('state_getRuntimeVersion', [], 9944)).specVersion);
+}
 
 export function bumpSpecVersion(filePath: string, nextSpecVersion?: number) {
-  console.log('Bumping the spec version');
   try {
     const fileContent = fs.readFileSync(filePath, 'utf-8');
     const lines = fileContent.split('\n');
@@ -43,3 +47,13 @@ export function bumpSpecVersion(filePath: string, nextSpecVersion?: number) {
     console.error(`An error occurred: ${error.message}`);
   }
 }
+
+// Bump the spec version in the runtime file, using the spec version of the network.
+export async function bumpSpecVersionAgainstNetwork(projectRoot: string): Promise<number> {
+  const currentSpecVersion = await getCurrentSpecVersion();
+  console.log('Current spec_version: ' + currentSpecVersion);
+  const nextSpecVersion = currentSpecVersion + 1;
+  console.log('Bumping the spec version to: ' + nextSpecVersion);
+  bumpSpecVersion(`${projectRoot}/state-chain/runtime/src/lib.rs`, nextSpecVersion);
+  return nextSpecVersion;
+}

localnet/init/scripts/start-all-engines.sh

@@ -1,10 +1,14 @@
 # Starts all the engines necessary for the network, or for an upgrade.
 
+# These need to match what's in the manage.py script.
+SC_RPC_PORT=9944
+HEALTH_PORT=5555
+
 ENGINE_P2P_PORT=3100
 LOG_PORT=30687
 for NODE in $SELECTED_NODES; do
     cp -R $LOCALNET_INIT_DIR/keyshare/$NODE_COUNT/$NODE.db /tmp/chainflip/$NODE
-    BINARY_ROOT_PATH=$BINARY_ROOT_PATH NODE_NAME=$NODE P2P_PORT=$ENGINE_P2P_PORT SC_RPC_PORT=$SC_RPC_PORT LOG_PORT=$LOG_PORT HEALTH_PORT=$HEALTH_PORT LOG_SUFFIX=$LOG_SUFFIX ./$LOCALNET_INIT_DIR/scripts/start-engine.sh
+    BINARY_ROOT_PATH=$BINARY_ROOT_PATH NODE_NAME=$NODE P2P_PORT=$ENGINE_P2P_PORT SC_RPC_PORT=$SC_RPC_PORT LOG_PORT=$LOG_PORT HEALTH_PORT=$HEALTH_PORT LOG_SUFFIX=$LOG_SUFFIX $LOCALNET_INIT_DIR/scripts/start-engine.sh
     echo "🚗 Starting chainflip-engine of $NODE ..."
     ((SC_RPC_PORT++))
     ((ENGINE_P2P_PORT++))