feat: backup RPC (#3951)

* refactor: settings for a single node grouped together

* chore: create future in btc retrier

* feat: include backup rpc setting

* feat: backup rpc

* test: fix broken settings test

test: unset env

* chore: remove template settings

* chore: migrate settings

* fix: if no settings set it doesn't mean migration fails

* chore: address minor review comments

* feat: include usage of backup rpc when setting provided

* chore: rename client var

* chore: improve backup settings

* fix: spawn weak for client tasks

* feat: clean up main

* fix: deser/ser correctly, use guard

* chore: use macro for settings env

* chore: remove unnecessary test env

* chore: remove migration

api/bin/chainflip-cli/src/settings.rs

@@ -252,7 +252,7 @@ mod tests {
 		.unwrap();
 
 		assert_eq!(settings.state_chain.ws_endpoint, "ws://localhost:9944");
-		assert_eq!(settings.eth.ws_node_endpoint, "ws://localhost:8545");
+		assert_eq!(settings.eth.nodes.primary.ws_node_endpoint, "ws://localhost:8545");
 	}
 
 	#[test]
@@ -273,6 +273,8 @@ mod tests {
 				eth_ws_node_endpoint: Some("ws://endpoint2:1234".to_owned()),
 				eth_http_node_endpoint: Some("http://endpoint3:1234".to_owned()),
 				eth_private_key_file: Some(PathBuf::from_str("eth_key_file").unwrap()),
+				eth_backup_ws_node_endpoint: Some("ws://endpoint4:1234".to_owned()),
+				eth_backup_http_node_endpoint: Some("http://endpoint5:1234".to_owned()),
 			},
 
 			cmd: CliCommand::Rotate {}, // Not used in this test
@@ -290,8 +292,25 @@ mod tests {
 			opts.state_chain_opts.state_chain_signing_key_file.unwrap(),
 			settings.state_chain.signing_key_file
 		);
-		assert_eq!(opts.eth_opts.eth_ws_node_endpoint.unwrap(), settings.eth.ws_node_endpoint);
-		assert_eq!(opts.eth_opts.eth_http_node_endpoint.unwrap(), settings.eth.http_node_endpoint);
+		assert_eq!(
+			opts.eth_opts.eth_ws_node_endpoint.unwrap(),
+			settings.eth.nodes.primary.ws_node_endpoint
+		);
+		assert_eq!(
+			opts.eth_opts.eth_http_node_endpoint.unwrap(),
+			settings.eth.nodes.primary.http_node_endpoint
+		);
+
+		let eth_backup_node = settings.eth.nodes.backup.unwrap();
+		assert_eq!(
+			opts.eth_opts.eth_backup_ws_node_endpoint.unwrap(),
+			eth_backup_node.ws_node_endpoint
+		);
+		assert_eq!(
+			opts.eth_opts.eth_backup_http_node_endpoint.unwrap(),
+			eth_backup_node.http_node_endpoint
+		);
+
 		assert_eq!(opts.eth_opts.eth_private_key_file.unwrap(), settings.eth.private_key_file);
 	}
 }

engine/Cargo.toml

@@ -80,6 +80,7 @@ ed25519-dalek = "1.0"
 pin-project = "1.0.12"
 rand = "0.8.4"
 reqwest = { version = "0.11.4", features = ["rustls-tls"] }
+toml = "0.7.4"
 tracing = "0.1"
 x25519-dalek = { version = "1.1", features = ["serde"] }
 zmq = { git = "https://github.com/chainflip-io/rust-zmq.git", tag = "chainflip-v0.9.2+1", features = [

engine/config/testing/config/Settings.toml

@@ -7,25 +7,24 @@ allow_local_ip = true
 # 32 byte hex secret key - associated with the node's public id (public key)
 signing_key_file = "./tests/test_keystore/alice_key"
 
-[eth]
-# Ethereum private key file path. Default is the docker secrets path. This file should contain a hex-encoded private key.
+[eth.node]
 http_node_endpoint = "http://localhost:8545"
 ws_node_endpoint = "ws://localhost:8545"
 
-[dot]
+[dot.node]
 # NB: You will need to manually add :443 to the url provided by the provider, as jsonrpsee wants one.
 ws_node_endpoint = "wss://my_fake_polkadot_rpc:443/<secret_key>"
 http_node_endpoint = "http://my_fake_polkadot_rpc:443/<secret_key>"
 
+[btc.node]
+http_node_endpoint = "http://localhost:18443"
+rpc_user = "username"
+rpc_password = "password"
+
 [health_check]
 hostname = "127.0.0.1"
 port = 5555
 
 [prometheus]
 hostname = "127.0.0.1"
 port = 5566
-
-[btc]
-http_node_endpoint = "http://localhost:18443"
-rpc_user = "username"
-rpc_password = "password"

engine/config/testing2/config/Settings.toml

@@ -0,0 +1,32 @@
+# Is just some random different settings from ../../testing/config/Settings.toml.
+# Isn't intended to be correct/real values!
+# It's also not ideal the paths have to specified with a /config prefix...
+[node_p2p]
+ip_address = "1.2.3.4"
+allow_local_ip = true
+
+[state_chain]
+# 32 byte hex secret key - associated with the node's public id (public key)
+signing_key_file = "./tests/test_keystore/alicia_keys"
+
+[eth.node]
+http_node_endpoint = "http://localhost:8545"
+ws_node_endpoint = "ws://localhost:8545"
+
+[dot.node]
+# NB: You will need to manually add :443 to the url provided by the provider, as jsonrpsee wants one.
+ws_node_endpoint = "wss://my_polkadot_rpc:443/<secret_key>"
+http_node_endpoint = "http://my_polkadot_rpc:443/<secret_key>"
+
+[btc.node]
+http_node_endpoint = "http://localhost:18443"
+rpc_user = "doge"
+rpc_password = "coin"
+
+[health_check]
+hostname = "127.0.0.2"
+port = 5555
+
+[prometheus]
+hostname = "127.0.2.1"
+port = 5566

engine/src/btc/retry_rpc.rs

@@ -1,14 +1,16 @@
 use bitcoin::{Block, BlockHash, Txid};
-use futures_core::Future;
 use utilities::task_scope::Scope;
 
 use crate::{
 	retrier::{Attempt, RequestLog, RetrierClient},
+	settings::{HttpBasicAuthEndpoint, NodeContainer},
 	witness::common::chain_source::{ChainClient, Header},
 };
 use cf_chains::Bitcoin;
 use core::time::Duration;
 
+use anyhow::Result;
+
 use super::rpc::{BlockHeader, BtcRpcApi, BtcRpcClient};
 
 #[derive(Clone)]
@@ -22,19 +24,23 @@ const MAX_CONCURRENT_SUBMISSIONS: u32 = 100;
 const MAX_BROADCAST_RETRIES: Attempt = 5;
 
 impl BtcRetryRpcClient {
-	pub fn new<BtcRpcClientFut: Future<Output = BtcRpcClient> + Send + 'static>(
+	pub fn new(
 		scope: &Scope<'_, anyhow::Error>,
-		btc_client: BtcRpcClientFut,
-	) -> Self {
-		Self {
+		nodes: NodeContainer<HttpBasicAuthEndpoint>,
+	) -> Result<Self> {
+		let primary = BtcRpcClient::new(nodes.primary)?;
+		let backup = nodes.backup.map(BtcRpcClient::new).transpose()?;
+
+		Ok(Self {
 			retry_client: RetrierClient::new(
 				scope,
 				"btc_rpc",
-				btc_client,
+				futures::future::ready(primary),
+				backup.map(futures::future::ready),
 				BITCOIN_RPC_TIMEOUT,
 				MAX_CONCURRENT_SUBMISSIONS,
 			),
-		}
+		})
 	}
 }
 

engine/src/btc/rpc.rs

@@ -8,7 +8,7 @@ use serde_json::json;
 
 use bitcoin::{block::Version, Amount, Block, BlockHash, Txid};
 
-use crate::settings;
+use crate::settings::HttpBasicAuthEndpoint;
 
 use anyhow::{Context, Result};
 
@@ -65,12 +65,12 @@ pub struct BtcRpcClient {
 }
 
 impl BtcRpcClient {
-	pub fn new(btc_settings: settings::Btc) -> Result<Self> {
+	pub fn new(basic_auth_endpoint: HttpBasicAuthEndpoint) -> Result<Self> {
 		Ok(Self {
 			client: Client::builder().build()?,
-			url: btc_settings.http_node_endpoint,
-			user: btc_settings.rpc_user,
-			password: btc_settings.rpc_password,
+			url: basic_auth_endpoint.http_node_endpoint,
+			user: basic_auth_endpoint.rpc_user,
+			password: basic_auth_endpoint.rpc_password,
 		})
 	}
 
@@ -222,7 +222,7 @@ mod tests {
 	#[tokio::test]
 	#[ignore = "requires local node, useful for manual testing"]
 	async fn test_btc_async() {
-		let client = BtcRpcClient::new(settings::Btc {
+		let client = BtcRpcClient::new(HttpBasicAuthEndpoint {
 			http_node_endpoint: "http://localhost:8332".to_string(),
 			rpc_user: "flip".to_string(),
 			rpc_password: "flip".to_string(),

engine/src/common.rs

@@ -142,3 +142,10 @@ impl<T: Clone + Send + 'static> Signal<T> {
 		}
 	}
 }
+
+pub fn option_inner<T, S>(option_tup: Option<(T, S)>) -> (Option<T>, Option<S>) {
+	match option_tup {
+		Some((t, s)) => (Some(t), Some(s)),
+		None => (None, None),
+	}
+}

engine/src/constants.rs

@@ -15,18 +15,25 @@ pub const DOT_AVERAGE_BLOCK_TIME: Duration = Duration::from_secs(6);
 
 // ======= Settings environment variables =======
 
-/// A HTTP node endpoint for Ethereum
-pub const ETH_HTTP_NODE_ENDPOINT: &str = "ETH__HTTP_NODE_ENDPOINT";
+pub const ETH_HTTP_NODE_ENDPOINT: &str = "ETH__NODE__HTTP_NODE_ENDPOINT";
+pub const ETH_WS_NODE_ENDPOINT: &str = "ETH__NODE__WS_NODE_ENDPOINT";
 
-/// A WebSocket node endpoint for Ethereum
-pub const ETH_WS_NODE_ENDPOINT: &str = "ETH__WS_NODE_ENDPOINT";
+pub const ETH_BACKUP_HTTP_NODE_ENDPOINT: &str = "ETH__BACKUP_NODE__HTTP_NODE_ENDPOINT";
+pub const ETH_BACKUP_WS_NODE_ENDPOINT: &str = "ETH__BACKUP_NODE__WS_NODE_ENDPOINT";
 
-pub const BTC_HTTP_NODE_ENDPOINT: &str = "BTC__HTTP_NODE_ENDPOINT";
-pub const BTC_RPC_USER: &str = "BTC__RPC_USER";
-pub const BTC_RPC_PASSWORD: &str = "BTC__RPC_PASSWORD";
+pub const BTC_HTTP_NODE_ENDPOINT: &str = "BTC__NODE__HTTP_NODE_ENDPOINT";
+pub const BTC_RPC_USER: &str = "BTC__NODE__RPC_USER";
+pub const BTC_RPC_PASSWORD: &str = "BTC__NODE__RPC_PASSWORD";
 
-pub const DOT_WS_NODE_ENDPOINT: &str = "DOT__WS_NODE_ENDPOINT";
-pub const DOT_HTTP_NODE_ENDPOINT: &str = "DOT__HTTP_NODE_ENDPOINT";
+pub const BTC_BACKUP_HTTP_NODE_ENDPOINT: &str = "BTC__BACKUP_NODE__HTTP_NODE_ENDPOINT";
+pub const BTC_BACKUP_RPC_USER: &str = "BTC__BACKUP_NODE__RPC_USER";
+pub const BTC_BACKUP_RPC_PASSWORD: &str = "BTC__BACKUP_NODE__RPC_PASSWORD";
+
+pub const DOT_WS_NODE_ENDPOINT: &str = "DOT__NODE__WS_NODE_ENDPOINT";
+pub const DOT_HTTP_NODE_ENDPOINT: &str = "DOT__NODE__HTTP_NODE_ENDPOINT";
+
+pub const DOT_BACKUP_WS_NODE_ENDPOINT: &str = "DOT__BACKUP_NODE__WS_NODE_ENDPOINT";
+pub const DOT_BACKUP_HTTP_NODE_ENDPOINT: &str = "DOT__BACKUP_NODE__HTTP_NODE_ENDPOINT";
 
 /// IP Address and port on which we listen for incoming p2p connections
 pub const NODE_P2P_IP_ADDRESS: &str = "NODE_P2P__IP_ADDRESS";

engine/src/dot/retry_rpc.rs

@@ -1,5 +1,7 @@
 use crate::{
+	common::option_inner,
 	retrier::Attempt,
+	settings::{NodeContainer, WsHttpEndpoints},
 	witness::common::chain_source::{ChainClient, Header},
 };
 use cf_chains::{
@@ -8,7 +10,7 @@ use cf_chains::{
 };
 use cf_primitives::PolkadotBlockNumber;
 use core::time::Duration;
-use futures_core::{Future, Stream};
+use futures_core::Stream;
 use sp_core::H256;
 use std::pin::Pin;
 use subxt::{
@@ -18,6 +20,8 @@ use utilities::task_scope::Scope;
 
 use crate::retrier::{RequestLog, RetrierClient};
 
+use anyhow::Result;
+
 use super::{
 	http_rpc::DotHttpRpcClient,
 	rpc::{DotSubClient, PolkadotHeader},
@@ -37,27 +41,40 @@ const MAX_CONCURRENT_SUBMISSIONS: u32 = 20;
 const MAX_BROADCAST_RETRIES: Attempt = 5;
 
 impl DotRetryRpcClient {
-	pub fn new<DotHttpRpcClientFut: Future<Output = DotHttpRpcClient> + Send + 'static>(
+	pub fn new(
 		scope: &Scope<'_, anyhow::Error>,
-		dot_rpc_client: DotHttpRpcClientFut,
-		dot_sub_client: DotSubClient,
-	) -> Self {
-		Self {
+		nodes: NodeContainer<WsHttpEndpoints>,
+	) -> Result<Self> {
+		let f_create_clients = |endpoints: WsHttpEndpoints| {
+			Result::<_, anyhow::Error>::Ok((
+				DotHttpRpcClient::new(endpoints.http_node_endpoint)?,
+				DotSubClient::new(&endpoints.ws_node_endpoint),
+			))
+		};
+
+		let (rpc_client, sub_client) = f_create_clients(nodes.primary)?;
+
+		let (backup_rpc_client, backup_sub_client) =
+			option_inner(nodes.backup.map(f_create_clients).transpose()?);
+
+		Ok(Self {
 			rpc_retry_client: RetrierClient::new(
 				scope,
 				"dot_rpc",
-				dot_rpc_client,
+				rpc_client,
+				backup_rpc_client,
 				POLKADOT_RPC_TIMEOUT,
 				MAX_CONCURRENT_SUBMISSIONS,
 			),
 			sub_retry_client: RetrierClient::new(
 				scope,
 				"dot_subscribe",
-				async move { dot_sub_client },
+				futures::future::ready(sub_client),
+				backup_sub_client.map(futures::future::ready),
 				POLKADOT_RPC_TIMEOUT,
 				MAX_CONCURRENT_SUBMISSIONS,
 			),
-		}
+		})
 	}
 }
 
@@ -291,9 +308,15 @@ mod tests {
 			async move {
 				let dot_retry_rpc_client = DotRetryRpcClient::new(
 					scope,
-					DotHttpRpcClient::new("http://127.0.0.1:9945".to_string()).unwrap(),
-					DotSubClient::new("ws://127.0.0.1:9945"),
-				);
+					NodeContainer {
+						primary: WsHttpEndpoints {
+							http_node_endpoint: "http://127.0.0.1:9945".to_string(),
+							ws_node_endpoint: "ws://127.0.0.1:9945".to_string(),
+						},
+						backup: None,
+					},
+				)
+				.unwrap();
 
 				let hash = dot_retry_rpc_client.block_hash(1).await.unwrap();
 				println!("Block hash: {}", hash);