fix: retry with search history enabled after some attempts (#5370)

* fix: retry with search history enabled after some attempts

wip

add const

capture sigs

* chore: increase number of retries

* doc: fix comment

engine/src/sol/retry_rpc.rs

@@ -9,6 +9,7 @@ use cf_chains::{
 };
 use cf_utilities::{make_periodic_tick, task_scope::Scope};
 use core::time::Duration;
+use std::pin::Pin;
 
 use anyhow::{anyhow, Result};
 use base64::{prelude::BASE64_STANDARD, Engine};
@@ -32,6 +33,8 @@ const MAX_BROADCAST_RETRIES: Attempt = 5;
 const GET_STATUS_BROADCAST_DELAY: u64 = 500u64;
 const GET_STATUS_BROADCAST_RETRIES: u64 = 10;
 
+const GET_SIGNATURE_STATUS_RETRY_LIMIT: Attempt = 10;
+
 impl SolRetryRpcClient {
 	pub async fn new(
 		scope: &Scope<'_, anyhow::Error>,
@@ -149,27 +152,61 @@ impl SolRetryRpcApi for SolRetryRpcClient {
 			)
 			.await
 	}
+
+	/// Gets signature status with `search_transaction_history`. If `search_transaction_history` is
+	/// set to false, it will retry with `search_transaction_history` set to true if it fails
+	/// `GET_SIGNATURE_STATUS_RETRY_LIMIT` times.
 	async fn get_signature_statuses(
 		&self,
 		signatures: &[SolSignature],
 		search_transaction_history: bool,
 	) -> Response<Vec<Option<TransactionStatus>>> {
 		let signatures = signatures.to_owned();
-		self.rpc_retry_client
-			.request(
+
+		let sig_status_generator = move |search_transaction_history| {
+			let signatures = signatures.clone();
+			(
 				RequestLog::new(
 					"getSignatureStatuses".to_string(),
 					Some(format!("{:?}, {:?}", signatures, search_transaction_history)),
 				),
-				Box::pin(move |client| {
+				Box::pin(move |client: SolRpcClient| {
 					let signatures = signatures.clone();
-					#[allow(clippy::redundant_async_block)]
 					Box::pin(async move {
 						client.get_signature_statuses(&signatures, search_transaction_history).await
-					})
+					}) as Pin<Box<dyn futures::Future<Output = anyhow::Result<_>> + Send>>
 				}),
 			)
-			.await
+		};
+
+		let get_signature_status_no_retry_limit = |search_transaction_history| {
+			let (request_log, sig_status_call) = sig_status_generator(search_transaction_history);
+			self.rpc_retry_client.request(request_log, sig_status_call)
+		};
+
+		if search_transaction_history {
+			get_signature_status_no_retry_limit(search_transaction_history).await
+		} else {
+			let (request_log, sig_status_call) = sig_status_generator(search_transaction_history);
+			match self
+				.rpc_retry_client
+				.request_with_limit(
+					request_log,
+					sig_status_call,
+					// We expect it to work without search history, but if it doesn't we retry with
+					// search history enabled we have seen that the fallback to enabling search
+					// history. We've seen this works in the wild.
+					GET_SIGNATURE_STATUS_RETRY_LIMIT,
+				)
+				.await
+			{
+				Ok(ok) => ok,
+				Err(e) => {
+					tracing::warn!("Failed to get signature statuses without search history: {e:?} Attempting with search history enabled");
+					get_signature_status_no_retry_limit(true).await
+				},
+			}
+		}
 	}
 	async fn get_transaction(
 		&self,