feat: retry program deployments in SVM tooling with higher fees

rust/main/utils/run-locally/src/solana.rs

@@ -159,7 +159,7 @@ pub fn build_solana_programs(solana_cli_tools_path: PathBuf) -> PathBuf {
         .working_dir(&out_path)
         .run()
         .join();
-    log!("Remove temporary solana files");
+    log!("Removing temporary solana files");
     fs::remove_file(concat_path(&out_path, "spl.tar.gz"))
         .expect("Failed to remove solana program archive");
 
@@ -232,8 +232,7 @@ pub fn start_solana_test_validator(
         .arg("environment", SOLANA_ENV_NAME)
         .arg("environments-dir", SOLANA_ENVS_DIR)
         .arg("built-so-dir", SBF_OUT_PATH)
-        .arg("overhead-config-file", SOLANA_OVERHEAD_CONFIG_FILE)
-        .flag("use-existing-keys");
+        .arg("overhead-config-file", SOLANA_OVERHEAD_CONFIG_FILE);
 
     sealevel_client_deploy_core
         .clone()

rust/sealevel/client/src/cmd_utils.rs

@@ -1,14 +1,17 @@
 use std::{
     collections::HashMap,
     fs::File,
-    io::Write,
+    io::{self, Write},
     path::{Path, PathBuf},
     process::{Command, Stdio},
     thread::sleep,
     time::Duration,
 };
 
-use solana_client::{client_error::ClientError, rpc_client::RpcClient};
+use solana_client::{
+    client_error::{ClientError, ClientErrorKind},
+    rpc_client::RpcClient,
+};
 use solana_sdk::{
     commitment_config::CommitmentConfig,
     pubkey::Pubkey,
@@ -44,37 +47,91 @@ pub(crate) fn account_exists(client: &RpcClient, account: &Pubkey) -> Result<boo
     Ok(exists)
 }
 
-pub(crate) fn deploy_program_idempotent(
+pub(crate) fn deploy_program(
     payer_keypair_path: &str,
-    program_keypair: &Keypair,
-    program_keypair_path: &str,
+    program_key_dir: &Path,
+    program_name: &str,
     program_path: &str,
     url: &str,
     local_domain: u32,
-) -> Result<(), ClientError> {
+) -> Result<Pubkey, ClientError> {
+    let (program_keypair, program_keypair_path) = create_or_get_keypair(
+        program_key_dir,
+        format!("{}-keypair.json", program_name).as_str(),
+    );
+    let program_id = program_keypair.pubkey();
+
     let client = RpcClient::new(url.to_string());
-    if !account_exists(&client, &program_keypair.pubkey())? {
-        deploy_program(
+    if account_exists(&client, &program_keypair.pubkey())? {
+        println!("Program {} already deployed", program_keypair.pubkey());
+        return Ok(program_id);
+    }
+
+    let (buffer_keypair, buffer_keypair_path) = create_or_get_keypair(
+        program_key_dir,
+        format!("{}-buffer.json", program_name).as_str(),
+    );
+
+    let mut compute_unit_price = get_compute_unit_price_micro_lamports_for_id(local_domain);
+
+    for attempt in 0..10 {
+        println!("Attempting program deploy Program ID: {}, buffer pubkey: {}, compute unit price: {}, attempt number {}", program_id, buffer_keypair.pubkey(), compute_unit_price, attempt);
+
+        if attempt > 0 {
+            println!(
+                "As this is not the first deploy attempt, the buffer {} is re-used",
+                buffer_keypair.pubkey()
+            );
+        }
+
+        if attempt_program_deploy(
             payer_keypair_path,
-            program_keypair_path,
+            program_name,
             program_path,
+            &program_keypair_path,
+            &buffer_keypair_path,
             url,
-            local_domain,
+            compute_unit_price,
+        )
+        .is_ok()
+        {
+            // Success!
+            return Ok(program_id);
+        }
+
+        // Failed to deploy program, try again with a higher compute unit price
+
+        println!(
+            "Failed to deploy program with compute unit price {}",
+            compute_unit_price
         );
-    } else {
-        println!("Program {} already deployed", program_keypair.pubkey());
+
+        // Bump by 10% each time if non-zero, otherwise start at 1000 micro lamports
+        compute_unit_price = if compute_unit_price > 0 {
+            compute_unit_price * 11 / 10
+        } else {
+            1000
+        };
+
+        println!(
+            "Sleeping 1s, then retrying with new compute unit price {}",
+            compute_unit_price
+        );
+        sleep(Duration::from_secs(1));
     }
 
-    Ok(())
+    Err(ClientErrorKind::Custom(format!("Failed to deploy program {}", program_name)).into())
 }
 
-pub(crate) fn deploy_program(
+fn attempt_program_deploy(
     payer_keypair_path: &str,
-    program_keypair_path: &str,
+    program_name: &str,
     program_path: &str,
+    program_keypair_path: &Path,
+    buffer_keypair_path: &Path,
     url: &str,
-    local_domain: u32,
-) {
+    compute_unit_price: u64,
+) -> Result<(), ClientError> {
     let mut command = vec![
         "solana",
         "--url",
@@ -87,19 +144,28 @@ pub(crate) fn deploy_program(
         "--upgrade-authority",
         payer_keypair_path,
         "--program-id",
-        program_keypair_path,
+        program_keypair_path.to_str().unwrap(),
+        "--buffer",
+        buffer_keypair_path.to_str().unwrap(),
     ];
 
-    let compute_unit_price = get_compute_unit_price_micro_lamports_for_id(local_domain).to_string();
-    if local_domain.eq(&SOLANA_DOMAIN) {
-        command.extend(vec!["--with-compute-unit-price", &compute_unit_price]);
-    }
+    let compute_unit_price_str = compute_unit_price.to_string();
+    command.extend(vec!["--with-compute-unit-price", &compute_unit_price_str]);
 
-    build_cmd(command.as_slice(), None, None);
+    // Success!
+    if let Ok(true) = run_cmd(command.as_slice(), None, None) {
+        // TODO: use commitment level instead of just sleeping here?
+        println!("Sleeping for 2 seconds to fully allow program to be deployed");
+        sleep(Duration::from_secs(2));
+
+        return Ok(());
+    }
 
-    // TODO: use commitment level instead of just sleeping here?
-    println!("Sleeping for 2 seconds to allow program to be deployed");
-    sleep(Duration::from_secs(2));
+    Err(ClientErrorKind::Custom(format!(
+        "Attempted program deploy failed for {}",
+        program_name
+    ))
+    .into())
 }
 
 pub(crate) fn create_new_directory(parent_dir: &Path, name: &str) -> PathBuf {
@@ -109,20 +175,14 @@ pub(crate) fn create_new_directory(parent_dir: &Path, name: &str) -> PathBuf {
     path
 }
 
-pub(crate) fn create_and_write_keypair(
-    key_dir: &Path,
-    key_name: &str,
-    use_existing_key: bool,
-) -> (Keypair, PathBuf) {
+pub(crate) fn create_or_get_keypair(key_dir: &Path, key_name: &str) -> (Keypair, PathBuf) {
     let path = key_dir.join(key_name);
 
-    if use_existing_key {
-        if let Ok(file) = File::open(path.clone()) {
-            println!("Using existing key at path {}", path.display());
-            let keypair_bytes: Vec<u8> = serde_json::from_reader(file).unwrap();
-            let keypair = Keypair::from_bytes(&keypair_bytes[..]).unwrap();
-            return (keypair, path);
-        }
+    if let Ok(file) = File::open(path.clone()) {
+        println!("Using existing key at path {}", path.display());
+        let keypair_bytes: Vec<u8> = serde_json::from_reader(file).unwrap();
+        let keypair = Keypair::from_bytes(&keypair_bytes[..]).unwrap();
+        return (keypair, path);
     }
 
     let keypair = Keypair::new();
@@ -136,8 +196,14 @@ pub(crate) fn create_and_write_keypair(
     (keypair, path)
 }
 
-fn build_cmd(cmd: &[&str], wd: Option<&str>, env: Option<&HashMap<&str, &str>>) {
+fn run_cmd(cmd: &[&str], wd: Option<&str>, env: Option<&HashMap<&str, &str>>) -> io::Result<bool> {
     assert!(!cmd.is_empty(), "Must specify a command!");
+    if cmd.is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            "Must specify a command!",
+        ));
+    }
     let mut c = Command::new(cmd[0]);
     c.args(&cmd[1..]);
     c.stdout(Stdio::inherit());
@@ -149,10 +215,6 @@ fn build_cmd(cmd: &[&str], wd: Option<&str>, env: Option<&HashMap<&str, &str>>)
         c.envs(env);
     }
     println!("Running command: {:?}", c);
-    let status = c.status().expect("Failed to run command");
-    assert!(
-        status.success(),
-        "Command returned non-zero exit code: {}",
-        cmd.join(" ")
-    );
+    let status = c.status()?;
+    Ok(status.success())
 }

rust/sealevel/client/src/core.rs

@@ -3,7 +3,6 @@ use hyperlane_sealevel_mailbox::protocol_fee::ProtocolFee;
 use serde::{Deserialize, Serialize};
 
 use solana_program::pubkey::Pubkey;
-use solana_sdk::signature::Signer;
 use solana_sdk::{compute_budget, compute_budget::ComputeBudgetInstruction};
 
 use std::collections::HashMap;
@@ -13,7 +12,7 @@ use crate::cmd_utils::get_compute_unit_price_micro_lamports_for_chain_name;
 use crate::ONE_SOL_IN_LAMPORTS;
 use crate::{
     artifacts::{read_json, write_json},
-    cmd_utils::{create_and_write_keypair, create_new_directory, deploy_program},
+    cmd_utils::{create_new_directory, deploy_program},
     multisig_ism::deploy_multisig_ism_message_id,
     Context, CoreCmd, CoreDeploy, CoreSubCmd,
 };
@@ -89,7 +88,6 @@ pub(crate) fn process_core_cmd(mut ctx: Context, cmd: CoreCmd) {
             let ism_program_id = deploy_multisig_ism_message_id(
                 &mut ctx,
                 &core.built_so_dir,
-                core.use_existing_keys,
                 &key_dir,
                 core.local_domain,
             );
@@ -123,23 +121,18 @@ fn deploy_mailbox(
     default_ism: Pubkey,
     local_domain: u32,
 ) -> Pubkey {
-    let (keypair, keypair_path) = create_and_write_keypair(
-        key_dir,
-        "hyperlane_sealevel_mailbox-keypair.json",
-        core.use_existing_keys,
-    );
-    let program_id = keypair.pubkey();
-
-    deploy_program(
+    let program_id = deploy_program(
         ctx.payer_keypair_path(),
-        keypair_path.to_str().unwrap(),
+        key_dir,
+        "hyperlane_sealevel_mailbox",
         core.built_so_dir
             .join("hyperlane_sealevel_mailbox.so")
             .to_str()
             .unwrap(),
         &ctx.client.url(),
         local_domain,
-    );
+    )
+    .unwrap();
 
     println!("Deployed Mailbox at program ID {}", program_id);
 
@@ -182,23 +175,18 @@ fn deploy_validator_announce(
     key_dir: &Path,
     mailbox_program_id: Pubkey,
 ) -> Pubkey {
-    let (keypair, keypair_path) = create_and_write_keypair(
-        key_dir,
-        "hyperlane_sealevel_validator_announce-keypair.json",
-        core.use_existing_keys,
-    );
-    let program_id = keypair.pubkey();
-
-    deploy_program(
+    let program_id = deploy_program(
         ctx.payer_keypair_path(),
-        keypair_path.to_str().unwrap(),
+        key_dir,
+        "hyperlane_sealevel_validator_announce",
         core.built_so_dir
             .join("hyperlane_sealevel_validator_announce.so")
             .to_str()
             .unwrap(),
         &ctx.client.url(),
         core.local_domain,
-    );
+    )
+    .unwrap();
 
     println!("Deployed ValidatorAnnounce at program ID {}", program_id);
 
@@ -225,13 +213,6 @@ fn deploy_igp(ctx: &mut Context, core: &CoreDeploy, key_dir: &Path) -> (Pubkey,
         instruction::{GasOracleConfig, GasOverheadConfig},
     };
 
-    let (keypair, keypair_path) = create_and_write_keypair(
-        key_dir,
-        "hyperlane_sealevel_igp-keypair.json",
-        core.use_existing_keys,
-    );
-    let program_id = keypair.pubkey();
-
     let mut gas_oracle_configs = core
         .gas_oracle_config_file
         .as_deref()
@@ -275,16 +256,18 @@ fn deploy_igp(ctx: &mut Context, core: &CoreDeploy, key_dir: &Path) -> (Pubkey,
         .into_values()
         .collect::<Vec<_>>();
 
-    deploy_program(
+    let program_id = deploy_program(
         ctx.payer_keypair_path(),
-        keypair_path.to_str().unwrap(),
+        key_dir,
+        "hyperlane_sealevel_igp",
         core.built_so_dir
             .join("hyperlane_sealevel_igp.so")
             .to_str()
             .unwrap(),
         &ctx.client.url(),
         core.local_domain,
-    );
+    )
+    .unwrap();
 
     println!("Deployed IGP at program ID {}", program_id);