diff --git a/.github/workflows/agent-release-artifacts.yml b/.github/workflows/agent-release-artifacts.yml index 76b782f5e1..d2f61e6322 100644 --- a/.github/workflows/agent-release-artifacts.yml +++ b/.github/workflows/agent-release-artifacts.yml @@ -16,7 +16,7 @@ env: jobs: prepare: - runs-on: ubuntu-latest + runs-on: larger-runner outputs: tag_date: ${{ steps.taggen.outputs.TAG_DATE }} tag_sha: ${{ steps.taggen.outputs.TAG_SHA }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7091183146..9481b2d145 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: '*' workflow_dispatch: concurrency: @@ -21,7 +20,7 @@ defaults: jobs: e2e: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - uses: actions/setup-node@v3 with: @@ -68,7 +67,9 @@ jobs: key: ${{ runner.os }}-yarn-cache-${{ hashFiles('./yarn.lock') }} - name: build test run: cargo build --release --bin run-locally - - name: run test + - name: run CosmWasm test + run: RUST_BACKTRACE=1 cargo test --package run-locally --bin run-locally -- cosmos::test --nocapture + - name: run test (excluding CosmWasm) run: ./target/release/run-locally env: E2E_CI_MODE: 'true' diff --git a/.github/workflows/mergify.yml.bak b/.github/workflows/mergify.yml.bak index 3cde1aee3c..2adc55518f 100644 --- a/.github/workflows/mergify.yml.bak +++ b/.github/workflows/mergify.yml.bak @@ -24,7 +24,7 @@ on: jobs: automerge: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - name: automerge @@ -38,7 +38,7 @@ jobs: # in rust.yml complete: - runs-on: ubuntu-latest + runs-on: larger-runner needs: [build, test, lint] steps: @@ -52,7 +52,7 @@ jobs: # in solidity.yml complete: - runs-on: ubuntu-latest + runs-on: larger-runner needs: [install, lint, test] steps: diff --git a/.github/workflows/monorepo-docker.yml b/.github/workflows/monorepo-docker.yml index 9952c3507e..af71889120 100644 --- a/.github/workflows/monorepo-docker.yml +++ b/.github/workflows/monorepo-docker.yml @@ -13,7 +13,7 @@ concurrency: cancel-in-progress: true jobs: check-env: - runs-on: ubuntu-latest + runs-on: larger-runner # assign output from step to job output outputs: gcloud-service-key: ${{ steps.gcloud-service-key.outputs.defined }} @@ -27,7 +27,7 @@ jobs: run: echo "::set-output name=defined::true" build-and-push-to-gcr: - runs-on: ubuntu-latest + runs-on: larger-runner # uses check-env to determine if secrets.GCLOUD_SERVICE_KEY is defined needs: [check-env] diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index e4f51e9ff1..bab2278fdc 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -14,7 +14,7 @@ env: jobs: yarn-install: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - uses: actions/checkout@v3 with: @@ -39,7 +39,7 @@ jobs: fi yarn-build: - runs-on: ubuntu-latest + runs-on: larger-runner needs: [yarn-install] steps: - uses: actions/checkout@v3 @@ -70,7 +70,7 @@ jobs: run: yarn build lint-prettier: - runs-on: ubuntu-latest + runs-on: larger-runner needs: [yarn-install] steps: - uses: actions/checkout@v3 @@ -94,7 +94,7 @@ jobs: fi test-ts: - runs-on: ubuntu-latest + runs-on: larger-runner needs: [yarn-build] steps: - uses: actions/checkout@v3 @@ -116,7 +116,7 @@ jobs: run: yarn workspace @hyperlane-xyz/infra run test # test-env: - # runs-on: ubuntu-latest + # runs-on: larger-runner # needs: [yarn-build] # strategy: # matrix: @@ -140,7 +140,7 @@ jobs: env: ETHERSCAN_API_KEY: '' - runs-on: ubuntu-latest + runs-on: larger-runner needs: [yarn-build] steps: @@ -189,7 +189,7 @@ jobs: sarif_file: ${{ steps.slither.outputs.sarif }} coverage-sol: - runs-on: ubuntu-latest + runs-on: larger-runner needs: [yarn-build] steps: diff --git a/.github/workflows/rust-docker.yml b/.github/workflows/rust-docker.yml index 7c75426f97..4c64a7cb1c 100644 --- a/.github/workflows/rust-docker.yml +++ b/.github/workflows/rust-docker.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true jobs: check-env: - runs-on: ubuntu-latest + runs-on: larger-runner # assign output from step to job output outputs: gcloud-service-key: ${{ steps.gcloud-service-key.outputs.defined }} @@ -26,7 +26,7 @@ jobs: run: echo "::set-output name=defined::true" build-and-push-to-gcr: - runs-on: ubuntu-latest + runs-on: larger-runner # uses check-env to determine if secrets.GCLOUD_SERVICE_KEY is defined needs: [check-env] diff --git a/.github/workflows/rust-skipped.yml b/.github/workflows/rust-skipped.yml index bd2669ac17..9a8d0e07ed 100644 --- a/.github/workflows/rust-skipped.yml +++ b/.github/workflows/rust-skipped.yml @@ -12,13 +12,13 @@ env: jobs: test-rs: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - run: 'echo "No test required" ' lint-rs: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - run: 'echo "No lint required" ' diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0546f86913..ebb5623de5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -25,7 +25,7 @@ defaults: jobs: test-rs: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - uses: actions/checkout@v3 @@ -51,7 +51,7 @@ jobs: run: cargo test lint-rs: - runs-on: ubuntu-latest + runs-on: larger-runner steps: - uses: actions/checkout@v3 diff --git a/rust/chains/hyperlane-cosmos/src/mailbox.rs b/rust/chains/hyperlane-cosmos/src/mailbox.rs index 1249c54d4e..657cf0a980 100644 --- a/rust/chains/hyperlane-cosmos/src/mailbox.rs +++ b/rust/chains/hyperlane-cosmos/src/mailbox.rs @@ -249,7 +249,7 @@ impl CosmosMailboxIndexer { /// The message dispatch event type from the CW contract. const MESSAGE_DISPATCH_EVENT_TYPE: &str = "mailbox_dispatch"; - /// Create a reference to a mailbox at a specific Ethereum address on some + /// Create a reference to a mailbox at a specific Cosmos address on some /// chain pub fn new( conf: ConnectionConf, diff --git a/rust/utils/run-locally/src/cosmos/mod.rs b/rust/utils/run-locally/src/cosmos/mod.rs index 75f97908e7..01e3864edc 100644 --- a/rust/utils/run-locally/src/cosmos/mod.rs +++ b/rust/utils/run-locally/src/cosmos/mod.rs @@ -1,12 +1,13 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::thread::sleep; -use std::time::Duration; +use std::time::{Duration, Instant}; use std::{env, fs}; use cosmwasm_schema::cw_serde; use hpl_interface::types::bech32_decode; use macro_rules_attribute::apply; +use maplit::hashmap; use tempfile::tempdir; mod cli; @@ -26,7 +27,7 @@ use crate::cosmos::link::link_networks; use crate::logging::log; use crate::program::Program; use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle}; -use crate::AGENT_BIN_PATH; +use crate::{fetch_metric, AGENT_BIN_PATH}; use cli::{OsmosisCLI, OsmosisEndpoint}; use self::deploy::deploy_cw_hyperlane; @@ -290,6 +291,7 @@ fn launch_cosmos_relayer( .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true") .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" }) .hyp_env("GASPAYMENTENFORCEMENT", "[{\"type\": \"none\"}]") + .hyp_env("METRICSPORT", 9093.to_string()) .spawn("RLY"); relayer @@ -300,6 +302,7 @@ const ENV_CW_HYPERLANE_PATH_KEY: &str = "E2E_CW_HYPERLANE_PATH"; #[allow(dead_code)] fn run_locally() { + const TIMEOUT_SECS: u64 = 60 * 10; let debug = false; log!("Building rust..."); @@ -453,6 +456,7 @@ fn run_locally() { ); // dispatch messages + let mut dispatched_messages = 0; for node in nodes.iter() { let targets = nodes @@ -469,6 +473,7 @@ fn run_locally() { } for target in targets { + dispatched_messages += 1; let cli = OsmosisCLI::new( osmosisd.clone(), node.launch_resp.home_path.to_str().unwrap(), @@ -504,7 +509,70 @@ fn run_locally() { relayer: hpl_rly.join(), }; - sleep(Duration::from_secs(20)); // wait for a long time + // Mostly copy-pasta from `rust/utils/run-locally/src/main.rs` + // TODO: refactor to share code + let loop_start = Instant::now(); + // give things a chance to fully start. + sleep(Duration::from_secs(5)); + let mut failure_occurred = false; + loop { + // look for the end condition. + if termination_invariants_met(dispatched_messages).unwrap_or(false) { + // end condition reached successfully + break; + } else if (Instant::now() - loop_start).as_secs() > TIMEOUT_SECS { + // we ran out of time + log!("timeout reached before message submission was confirmed"); + failure_occurred = true; + break; + } + + sleep(Duration::from_secs(5)); + } + + if failure_occurred { + panic!("E2E tests failed"); + } else { + log!("E2E tests passed"); + } +} + +fn termination_invariants_met(messages_expected: u32) -> eyre::Result { + let gas_payments_scraped = fetch_metric( + "9093", + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payment"}, + )? + .iter() + .sum::(); + let expected_gas_payments = messages_expected; + if gas_payments_scraped != expected_gas_payments { + log!( + "Scraper has scraped {} gas payments, expected {}", + gas_payments_scraped, + expected_gas_payments + ); + return Ok(false); + } + + let delivered_messages_scraped = fetch_metric( + "9093", + "hyperlane_operations_processed_count", + &hashmap! {"phase" => "confirmed"}, + )? + .iter() + .sum::(); + if delivered_messages_scraped != messages_expected { + log!( + "Relayer confirmed {} submitted messages, expected {}", + delivered_messages_scraped, + messages_expected + ); + return Ok(false); + } + + log!("Termination invariants have been meet"); + Ok(true) } #[cfg(test)] diff --git a/rust/utils/run-locally/src/cosmos/utils.rs b/rust/utils/run-locally/src/cosmos/utils.rs index c6cc7b5cca..759038d09d 100644 --- a/rust/utils/run-locally/src/cosmos/utils.rs +++ b/rust/utils/run-locally/src/cosmos/utils.rs @@ -8,7 +8,9 @@ use crate::utils::TaskHandle; pub(crate) fn sed(from: &str, to: &str, file: &str) { Program::new("sed") .raw_arg("-i") - .cmd("") + // Temporary fix to get `sed` working on linux + // Note that this breaks the script on mac + // .cmd("") .cmd(format!("s/{from}/{to}/g")) .cmd(file) .run()