Merge branch 'trevor/new-featv3-cosmos-oct-28' into dan/more-cw-agent…

…-remediations

.github/workflows/agent-release-artifacts.yml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   prepare:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     outputs:
       tag_date: ${{ steps.taggen.outputs.TAG_DATE }}
       tag_sha: ${{ steps.taggen.outputs.TAG_SHA }}

.github/workflows/e2e.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: '*'
   workflow_dispatch:
 
 concurrency:
@@ -21,7 +20,7 @@ defaults:
 
 jobs:
   e2e:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     steps:
       - uses: actions/setup-node@v3
         with:
@@ -68,7 +67,9 @@ jobs:
           key: ${{ runner.os }}-yarn-cache-${{ hashFiles('./yarn.lock') }}
       - name: build test
         run: cargo build --release --bin run-locally
-      - name: run test
+      - name: run CosmWasm test
+        run: RUST_BACKTRACE=1 cargo test --package run-locally --bin run-locally -- cosmos::test --nocapture
+      - name: run test (excluding CosmWasm)
         run: ./target/release/run-locally
         env:
           E2E_CI_MODE: 'true'

.github/workflows/mergify.yml.bak

@@ -24,7 +24,7 @@ on:
 jobs:
   automerge:
 
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     steps:
       - name: automerge
@@ -38,7 +38,7 @@ jobs:
 
 # in rust.yml
   complete:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [build, test, lint]
 
     steps:
@@ -52,7 +52,7 @@ jobs:
 
 # in solidity.yml
   complete:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [install, lint, test]
 
     steps:

.github/workflows/monorepo-docker.yml

@@ -13,7 +13,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   check-env:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     # assign output from step to job output
     outputs:
       gcloud-service-key: ${{ steps.gcloud-service-key.outputs.defined }}
@@ -27,7 +27,7 @@ jobs:
         run: echo "::set-output name=defined::true"
 
   build-and-push-to-gcr:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     # uses check-env to determine if secrets.GCLOUD_SERVICE_KEY is defined
     needs: [check-env]

.github/workflows/node.yml

@@ -14,7 +14,7 @@ env:
 
 jobs:
   yarn-install:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     steps:
       - uses: actions/checkout@v3
         with:
@@ -39,7 +39,7 @@ jobs:
           fi
 
   yarn-build:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [yarn-install]
     steps:
       - uses: actions/checkout@v3
@@ -70,7 +70,7 @@ jobs:
         run: yarn build
 
   lint-prettier:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [yarn-install]
     steps:
       - uses: actions/checkout@v3
@@ -94,7 +94,7 @@ jobs:
           fi
 
   test-ts:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [yarn-build]
     steps:
       - uses: actions/checkout@v3
@@ -116,7 +116,7 @@ jobs:
         run: yarn workspace @hyperlane-xyz/infra run test
 
   # test-env:
-  #   runs-on: ubuntu-latest
+  #   runs-on: larger-runner
   #   needs: [yarn-build]
   #   strategy:
   #     matrix:
@@ -140,7 +140,7 @@ jobs:
     env:
       ETHERSCAN_API_KEY: ''
 
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [yarn-build]
 
     steps:
@@ -189,7 +189,7 @@ jobs:
           sarif_file: ${{ steps.slither.outputs.sarif }}
 
   coverage-sol:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     needs: [yarn-build]
 
     steps:

.github/workflows/rust-docker.yml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   check-env:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
     # assign output from step to job output
     outputs:
       gcloud-service-key: ${{ steps.gcloud-service-key.outputs.defined }}
@@ -26,7 +26,7 @@ jobs:
         run: echo "::set-output name=defined::true"
 
   build-and-push-to-gcr:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     # uses check-env to determine if secrets.GCLOUD_SERVICE_KEY is defined
     needs: [check-env]

.github/workflows/rust-skipped.yml

@@ -12,13 +12,13 @@ env:
 
 jobs:
   test-rs:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     steps:
       - run: 'echo "No test required" '
 
   lint-rs:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     steps:
       - run: 'echo "No lint required" '

.github/workflows/rust.yml

@@ -25,7 +25,7 @@ defaults:
 
 jobs:
   test-rs:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     steps:
       - uses: actions/checkout@v3
@@ -51,7 +51,7 @@ jobs:
         run: cargo test
 
   lint-rs:
-    runs-on: ubuntu-latest
+    runs-on: larger-runner
 
     steps:
       - uses: actions/checkout@v3

rust/chains/hyperlane-cosmos/src/mailbox.rs

@@ -249,7 +249,7 @@ impl CosmosMailboxIndexer {
     /// The message dispatch event type from the CW contract.
     const MESSAGE_DISPATCH_EVENT_TYPE: &str = "mailbox_dispatch";
 
-    /// Create a reference to a mailbox at a specific Ethereum address on some
+    /// Create a reference to a mailbox at a specific Cosmos address on some
     /// chain
     pub fn new(
         conf: ConnectionConf,

rust/utils/run-locally/src/cosmos/mod.rs

@@ -1,12 +1,13 @@
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 use std::thread::sleep;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use std::{env, fs};
 
 use cosmwasm_schema::cw_serde;
 use hpl_interface::types::bech32_decode;
 use macro_rules_attribute::apply;
+use maplit::hashmap;
 use tempfile::tempdir;
 
 mod cli;
@@ -26,7 +27,7 @@ use crate::cosmos::link::link_networks;
 use crate::logging::log;
 use crate::program::Program;
 use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle};
-use crate::AGENT_BIN_PATH;
+use crate::{fetch_metric, AGENT_BIN_PATH};
 use cli::{OsmosisCLI, OsmosisEndpoint};
 
 use self::deploy::deploy_cw_hyperlane;
@@ -290,6 +291,7 @@ fn launch_cosmos_relayer(
         .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true")
         .hyp_env("TRACING_LEVEL", if debug { "debug" } else { "info" })
         .hyp_env("GASPAYMENTENFORCEMENT", "[{\"type\": \"none\"}]")
+        .hyp_env("METRICSPORT", 9093.to_string())
         .spawn("RLY");
 
     relayer
@@ -300,6 +302,7 @@ const ENV_CW_HYPERLANE_PATH_KEY: &str = "E2E_CW_HYPERLANE_PATH";
 
 #[allow(dead_code)]
 fn run_locally() {
+    const TIMEOUT_SECS: u64 = 60 * 10;
     let debug = false;
 
     log!("Building rust...");
@@ -453,6 +456,7 @@ fn run_locally() {
     );
 
     // dispatch messages
+    let mut dispatched_messages = 0;
 
     for node in nodes.iter() {
         let targets = nodes
@@ -469,6 +473,7 @@ fn run_locally() {
         }
 
         for target in targets {
+            dispatched_messages += 1;
             let cli = OsmosisCLI::new(
                 osmosisd.clone(),
                 node.launch_resp.home_path.to_str().unwrap(),
@@ -504,7 +509,70 @@ fn run_locally() {
         relayer: hpl_rly.join(),
     };
 
-    sleep(Duration::from_secs(20)); // wait for a long time
+    // Mostly copy-pasta from `rust/utils/run-locally/src/main.rs`
+    // TODO: refactor to share code
+    let loop_start = Instant::now();
+    // give things a chance to fully start.
+    sleep(Duration::from_secs(5));
+    let mut failure_occurred = false;
+    loop {
+        // look for the end condition.
+        if termination_invariants_met(dispatched_messages).unwrap_or(false) {
+            // end condition reached successfully
+            break;
+        } else if (Instant::now() - loop_start).as_secs() > TIMEOUT_SECS {
+            // we ran out of time
+            log!("timeout reached before message submission was confirmed");
+            failure_occurred = true;
+            break;
+        }
+
+        sleep(Duration::from_secs(5));
+    }
+
+    if failure_occurred {
+        panic!("E2E tests failed");
+    } else {
+        log!("E2E tests passed");
+    }
+}
+
+fn termination_invariants_met(messages_expected: u32) -> eyre::Result<bool> {
+    let gas_payments_scraped = fetch_metric(
+        "9093",
+        "hyperlane_contract_sync_stored_events",
+        &hashmap! {"data_type" => "gas_payment"},
+    )?
+    .iter()
+    .sum::<u32>();
+    let expected_gas_payments = messages_expected;
+    if gas_payments_scraped != expected_gas_payments {
+        log!(
+            "Scraper has scraped {} gas payments, expected {}",
+            gas_payments_scraped,
+            expected_gas_payments
+        );
+        return Ok(false);
+    }
+
+    let delivered_messages_scraped = fetch_metric(
+        "9093",
+        "hyperlane_operations_processed_count",
+        &hashmap! {"phase" => "confirmed"},
+    )?
+    .iter()
+    .sum::<u32>();
+    if delivered_messages_scraped != messages_expected {
+        log!(
+            "Relayer confirmed {} submitted messages, expected {}",
+            delivered_messages_scraped,
+            messages_expected
+        );
+        return Ok(false);
+    }
+
+    log!("Termination invariants have been meet");
+    Ok(true)
 }
 
 #[cfg(test)]

rust/utils/run-locally/src/cosmos/utils.rs

@@ -8,7 +8,9 @@ use crate::utils::TaskHandle;
 pub(crate) fn sed(from: &str, to: &str, file: &str) {
     Program::new("sed")
         .raw_arg("-i")
-        .cmd("")
+        // Temporary fix to get `sed` working on linux
+        // Note that this breaks the script on mac
+        // .cmd("")
         .cmd(format!("s/{from}/{to}/g"))
         .cmd(file)
         .run()