From 09e07f3fc41c2e0fe3a91af42a65a98085c561d4 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Wed, 23 Oct 2024 20:26:18 +0200 Subject: [PATCH 1/7] feat: add run_with_required_agents function for TryCP scenarios This calls `run` and checks that it completes with at least the passed number of agents. This can be overridden via the `MIN_REQUIRED_AGENTS` environment variable. --- bindings/trycp_runner/src/common.rs | 29 +++++++++++++++++-- bindings/trycp_runner/src/lib.rs | 2 +- scenarios/remote_call_rate/src/main.rs | 4 +-- scenarios/remote_signals/src/main.rs | 4 +-- scenarios/trycp_write_validated/src/main.rs | 4 +-- .../two_party_countersigning/src/main.rs | 4 +-- scenarios/validation_receipts/src/main.rs | 4 +-- 7 files changed, 33 insertions(+), 18 deletions(-) diff --git a/bindings/trycp_runner/src/common.rs b/bindings/trycp_runner/src/common.rs index 421840db..11749661 100644 --- a/bindings/trycp_runner/src/common.rs +++ b/bindings/trycp_runner/src/common.rs @@ -1,6 +1,6 @@ use crate::context::TryCPAgentContext; use crate::runner_context::TryCPRunnerContext; -use anyhow::Context; +use anyhow::{bail, Context}; use holochain_client::AuthorizeSigningCredentialsPayload; use holochain_conductor_api::{CellInfo, IssueAppAuthenticationTokenPayload}; use holochain_types::app::{AppBundle, AppBundleSource, InstallAppPayload}; @@ -11,7 +11,8 @@ use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; use trycp_client_instrumented::prelude::TryCPClient; use wind_tunnel_runner::prelude::{ - AgentContext, HookResult, UserValuesConstraint, WindTunnelResult, + run, AgentContext, HookResult, ScenarioDefinitionBuilder, UserValuesConstraint, + WindTunnelResult, }; /// Connects to a TryCP server using the current agent index and the list of targets. @@ -466,3 +467,27 @@ where .map_err(|e| anyhow::anyhow!("Decoding failure: {:?}", e)) }) } + +/// Call [`run`] for a scenario and check that it completed with the minimum required agents. +/// +/// The value of `min_required_agents` can be overridden with the environment variable +/// `MIN_REQUIRED_AGENTS` when running a scenario. +pub fn run_with_required_agents( + definition: ScenarioDefinitionBuilder, + min_required_agents: usize, +) -> anyhow::Result<()> { + let agents_at_completion = run(definition)?; + + let min_required_agents = std::env::var("MIN_REQUIRED_AGENTS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(min_required_agents); + + if agents_at_completion < min_required_agents { + bail!("Not enough agents ran scenario to completion: expected at least {min_required_agents}, actual {agents_at_completion}"); + } + + println!("Finished with {} agents", agents_at_completion); + + Ok(()) +} diff --git a/bindings/trycp_runner/src/lib.rs b/bindings/trycp_runner/src/lib.rs index 7a5a878d..c80a0d18 100644 --- a/bindings/trycp_runner/src/lib.rs +++ b/bindings/trycp_runner/src/lib.rs @@ -9,7 +9,7 @@ pub mod prelude { pub use crate::cli::WindTunnelTryCPScenarioCli; pub use crate::common::{ call_zome, connect_trycp_client, disconnect_trycp_client, dump_logs, install_app, - reset_trycp_remote, shutdown_remote, try_wait_for_min_peers, + reset_trycp_remote, run_with_required_agents, shutdown_remote, try_wait_for_min_peers, }; pub use crate::context::{DefaultScenarioValues, TryCPAgentContext}; pub use crate::definition::TryCPScenarioDefinitionBuilder; diff --git a/scenarios/remote_call_rate/src/main.rs b/scenarios/remote_call_rate/src/main.rs index e3e582b1..a4d4d6e0 100644 --- a/scenarios/remote_call_rate/src/main.rs +++ b/scenarios/remote_call_rate/src/main.rs @@ -158,9 +158,7 @@ fn main() -> WindTunnelResult<()> { .use_agent_behaviour(agent_behaviour) .use_agent_teardown(agent_teardown); - let agents_at_completion = run(builder)?; - - println!("Finished with {} agents", agents_at_completion); + run_with_required_agents(builder, 1)?; Ok(()) } diff --git a/scenarios/remote_signals/src/main.rs b/scenarios/remote_signals/src/main.rs index f2b9c984..44a75a74 100644 --- a/scenarios/remote_signals/src/main.rs +++ b/scenarios/remote_signals/src/main.rs @@ -237,9 +237,7 @@ fn main() -> WindTunnelResult<()> { .use_agent_behaviour(agent_behaviour) .use_agent_teardown(agent_teardown); - let agents_at_completion = run(builder)?; - - println!("Finished with {} agents", agents_at_completion); + run_with_required_agents(builder, 1)?; Ok(()) } diff --git a/scenarios/trycp_write_validated/src/main.rs b/scenarios/trycp_write_validated/src/main.rs index b8e4744d..5ce47072 100644 --- a/scenarios/trycp_write_validated/src/main.rs +++ b/scenarios/trycp_write_validated/src/main.rs @@ -86,9 +86,7 @@ fn main() -> WindTunnelResult<()> { .use_agent_behaviour(agent_behaviour) .use_agent_teardown(agent_teardown); - let agents_at_completion = run(builder)?; - - println!("Finished with {} agents", agents_at_completion); + run_with_required_agents(builder, 1)?; Ok(()) } diff --git a/scenarios/two_party_countersigning/src/main.rs b/scenarios/two_party_countersigning/src/main.rs index e0e21217..3d6e45e1 100644 --- a/scenarios/two_party_countersigning/src/main.rs +++ b/scenarios/two_party_countersigning/src/main.rs @@ -653,9 +653,7 @@ fn main() -> WindTunnelResult<()> { .use_named_agent_behaviour("participate", agent_behaviour_participate) .use_agent_teardown(agent_teardown); - let agents_at_completion = run(builder)?; - - println!("Finished with {} agents", agents_at_completion); + run_with_required_agents(builder, 1)?; Ok(()) } diff --git a/scenarios/validation_receipts/src/main.rs b/scenarios/validation_receipts/src/main.rs index 2b4f03f2..bfac2109 100644 --- a/scenarios/validation_receipts/src/main.rs +++ b/scenarios/validation_receipts/src/main.rs @@ -194,9 +194,7 @@ fn main() -> WindTunnelResult<()> { .use_agent_behaviour(agent_behaviour) .use_agent_teardown(agent_teardown); - let agents_at_completion = run(builder)?; - - println!("Finished with {} agents", agents_at_completion); + run_with_required_agents(builder, 1)?; Ok(()) } From 663f147ee47c31b64d5f8aff222b8426464caa35 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Wed, 23 Oct 2024 20:26:55 +0200 Subject: [PATCH 2/7] docs: fix broken type link in doc-comment --- framework/runner/src/context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/runner/src/context.rs b/framework/runner/src/context.rs index 0698fcc7..a5555cfa 100644 --- a/framework/runner/src/context.rs +++ b/framework/runner/src/context.rs @@ -138,7 +138,7 @@ impl AgentContext { &self.agent_name } - /// The user-supplied value from [ScenarioDefinitionBuilder::use_named_agent_behaviour]. + /// The user-supplied value from [crate::prelude::ScenarioDefinitionBuilder::use_named_agent_behaviour]. /// /// From within the behaviour, you know which behaviour you are assigned to. This is useful for /// the setup and teardown hooks where you may need to adjust the actions your agent takes From 9c96fe22fa2c863600405bfc0c0a0be0cc99708d Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Thu, 24 Oct 2024 14:17:02 +0200 Subject: [PATCH 3/7] docs: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b50c6e1..fd104f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A new tool for summarising scenario outcomes. This is called the `summariser` which is possibly a working title! The tool is specific to the scenarios in this project but does have some re-usable pieces. It remains to be decided whether we will separate those parts out and publish them as a crate. For now, this is private to the project. +- `run_with_required_agents` function for TryCP scenarios that fails if the number of agents that completed the scenario + is less than the passed `min_required_agents`. Can be overridden with the `MIN_REQUIRED_AGENTS` environment variable. ### Changed - Updated to new Holochain client version 0.5.0-alpha.4 which allowed `&mut self` to be replaced with `&self` in admin From eef8df62ad400c02c1ecdf225f0aec21adc0426a Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Thu, 24 Oct 2024 16:45:03 +0200 Subject: [PATCH 4/7] chore: increase TryCP test scenario duration to 30s in CI --- .github/workflows/test.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 65558a64..9a538e39 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -177,8 +177,8 @@ jobs: # Start a TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &" - # Run the scenario for 10 seconds - RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress + # Run the scenario for 30 seconds + RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress # Stop the TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp" @@ -194,8 +194,8 @@ jobs: # Start a TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &" - # Run the scenario for 10 seconds - RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_call_rate -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress + # Run the scenario for 30 seconds + RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_call_rate -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress # Stop the TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp" @@ -209,8 +209,8 @@ jobs: # Start a TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &" - # Run the scenario for 10 seconds - RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#two_party_countersigning -- --targets targets-ci.yaml --behaviour initiate:1 --behaviour participate:1 --instances-per-target 2 --duration 10 --no-progress + # Run the scenario for 30 seconds + RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#two_party_countersigning -- --targets targets-ci.yaml --behaviour initiate:1 --behaviour participate:1 --instances-per-target 2 --duration 30 --no-progress # Stop the TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp" @@ -226,8 +226,8 @@ jobs: # Start a TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &" - # Run the scenario for 10 seconds - RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#validation_receipts -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress + # Run the scenario for 30 seconds + RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#validation_receipts -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress # Stop the TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp" @@ -243,8 +243,8 @@ jobs: # Start a TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &" - # Run the scenario for 10 seconds - RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_signals -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress + # Run the scenario for 30 seconds + RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_signals -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress # Stop the TryCP instance nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp" From a35f96c91364aad6970a66c45e180f12ae631580 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 28 Oct 2024 15:39:47 +0100 Subject: [PATCH 5/7] feat: log warning when MIN_REQUIRED_AGENTS invalid --- bindings/trycp_runner/src/common.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bindings/trycp_runner/src/common.rs b/bindings/trycp_runner/src/common.rs index 11749661..253e3583 100644 --- a/bindings/trycp_runner/src/common.rs +++ b/bindings/trycp_runner/src/common.rs @@ -6,6 +6,7 @@ use holochain_conductor_api::{CellInfo, IssueAppAuthenticationTokenPayload}; use holochain_types::app::{AppBundle, AppBundleSource, InstallAppPayload}; use holochain_types::prelude::RoleName; use holochain_types::websocket::AllowedOrigins; +use log::warn; use std::path::PathBuf; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; @@ -480,7 +481,11 @@ pub fn run_with_required_agents Date: Mon, 28 Oct 2024 15:46:27 +0100 Subject: [PATCH 6/7] docs: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd104f2d..33c63f7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 other words, the `agent` is not reported when calling a clone cell. - All metrics are now reported in seconds, as an `f64`. There were some types still using milliseconds which made reporting across scenarios more complex. +- Increased TryCP test scenario duration to 30s in CI [Test Workflow](.github/workflows/test.yaml). ### Deprecated ### Removed From ee58fec6ad3698a23131ed815fe4f63440c266a9 Mon Sep 17 00:00:00 2001 From: Callum Dunster Date: Mon, 28 Oct 2024 15:48:33 +0100 Subject: [PATCH 7/7] feat: log debug message when MIN_REQUIRED_AGENTS is not set --- bindings/trycp_runner/src/common.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/trycp_runner/src/common.rs b/bindings/trycp_runner/src/common.rs index 253e3583..ae2821e5 100644 --- a/bindings/trycp_runner/src/common.rs +++ b/bindings/trycp_runner/src/common.rs @@ -6,7 +6,7 @@ use holochain_conductor_api::{CellInfo, IssueAppAuthenticationTokenPayload}; use holochain_types::app::{AppBundle, AppBundleSource, InstallAppPayload}; use holochain_types::prelude::RoleName; use holochain_types::websocket::AllowedOrigins; -use log::warn; +use log::{debug, warn}; use std::path::PathBuf; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; @@ -480,6 +480,7 @@ pub fn run_with_required_agents