feat: add run_with_required_agents function for TryCP scenarios

.github/workflows/test.yaml

@@ -177,8 +177,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -194,8 +194,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_call_rate -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_call_rate -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -209,8 +209,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#two_party_countersigning -- --targets targets-ci.yaml --behaviour initiate:1 --behaviour participate:1 --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#two_party_countersigning -- --targets targets-ci.yaml --behaviour initiate:1 --behaviour participate:1 --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -226,8 +226,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#validation_receipts -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#validation_receipts -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -243,8 +243,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_signals -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_signals -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"

CHANGELOG.md

@@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A new tool for summarising scenario outcomes. This is called the `summariser` which is possibly a working title! The 
   tool is specific to the scenarios in this project but does have some re-usable pieces. It remains to be decided whether
   we will separate those parts out and publish them as a crate. For now, this is private to the project.
+- `run_with_required_agents` function for TryCP scenarios that fails if the number of agents that completed the scenario
+  is less than the passed `min_required_agents`. Can be overridden with the `MIN_REQUIRED_AGENTS` environment variable.
 
 ### Changed
 - Updated to new Holochain client version 0.5.0-alpha.4 which allowed `&mut self` to be replaced with `&self` in admin
@@ -34,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   other words, the `agent` is not reported when calling a clone cell.
 - All metrics are now reported in seconds, as an `f64`. There were some types still using milliseconds which made reporting
   across scenarios more complex.
+- Increased TryCP test scenario duration to 30s in CI [Test Workflow](.github/workflows/test.yaml).
 
 ### Deprecated
 ### Removed

bindings/trycp_runner/src/common.rs

@@ -1,17 +1,19 @@
 use crate::context::TryCPAgentContext;
 use crate::runner_context::TryCPRunnerContext;
-use anyhow::Context;
+use anyhow::{bail, Context};
 use holochain_client::AuthorizeSigningCredentialsPayload;
 use holochain_conductor_api::{CellInfo, IssueAppAuthenticationTokenPayload};
 use holochain_types::app::{AppBundle, AppBundleSource, InstallAppPayload};
 use holochain_types::prelude::RoleName;
 use holochain_types::websocket::AllowedOrigins;
+use log::{debug, warn};
 use std::path::PathBuf;
 use std::sync::{Arc, OnceLock};
 use std::time::{Duration, Instant};
 use trycp_client_instrumented::prelude::TryCPClient;
 use wind_tunnel_runner::prelude::{
-    AgentContext, HookResult, UserValuesConstraint, WindTunnelResult,
+    run, AgentContext, HookResult, ScenarioDefinitionBuilder, UserValuesConstraint,
+    WindTunnelResult,
 };
 
 /// Connects to a TryCP server using the current agent index and the list of targets.
@@ -466,3 +468,32 @@ where
             .map_err(|e| anyhow::anyhow!("Decoding failure: {:?}", e))
     })
 }
+
+/// Call [`run`] for a scenario and check that it completed with the minimum required agents.
+///
+/// The value of `min_required_agents` can be overridden with the environment variable
+/// `MIN_REQUIRED_AGENTS` when running a scenario.
+pub fn run_with_required_agents<RV: UserValuesConstraint, V: UserValuesConstraint>(
+    definition: ScenarioDefinitionBuilder<RV, V>,
+    min_required_agents: usize,
+) -> anyhow::Result<()> {
+    let agents_at_completion = run(definition)?;
+
+    let min_required_agents = std::env::var("MIN_REQUIRED_AGENTS")
+        .inspect_err(|_| debug!("MIN_REQUIRED_AGENTS not set, using default value"))
+        .ok()
+        .and_then(|v| {
+            v.parse()
+                .inspect_err(|_| warn!("Invalid MIN_REQUIRED_AGENTS value. Using default"))
+                .ok()
+        })
+        .unwrap_or(min_required_agents);
+
+    if agents_at_completion < min_required_agents {
+        bail!("Not enough agents ran scenario to completion: expected at least {min_required_agents}, actual {agents_at_completion}");
+    }
+
+    println!("Finished with {} agents", agents_at_completion);
+
+    Ok(())
+}

bindings/trycp_runner/src/lib.rs

@@ -9,7 +9,7 @@ pub mod prelude {
     pub use crate::cli::WindTunnelTryCPScenarioCli;
     pub use crate::common::{
         call_zome, connect_trycp_client, disconnect_trycp_client, dump_logs, install_app,
-        reset_trycp_remote, shutdown_remote, try_wait_for_min_peers,
+        reset_trycp_remote, run_with_required_agents, shutdown_remote, try_wait_for_min_peers,
     };
     pub use crate::context::{DefaultScenarioValues, TryCPAgentContext};
     pub use crate::definition::TryCPScenarioDefinitionBuilder;

framework/runner/src/context.rs

@@ -138,7 +138,7 @@ impl<RV: UserValuesConstraint, V: UserValuesConstraint> AgentContext<RV, V> {
         &self.agent_name
     }
 
-    /// The user-supplied value from [ScenarioDefinitionBuilder::use_named_agent_behaviour].
+    /// The user-supplied value from [crate::prelude::ScenarioDefinitionBuilder::use_named_agent_behaviour].
     ///
     /// From within the behaviour, you know which behaviour you are assigned to. This is useful for
     /// the setup and teardown hooks where you may need to adjust the actions your agent takes

scenarios/remote_call_rate/src/main.rs

@@ -158,9 +158,7 @@ fn main() -> WindTunnelResult<()> {
     .use_agent_behaviour(agent_behaviour)
     .use_agent_teardown(agent_teardown);
 
-    let agents_at_completion = run(builder)?;
-
-    println!("Finished with {} agents", agents_at_completion);
+    run_with_required_agents(builder, 1)?;
 
     Ok(())
 }

scenarios/remote_signals/src/main.rs

@@ -237,9 +237,7 @@ fn main() -> WindTunnelResult<()> {
     .use_agent_behaviour(agent_behaviour)
     .use_agent_teardown(agent_teardown);
 
-    let agents_at_completion = run(builder)?;
-
-    println!("Finished with {} agents", agents_at_completion);
+    run_with_required_agents(builder, 1)?;
 
     Ok(())
 }

scenarios/trycp_write_validated/src/main.rs

@@ -86,9 +86,7 @@ fn main() -> WindTunnelResult<()> {
         .use_agent_behaviour(agent_behaviour)
         .use_agent_teardown(agent_teardown);
 
-    let agents_at_completion = run(builder)?;
-
-    println!("Finished with {} agents", agents_at_completion);
+    run_with_required_agents(builder, 1)?;
 
     Ok(())
 }

scenarios/two_party_countersigning/src/main.rs

@@ -653,9 +653,7 @@ fn main() -> WindTunnelResult<()> {
     .use_named_agent_behaviour("participate", agent_behaviour_participate)
     .use_agent_teardown(agent_teardown);
 
-    let agents_at_completion = run(builder)?;
-
-    println!("Finished with {} agents", agents_at_completion);
+    run_with_required_agents(builder, 1)?;
 
     Ok(())
 }

scenarios/validation_receipts/src/main.rs

@@ -194,9 +194,7 @@ fn main() -> WindTunnelResult<()> {
     .use_agent_behaviour(agent_behaviour)
     .use_agent_teardown(agent_teardown);
 
-    let agents_at_completion = run(builder)?;
-
-    println!("Finished with {} agents", agents_at_completion);
+    run_with_required_agents(builder, 1)?;
 
     Ok(())
 }