Merge remote-tracking branch 'holochain/main' into holochain-chc-perf…

…ormance-testing
holochain · Oct 29, 2024 · 7e0f152 · 7e0f152
2 parents adedffb + f2cd641
commit 7e0f152
Show file tree

Hide file tree

Showing 139 changed files with 7,677 additions and 697 deletions.
diff --git a/.github/workflows/cargo_update.yaml b/.github/workflows/cargo_update.yaml
@@ -24,9 +24,9 @@ jobs:
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
         with:
-            commit-message: "Update Cargo.lock"
-            title: "Update Cargo.lock"
-            body: "This PR updates the Cargo.lock file."
-            branch: "update-cargo-lock"
-            branch-suffix: "short-commit-hash"
-            base: "main"
+          commit-message: "Update Cargo.lock"
+          title: "Update Cargo.lock"
+          body: "This PR updates the Cargo.lock file."
+          branch: "update-cargo-lock"
+          branch-suffix: "short-commit-hash"
+          base: "main"
diff --git a/.github/workflows/flake_update.yaml b/.github/workflows/flake_update.yaml
@@ -24,9 +24,9 @@ jobs:
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
         with:
-            commit-message: "Update flake.lock"
-            title: "Update flake.lock"
-            body: "This PR updates the flake.lock file."
-            branch: "update-flake-lock"
-            branch-suffix: "short-commit-hash"
-            base: "main"
+          commit-message: "Update flake.lock"
+          title: "Update flake.lock"
+          body: "This PR updates the flake.lock file."
+          branch: "update-flake-lock"
+          branch-suffix: "short-commit-hash"
+          base: "main"
diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml
@@ -7,14 +7,18 @@ env:
   INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
   WT_METRICS_DIR: "${{ github.workspace }}/telegraf/metrics"
 
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
 jobs:
   local-test:
     runs-on: [self-hosted, wind-tunnel]
     strategy:
       fail-fast: false
       matrix:
         # To run a local test with default configuration, add the scenario name to this array.
-        scenario: [ zome_call_single_value, single_write_many_read, write_read, write_query, local_signals, write_validated ]
+        scenario: [zome_call_single_value, single_write_many_read, write_read, write_query, local_signals, write_validated]
         # To run a local test with additional configuration, add the scenario name and `extra-args` as an `include` item.
         include:
           - scenario: dht_sync_lag
@@ -87,7 +91,7 @@ jobs:
       fail-fast: false
       matrix:
         # To run a test with TryCP and default configuration, add the scenario name to this array.
-        scenario: [ trycp_write_validated, remote_call_rate, validation_receipts ]
+        scenario: [trycp_write_validated, remote_call_rate, validation_receipts, remote_signals]
         # To run a test with TryCP and additional configuration, add the scenario name and `extra-args` as an `include` item.
         include:
           - scenario: two_party_countersigning
@@ -105,7 +109,7 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          RUST_LOG=info CONDUCTOR_CONFIG="CI" TRYCP_RUST_LOG="info" MIN_PEERS=2 nix run .#${{ matrix.scenario }} -- --targets targets-ci.yaml --instances-per-target 2 --duration 120 --no-progress --reporter influx-file ${{ matrix.extra-args }} > >(tee logs/scenario-stdout.log) 2> >(tee logs/scenario-stderr.log >&2)
+          RUST_LOG=info TRYCP_RUST_LOG="info" MIN_PEERS=2 nix run .#${{ matrix.scenario }} -- --targets targets.yaml --instances-per-target 2 --duration 120 --no-progress --reporter influx-file ${{ matrix.extra-args }} > >(tee logs/scenario-stdout.log) 2> >(tee logs/scenario-stderr.log >&2)
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -2,13 +2,13 @@ name: "test"
 
 on:
   push:
-    branches: [main, develop]
+    branches: [main]
   pull_request:
-    branches: [main, develop]
+    branches: [main]
   merge_group:
 
 concurrency:
-  group: ${{ github.ref_name }}
+  group: ${{ github.workflow }}-${{ github.ref_name }}
   cancel-in-progress: true
 
 jobs:
@@ -49,6 +49,14 @@ jobs:
         run: |
           nix develop .#ci -c bash -c "shellcheck scripts/*.sh"
 
+      - name: Check TOML formatting
+        run: |
+          nix develop .#ci -c bash -c "./scripts/format-toml.sh --check"
+
+      - name: Check YAML formatting
+        run: |
+          nix develop .#ci -c bash -c "./scripts/format-yaml.sh -lint"
+
       - name: Check Nix formatting
         run: |
           nix develop .#ci -c bash -c "source ./scripts/checks.sh && check_nix_fmt"
@@ -80,20 +88,6 @@ jobs:
 
           pkill hc && pkill holochain && pkill lair-keystore
 
-      - name: Smoke test - zome_call_single_value with CHC enabled
-        run: |
-          # start the CHC reference implementation service
-          nix develop .#ci -c bash -c "hc-chc-service --port 8181 &"
-
-          # Start a sandbox conductor and run it in the background
-          nix develop .#ci -c bash -c "hc s clean && echo "1234" | hc s --piped create --chc-url http://localhost:8181 && echo "1234" | hc s --piped -f 8888 run &"
-
-          # TODO using `localhost` is resolving to an IPv6 address, but why is that giving a connection refused?
-          # Run the scenario for 5 seconds
-          RUST_LOG=info nix run .#zome_call_single_value -- --connection-string ws://localhost:8888 --duration 5 --no-progress
-
-          pkill hc && pkill holochain && pkill lair-keystore
-
       - name: Smoke test - single_write_many_read
         run: |
           # Start a sandbox conductor and run it in the background
@@ -104,19 +98,6 @@ jobs:
 
           pkill hc && pkill holochain && pkill lair-keystore
 
-      - name: Smoke test - single_write_many_read with CHC enabled
-        run: |
-          # start the CHC reference implementation service
-          nix develop .#ci -c bash -c "hc-chc-service --port 8181 &"
-
-          # Start a sandbox conductor and run it in the background
-          nix develop .#ci -c bash -c "hc s clean && echo "1234" | hc s --piped create --chc-url http://localhost:8181 && echo "1234" | hc s --piped -f 8888 run &"
-
-          # Run the scenario for 5 seconds
-          RUST_LOG=info nix run .#single_write_many_read -- --connection-string ws://localhost:8888 --duration 5 --no-progress
-
-          pkill hc && pkill holochain && pkill lair-keystore
-
       - name: Smoke test - dht_sync_lag
         run: |
           # Start a sandbox conductor and run it in the background
@@ -156,7 +137,7 @@ jobs:
           nix develop .#ci -c bash -c "hc s clean && echo "1234" | hc s --piped create && echo "1234" | hc s --piped -f 8888 run &"
 
           # Run the scenario for 5 seconds
-          RUST_LOG=info nix run .#first_call -- --connection-string ws://localhost:8888 --agents 1 --behaviour local:1 --duration 5 --no-progress
+          RUST_LOG=info nix run .#first_call -- --connection-string ws://localhost:8888 --duration 5 --no-progress
 
           pkill hc && pkill holochain && pkill lair-keystore
 
@@ -264,8 +245,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#trycp_write_validated -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -301,8 +282,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_call_rate -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_call_rate -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -336,8 +317,8 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#two_party_countersigning -- --targets targets-ci.yaml --behaviour initiate:1 --behaviour participate:1 --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#two_party_countersigning -- --targets targets-ci.yaml --behaviour initiate:1 --behaviour participate:1 --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -371,8 +352,25 @@ jobs:
           # Start a TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
 
-          # Run the scenario for 10 seconds
-          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#validation_receipts -- --targets targets-ci.yaml --instances-per-target 2 --duration 10 --no-progress
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#validation_receipts -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
+
+          # Stop the TryCP instance
+          nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
+          # Stop local network services
+          pkill hc-run-local
+
+      - name: Smoke test - remote_signals
+        run: |
+          set -x
+
+          # Start local network services
+          nix develop .#ci -c bash -c "hc-run-local-services --bootstrap-port 4422 --signal-port 4423 &"
+          # Start a TryCP instance
+          nix develop .#ci -c bash -c "source ./scripts/trycp.sh && start_trycp &"
+
+          # Run the scenario for 30 seconds
+          RUST_LOG=warn CONDUCTOR_CONFIG="CI" MIN_PEERS=2 nix run .#remote_signals -- --targets targets-ci.yaml --instances-per-target 2 --duration 30 --no-progress
 
           # Stop the TryCP instance
           nix develop .#ci -c bash -c "source ./scripts/trycp.sh && stop_trycp"
@@ -412,7 +410,7 @@ jobs:
           scenario_names | xargs -I % nix bundle --option sandbox false .#packages.x86_64-linux.%
 
           mkdir -p dist
-          scenario_names | xargs -I % cp % dist/
+          scenario_names | xargs -I % cp %-arx dist/%
 
       - name: Archive scenario bundles
         if: runner.os == 'Linux'

diff --git a/.gitignore b/.gitignore
@@ -30,3 +30,5 @@ logs/**/*.log
 *.key
 /data
 scenarios/*/logs
+run_summary.jsonl
+summariser-report-*.json
diff --git a/.yamlfmt.yaml b/.yamlfmt.yaml
@@ -0,0 +1,5 @@
+formatter:
+  type: basic
+  retain_line_breaks_single: true
+  trim_trailing_whitespace: true
+  eof_newline: true
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Check in the `happ_builder` whether `hc` and `cargo` are available. This is used by the scenario build script to skip
   building happs if build tools are not available. This allows the project to be loaded in an environment where the
   tools aren't available.
+- A new tool for summarising scenario outcomes. This is called the `summariser` which is possibly a working title! The 
+  tool is specific to the scenarios in this project but does have some re-usable pieces. It remains to be decided whether
+  we will separate those parts out and publish them as a crate. For now, this is private to the project.
+- `run_with_required_agents` function for TryCP scenarios that fails if the number of agents that completed the scenario
+  is less than the passed `min_required_agents`. Can be overridden with the `MIN_REQUIRED_AGENTS` environment variable.
 
 ### Changed
 - Updated to new Holochain client version 0.5.0-alpha.4 which allowed `&mut self` to be replaced with `&self` in admin
@@ -25,10 +30,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   broadcast channel that it uses internally. Shutdown failures used to panic but it a `ShutdownHandle` happens to not
   have any subscribers then that should not be considered a fatal error. It will now log a warning instead.
 - Metrics now automatically include `run_id` and `scenario_name` tags.
+- Update `trycp_client` and `trycp_api` dependencies to `v0.17.0-dev.6`. [#117](https://github.com/holochain/wind-tunnel/pull/117)
+- When making zome calls with the TryCP client bindings, the `agent` is now reported on the metrics, taken from the target
+  cell_id for the call. For the wrapped `holochain_client`, this is only done when the call target is `CellId`. Or in 
+  other words, the `agent` is not reported when calling a clone cell.
+- All metrics are now reported in seconds, as an `f64`. There were some types still using milliseconds which made reporting
+  across scenarios more complex.
+- Increased TryCP test scenario duration to 30s in CI [Test Workflow](.github/workflows/test.yaml).
 
 ### Deprecated
 ### Removed
 ### Fixed
+- Run the TryCP scenarios in the [Performance Workflow](.github/workflows/performance.yaml) on the Holo Ports defined in [targets.yaml](targets.yaml). [#117](https://github.com/holochain/wind-tunnel/pull/117)
+
 ### Security
 
 ## [0.2.0-alpha.2] - 2024-05-25