Metrics capture

README.md

@@ -242,6 +242,25 @@ This will do a one-time setup for InfluxDB and also configure your shell environ
 You can now navigate to the InfluxDB [dashboard](http://localhost:8087) and log in with `windtunnel`/`windtunnel`. The variables and dashboards you need will already be set up,
 so you can now run your scenario and the metrics will be pushed to InfluxDB.
 
+#### Running Telegraf
+
+This is used for pushing system metrics to InfluxDB. This is not required locally but if you would like to run it then you can do so from inside the Nix shell:
+
+```bash
+use_influx
+start_telegraf
+```
+
+#### Running Holochain
+
+For a zero-config and quick way to run Holochain, you can use the following command:
+
+```bash
+hc s clean && echo "1234" | hc s --piped create && echo "1234" | hc s --piped -f 8888 run
+```
+
+For more advanced scenarios or for distributed tests, this is not appropriate!
+
 #### Running scenarios
 
 Each scenario is expected to provide a README.md with at least:

flake.nix

@@ -26,13 +26,16 @@
                 packages = [
                     pkgs.influxdb2-cli
                     pkgs.influxdb2-server
+                    # TODO https://docs.influxdata.com/telegraf/v1/install/#ntp
+                    pkgs.telegraf
                     pkgs.yq
                     pkgs.httpie
                     pkgs.shellcheck
                 ];
 
                 shellHook = ''
                     source ./scripts/influx.sh
+                    source ./scripts/telegraf.sh
                 '';
             };
         };

framework/instruments/src/report/metrics_report.rs

@@ -2,14 +2,17 @@ use crate::report::ReportCollector;
 use crate::OperationRecord;
 use anyhow::Context;
 use influxdb::{Client, InfluxDbWriteable, Timestamp, WriteQuery};
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
 use std::time::SystemTime;
 use tokio::runtime::Runtime;
 use tokio::select;
 use tokio::sync::mpsc::UnboundedSender;
 use wind_tunnel_core::prelude::DelegatedShutdownListener;
 
 pub struct MetricsReportCollector {
-    pub writer: UnboundedSender<influxdb::WriteQuery>,
+    pub writer: UnboundedSender<WriteQuery>,
+    pub flush_complete: Arc<AtomicBool>,
 }
 
 impl MetricsReportCollector {
@@ -29,9 +32,14 @@ impl MetricsReportCollector {
             "Cannot configure metrics reporter without environment variable `INFLUX_TOKEN`",
         )?);
 
-        let writer = start_metrics_write_task(runtime, shutdown_listener, client);
+        let flush_complete = Arc::new(AtomicBool::new(false));
+        let writer =
+            start_metrics_write_task(runtime, shutdown_listener, client, flush_complete.clone());
 
-        Ok(Self { writer })
+        Ok(Self {
+            writer,
+            flush_complete,
+        })
     }
 }
 
@@ -63,14 +71,35 @@ impl ReportCollector for MetricsReportCollector {
     }
 
     fn finalize(&self) {
-        // Not required for metrics currently.
+        let wait_started = std::time::Instant::now();
+        let mut notify_timer = std::time::Instant::now();
+        while !self
+            .flush_complete
+            .load(std::sync::atomic::Ordering::Relaxed)
+        {
+            if notify_timer.elapsed().as_secs() > 10 {
+                log::warn!(
+                    "Still waiting for metrics to flush after {} seconds.",
+                    wait_started.elapsed().as_secs()
+                );
+                notify_timer = std::time::Instant::now();
+            }
+
+            std::thread::sleep(std::time::Duration::from_millis(100));
+        }
+
+        log::debug!(
+            "Metrics flushed after {} seconds",
+            wait_started.elapsed().as_secs()
+        );
     }
 }
 
 fn start_metrics_write_task(
     runtime: &Runtime,
     mut shutdown_listener: DelegatedShutdownListener,
     client: Client,
+    flush_complete: Arc<AtomicBool>,
 ) -> UnboundedSender<WriteQuery> {
     let (writer, mut receiver) = tokio::sync::mpsc::unbounded_channel();
     runtime.spawn(async move {
@@ -92,7 +121,7 @@ fn start_metrics_write_task(
             }
         }
 
-        log::trace!("Draining any remaining metrics before shutting down...");
+        log::debug!("Draining any remaining metrics before shutting down...");
         let mut drain_count = 0;
 
         // Drain remaining metrics before shutting down
@@ -101,9 +130,16 @@ fn start_metrics_write_task(
                 log::warn!("Failed to send metric to InfluxDB: {}", e);
             }
             drain_count += 1;
+
+            if drain_count % 1000 == 0 {
+                log::debug!("Drained {} remaining metrics", drain_count);
+            }
         }
 
         log::debug!("Drained {} remaining metrics", drain_count);
+
+        flush_complete.store(true, std::sync::atomic::Ordering::Relaxed);
     });
+
     writer
 }

influx/config.toml

@@ -0,0 +1,6 @@
+# All options here https://docs.influxdata.com/influxdb/v2/reference/config-options/
+
+e2e-testing = true # Allows clearing stores between tests.
+http-bind-address = ":8087"
+reporting-disabled = true # Don't send telemetry back to Influx
+storage-validate-keys = true

influx/templates/dashboards/host.json

@@ -0,0 +1 @@
+[{"apiVersion":"influxdata.com/v2alpha1","kind":"Dashboard","metadata":{"name":"dangling-ellis-d00001"},"spec":{"charts":[{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"bbgfGLDcmvRyjmdEPN_br","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"4XLcdPRXBGxrCscxDmFo6","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"ebNrVA4Aaqs8oQ-gGuNx9","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"UAzJW5wnmATisyBlIYM4a","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"bKiBk-i5EVIrvDABhvW1B","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"hqlCB4If0HDV4rqhCmwRu","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"HK2fpW80wLddL7zfODftf","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"v1uO3eQ0KOkfC3nkaLEHa","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"CPU usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n  |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n  |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n  |> filter(fn: (r) => r[\"host\"] == v.Host)\n  |> map(fn: (r) => ({ r with _value: 100.0 - r._value }))\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":12,"widthRatio":1,"xCol":"_time","yCol":"_value","yTickStep":10,"yTotalTicks":11},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"T04f3vqJz3Mjqm6GY8UmD","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"zszcrhtv_2Fz7fPb6vd4g","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"5yZNdcVUb86LhFe-rm2jb","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"6SGv1m0rFrKZeGxs3lcbP","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"DNyPObWEpbOdHN8glzsz2","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"CMBvuj6rYCW1kFRVtlS7S","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"r2Sm6VWRNBr3c9uomqOUL","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"L1S6FXSzV1Z6PPkK9CWy-","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Memory usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n  |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n  |> filter(fn: (r) => r[\"host\"] == v.Host)\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":12,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"Wj5FBtPBcGEI2vmoUdObP","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"PSSgpSdOlzvYZ8s-c125B","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"ncp2frG8uPAhxSMi-O46h","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"__8btrVO7E6GJVcSWx8yb","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"c-1IiIpiCY0lFkIKto7Si","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"tpwua7WAQ5Q_gaLPp9jZS","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"fGmIYLRM9iI_3jvBJUMhz","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"1yGI-wkNyTXRsdUnvDzir","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Read","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"diskio\")\n  |> filter(fn: (r) => r[\"_field\"] == \"read_bytes\")\n  |> filter(fn: (r) => r[\"host\"] == v.Host)\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":8},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"Wj5FBtPBcGEI2vmoUdObP","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"PSSgpSdOlzvYZ8s-c125B","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"ncp2frG8uPAhxSMi-O46h","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"__8btrVO7E6GJVcSWx8yb","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"c-1IiIpiCY0lFkIKto7Si","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"tpwua7WAQ5Q_gaLPp9jZS","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"fGmIYLRM9iI_3jvBJUMhz","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"1yGI-wkNyTXRsdUnvDzir","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Write","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"diskio\")\n  |> filter(fn: (r) => r[\"_field\"] == \"write_bytes\")\n  |> filter(fn: (r) => r[\"host\"] == v.Host)\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n  |> yield(name: \"mean\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":8}],"name":"Host"}}]