Skip to content

Commit

Permalink
Metrics capture
Browse files Browse the repository at this point in the history
  • Loading branch information
ThetaSinner committed Mar 7, 2024
1 parent b28ad06 commit d25caa2
Show file tree
Hide file tree
Showing 16 changed files with 13,396 additions and 15 deletions.
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,25 @@ This will do a one-time setup for InfluxDB and also configure your shell environ
You can now navigate to the InfluxDB [dashboard](http://localhost:8087) and log in with `windtunnel`/`windtunnel`. The variables and dashboards you need will already be set up,
so you can now run your scenario and the metrics will be pushed to InfluxDB.

#### Running Telegraf

This is used for pushing system metrics to InfluxDB. This is not required locally but if you would like to run it then you can do so from inside the Nix shell:

```bash
use_influx
start_telegraf
```

#### Running Holochain

For a zero-config and quick way to run Holochain, you can use the following command:

```bash
hc s clean && echo "1234" | hc s --piped create && echo "1234" | hc s --piped -f 8888 run
```

For more advanced scenarios or for distributed tests, this is not appropriate!

#### Running scenarios

Each scenario is expected to provide a README.md with at least:
Expand Down
3 changes: 3 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,16 @@
packages = [
pkgs.influxdb2-cli
pkgs.influxdb2-server
# TODO https://docs.influxdata.com/telegraf/v1/install/#ntp
pkgs.telegraf
pkgs.yq
pkgs.httpie
pkgs.shellcheck
];

shellHook = ''
source ./scripts/influx.sh
source ./scripts/telegraf.sh
'';
};
};
Expand Down
46 changes: 41 additions & 5 deletions framework/instruments/src/report/metrics_report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@ use crate::report::ReportCollector;
use crate::OperationRecord;
use anyhow::Context;
use influxdb::{Client, InfluxDbWriteable, Timestamp, WriteQuery};
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::SystemTime;
use tokio::runtime::Runtime;
use tokio::select;
use tokio::sync::mpsc::UnboundedSender;
use wind_tunnel_core::prelude::DelegatedShutdownListener;

pub struct MetricsReportCollector {
pub writer: UnboundedSender<influxdb::WriteQuery>,
pub writer: UnboundedSender<WriteQuery>,
pub flush_complete: Arc<AtomicBool>,
}

impl MetricsReportCollector {
Expand All @@ -29,9 +32,14 @@ impl MetricsReportCollector {
"Cannot configure metrics reporter without environment variable `INFLUX_TOKEN`",
)?);

let writer = start_metrics_write_task(runtime, shutdown_listener, client);
let flush_complete = Arc::new(AtomicBool::new(false));
let writer =
start_metrics_write_task(runtime, shutdown_listener, client, flush_complete.clone());

Ok(Self { writer })
Ok(Self {
writer,
flush_complete,
})
}
}

Expand Down Expand Up @@ -63,14 +71,35 @@ impl ReportCollector for MetricsReportCollector {
}

fn finalize(&self) {
// Not required for metrics currently.
let wait_started = std::time::Instant::now();
let mut notify_timer = std::time::Instant::now();
while !self
.flush_complete
.load(std::sync::atomic::Ordering::Relaxed)
{
if notify_timer.elapsed().as_secs() > 10 {
log::warn!(
"Still waiting for metrics to flush after {} seconds.",
wait_started.elapsed().as_secs()
);
notify_timer = std::time::Instant::now();
}

std::thread::sleep(std::time::Duration::from_millis(100));
}

log::debug!(
"Metrics flushed after {} seconds",
wait_started.elapsed().as_secs()
);
}
}

fn start_metrics_write_task(
runtime: &Runtime,
mut shutdown_listener: DelegatedShutdownListener,
client: Client,
flush_complete: Arc<AtomicBool>,
) -> UnboundedSender<WriteQuery> {
let (writer, mut receiver) = tokio::sync::mpsc::unbounded_channel();
runtime.spawn(async move {
Expand All @@ -92,7 +121,7 @@ fn start_metrics_write_task(
}
}

log::trace!("Draining any remaining metrics before shutting down...");
log::debug!("Draining any remaining metrics before shutting down...");
let mut drain_count = 0;

// Drain remaining metrics before shutting down
Expand All @@ -101,9 +130,16 @@ fn start_metrics_write_task(
log::warn!("Failed to send metric to InfluxDB: {}", e);
}
drain_count += 1;

if drain_count % 1000 == 0 {
log::debug!("Drained {} remaining metrics", drain_count);
}
}

log::debug!("Drained {} remaining metrics", drain_count);

flush_complete.store(true, std::sync::atomic::Ordering::Relaxed);
});

writer
}
6 changes: 6 additions & 0 deletions influx/config.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# All options here https://docs.influxdata.com/influxdb/v2/reference/config-options/

e2e-testing = true # Allows clearing stores between tests.
http-bind-address = ":8087"
reporting-disabled = true # Don't send telemetry back to Influx
storage-validate-keys = true
6 changes: 0 additions & 6 deletions influx/config.yaml

This file was deleted.

1 change: 1 addition & 0 deletions influx/templates/dashboards/host.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"apiVersion":"influxdata.com/v2alpha1","kind":"Dashboard","metadata":{"name":"dangling-ellis-d00001"},"spec":{"charts":[{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"bbgfGLDcmvRyjmdEPN_br","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"4XLcdPRXBGxrCscxDmFo6","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"ebNrVA4Aaqs8oQ-gGuNx9","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"UAzJW5wnmATisyBlIYM4a","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"bKiBk-i5EVIrvDABhvW1B","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"hqlCB4If0HDV4rqhCmwRu","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"HK2fpW80wLddL7zfODftf","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"v1uO3eQ0KOkfC3nkaLEHa","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"CPU usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"host\"] == v.Host)\n |> map(fn: (r) => ({ r with _value: 100.0 - r._value }))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":12,"widthRatio":1,"xCol":"_time","yCol":"_value","yTickStep":10,"yTotalTicks":11},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"T04f3vqJz3Mjqm6GY8UmD","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"zszcrhtv_2Fz7fPb6vd4g","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"5yZNdcVUb86LhFe-rm2jb","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"6SGv1m0rFrKZeGxs3lcbP","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"DNyPObWEpbOdHN8glzsz2","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"CMBvuj6rYCW1kFRVtlS7S","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"r2Sm6VWRNBr3c9uomqOUL","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"L1S6FXSzV1Z6PPkK9CWy-","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Memory usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"host\"] == v.Host)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":12,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"Wj5FBtPBcGEI2vmoUdObP","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"PSSgpSdOlzvYZ8s-c125B","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"ncp2frG8uPAhxSMi-O46h","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"__8btrVO7E6GJVcSWx8yb","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"c-1IiIpiCY0lFkIKto7Si","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"tpwua7WAQ5Q_gaLPp9jZS","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"fGmIYLRM9iI_3jvBJUMhz","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"1yGI-wkNyTXRsdUnvDzir","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Read","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"diskio\")\n |> filter(fn: (r) => r[\"_field\"] == \"read_bytes\")\n |> filter(fn: (r) => r[\"host\"] == v.Host)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":8},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"Wj5FBtPBcGEI2vmoUdObP","name":"Color Blind Friendly - Light","type":"scale","hex":"#FFFFFF"},{"id":"PSSgpSdOlzvYZ8s-c125B","name":"Color Blind Friendly - Light","type":"scale","hex":"#E69F00"},{"id":"ncp2frG8uPAhxSMi-O46h","name":"Color Blind Friendly - Light","type":"scale","hex":"#56B4E9"},{"id":"__8btrVO7E6GJVcSWx8yb","name":"Color Blind Friendly - Light","type":"scale","hex":"#009E73"},{"id":"c-1IiIpiCY0lFkIKto7Si","name":"Color Blind Friendly - Light","type":"scale","hex":"#F0E442"},{"id":"tpwua7WAQ5Q_gaLPp9jZS","name":"Color Blind Friendly - Light","type":"scale","hex":"#0072B2"},{"id":"fGmIYLRM9iI_3jvBJUMhz","name":"Color Blind Friendly - Light","type":"scale","hex":"#D55E00"},{"id":"1yGI-wkNyTXRsdUnvDzir","name":"Color Blind Friendly - Light","type":"scale","hex":"#CC79A7"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Write","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"windtunnel\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"diskio\")\n |> filter(fn: (r) => r[\"_field\"] == \"write_bytes\")\n |> filter(fn: (r) => r[\"host\"] == v.Host)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":8}],"name":"Host"}}]
Loading

0 comments on commit d25caa2

Please sign in to comment.