From 42991518f7e7dc9f99cf4b1198890cadbf511ec7 Mon Sep 17 00:00:00 2001 From: Yuchan Lee Date: Mon, 23 Sep 2024 23:28:29 +0900 Subject: [PATCH] Add new metric: file size of documents per vault / tag (#15) --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 14 +- examples/grafana-dashboard/dashboard.json | 125 +++++++++++++++++- .../grafana-dashboard/docker-compose.yaml | 2 + src/main.rs | 14 +- src/metrics_collector/build_info.rs | 2 +- src/metrics_collector/document.rs | 97 +++++++++++++- tests/fixtures/document.json | 53 +++++++- 9 files changed, 287 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a271bc9..1d6afb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -758,7 +758,7 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "onepassword-exporter" -version = "0.2.1" +version = "0.3.0" dependencies = [ "anyhow", "bytes", diff --git a/Cargo.toml b/Cargo.toml index a39b2d3..e7530dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "onepassword-exporter" -version = "0.2.1" +version = "0.3.0" edition = "2021" [dependencies] diff --git a/README.md b/README.md index d14b65b..284c02d 100644 --- a/README.md +++ b/README.md @@ -81,16 +81,22 @@ Here is full example of available metrics, with all metrics enabled: op_account_current{created_at="2023-03-19T05:06:27Z",domain="my",id="??????????????????????????",name="**********",state="ACTIVE",type="FAMILY"} 1 # HELP op_document_count_per_tag Number of documents per tag. # TYPE op_document_count_per_tag gauge -op_document_count_per_tag{tag="test"} 1 +op_document_count_per_tag{tag="test"} 4 # HELP op_document_count_per_vault Number of documents per vault. # TYPE op_document_count_per_vault gauge -op_document_count_per_vault{vault="36vhq4xz3r6hnemzadk33evi4a"} 1 +op_document_count_per_vault{vault="36vhq4xz3r6hnemzadk33evi4a"} 4 # HELP op_document_count_total Total number of documents. # TYPE op_document_count_total gauge -op_document_count_total 1 +op_document_count_total 4 +# HELP op_document_file_size_per_tag_bytes Size of file in documents per tag, in bytes. +# TYPE op_document_file_size_per_tag_bytes gauge +op_document_file_size_per_tag_bytes{tag="test"} 10494986 +# HELP op_document_file_size_per_vault_bytes Size of file in documents per vault, in bytes. +# TYPE op_document_file_size_per_vault_bytes gauge +op_document_file_size_per_vault_bytes{vault="36vhq4xz3r6hnemzadk33evi4a"} 10494986 # HELP op_exporter_buildinfo Build information of this exporter. # TYPE op_exporter_buildinfo gauge -op_exporter_buildinfo{version="0.2.0"} 1 +op_exporter_buildinfo{version="0.3.0"} 1 # HELP op_group_count_total Total number of groups. # TYPE op_group_count_total gauge op_group_count_total 4 diff --git a/examples/grafana-dashboard/dashboard.json b/examples/grafana-dashboard/dashboard.json index 1a165e0..c8e69c5 100644 --- a/examples/grafana-dashboard/dashboard.json +++ b/examples/grafana-dashboard/dashboard.json @@ -1,4 +1,44 @@ { + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -19,6 +59,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": null, "links": [], "panels": [ { @@ -1793,7 +1834,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "op_document_count_per_vault", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1837,7 +1878,28 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "op_document_file_size_per_vault_bytes" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "op_document_file_size_per_vault_bytes" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] }, "gridPos": { "h": 10, @@ -1875,6 +1937,23 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_document_file_size_per_vault_bytes", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{vault}}", + "range": true, + "refId": "B", + "useBackend": false } ], "title": "Documents per Vault", @@ -2332,7 +2411,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "op_document_count_per_tag", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -2376,7 +2455,28 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "op_document_file_size_per_tag_bytes" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "op_document_file_size_per_tag_bytes" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] }, "gridPos": { "h": 10, @@ -2414,6 +2514,23 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_document_file_size_per_tag_bytes", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{tag}}", + "range": true, + "refId": "B", + "useBackend": false } ], "title": "Documents per Tag", diff --git a/examples/grafana-dashboard/docker-compose.yaml b/examples/grafana-dashboard/docker-compose.yaml index 6290fa9..39e413a 100644 --- a/examples/grafana-dashboard/docker-compose.yaml +++ b/examples/grafana-dashboard/docker-compose.yaml @@ -27,6 +27,7 @@ services: ports: - ${GRAFANA_HOST:-127.0.0.1}:${GRAFANA_PORT:-3000}:3000 volumes: + - grafana-data:/var/lib/grafana - ./dashboard.json:/var/lib/grafana/dashboards/dashboard.json configs: - source: datasource.yaml @@ -87,4 +88,5 @@ configs: - onepassword-exporter:9999 volumes: + grafana-data: prometheus-data: diff --git a/src/main.rs b/src/main.rs index fa69be4..65c5952 100644 --- a/src/main.rs +++ b/src/main.rs @@ -106,16 +106,22 @@ mod tests { op_account_current{created_at="2023-03-19T05:06:27Z",domain="my",id="??????????????????????????",name="**********",state="ACTIVE",type="FAMILY"} 1 # HELP op_document_count_per_tag Number of documents per tag. # TYPE op_document_count_per_tag gauge -op_document_count_per_tag{tag="test"} 1 +op_document_count_per_tag{tag="test"} 4 # HELP op_document_count_per_vault Number of documents per vault. # TYPE op_document_count_per_vault gauge -op_document_count_per_vault{vault="36vhq4xz3r6hnemzadk33evi4a"} 1 +op_document_count_per_vault{vault="36vhq4xz3r6hnemzadk33evi4a"} 4 # HELP op_document_count_total Total number of documents. # TYPE op_document_count_total gauge -op_document_count_total 1 +op_document_count_total 4 +# HELP op_document_file_size_per_tag_bytes Size of file in documents per tag, in bytes. +# TYPE op_document_file_size_per_tag_bytes gauge +op_document_file_size_per_tag_bytes{tag="test"} 10494986 +# HELP op_document_file_size_per_vault_bytes Size of file in documents per vault, in bytes. +# TYPE op_document_file_size_per_vault_bytes gauge +op_document_file_size_per_vault_bytes{vault="36vhq4xz3r6hnemzadk33evi4a"} 10494986 # HELP op_exporter_buildinfo Build information of this exporter. # TYPE op_exporter_buildinfo gauge -op_exporter_buildinfo{version="0.2.0"} 1 +op_exporter_buildinfo{version="0.3.0"} 1 # HELP op_group_count_total Total number of groups. # TYPE op_group_count_total gauge op_group_count_total 4 diff --git a/src/metrics_collector/build_info.rs b/src/metrics_collector/build_info.rs index 7be37dc..0b0e200 100644 --- a/src/metrics_collector/build_info.rs +++ b/src/metrics_collector/build_info.rs @@ -36,7 +36,7 @@ mod tests { // Assert assert_eq!( OP_EXPORTER_BUILDINFO - .get_metric_with_label_values(&["0.2.0"])? + .get_metric_with_label_values(&["0.3.0"])? .get(), 1 ); diff --git a/src/metrics_collector/document.rs b/src/metrics_collector/document.rs index de08f77..2b2a35d 100644 --- a/src/metrics_collector/document.rs +++ b/src/metrics_collector/document.rs @@ -24,6 +24,18 @@ lazy_static! { &["tag"] ) .unwrap(); + static ref OP_DOCUMENT_FILE_SIZE_PER_VAULT: IntGaugeVec = register_int_gauge_vec!( + "op_document_file_size_per_vault_bytes", + "Size of file in documents per vault, in bytes.", + &["vault"], + ) + .unwrap(); + static ref OP_DOCUMENT_FILE_SIZE_PER_TAG: IntGaugeVec = register_int_gauge_vec!( + "op_document_file_size_per_tag_bytes", + "Size of file in documents per tag, in bytes.", + &["tag"], + ) + .unwrap(); } #[derive(Deserialize, Debug)] @@ -36,6 +48,8 @@ struct Document { #[allow(dead_code)] pub(crate) version: i32, pub(crate) vault: DocumentVault, + #[serde(rename = "overview.ainfo")] + pub(crate) overview_ainfo: Option, #[allow(dead_code)] pub(crate) last_edited_by: String, #[allow(dead_code)] @@ -68,13 +82,26 @@ impl OpMetricsCollector { // Gather metrics let mut count_per_vault = HashMap::new(); let mut count_per_tag = HashMap::new(); + let mut file_size_per_vault = HashMap::new(); + let mut file_size_per_tag = HashMap::new(); + for document in &documents { let vault_id = document.vault.id.clone(); - *count_per_vault.entry(vault_id).or_insert(0) += 1; - + let file_size = match &document.overview_ainfo { + Some(overview_ainfo) => match parse_file_size_bytes(overview_ainfo) { + Some(file_size) => file_size, + None => 0, + }, + None => 0, + }; let tags = document.tags.clone().unwrap_or_default(); + + *count_per_vault.entry(vault_id.clone()).or_insert(0) += 1; + *file_size_per_vault.entry(vault_id).or_insert(0) += file_size; + for tag in tags { - *count_per_tag.entry(tag).or_insert(0) += 1; + *count_per_tag.entry(tag.clone()).or_insert(0) += 1; + *file_size_per_tag.entry(tag).or_insert(0) += file_size; } } @@ -93,9 +120,55 @@ impl OpMetricsCollector { .with_label_values(&[tag]) .set(*count); }); + file_size_per_vault.iter().for_each(|(vault, size)| { + OP_DOCUMENT_FILE_SIZE_PER_VAULT + .with_label_values(&[vault]) + .set(*size); + }); + file_size_per_tag.iter().for_each(|(tag, size)| { + OP_DOCUMENT_FILE_SIZE_PER_TAG + .with_label_values(&[tag]) + .set(*size); + }); } } +/// Parse file size (e.g. `"10 bytes"`, `"1 KB"`) to bytes. +// * NOTE: Accurate file sizes not available in the output because 1Password truncate it when goes over KB. +fn parse_file_size_bytes(s: &String) -> Option { + // Split size and unit + let (size, unit) = match s.split_once(' ') { + Some((size, unit)) => (size, unit), + None => { + log::error!("Invalid file size format: {}", s); + return None; + } + }; + + // Parse size + let size = match size.parse::() { + Ok(size) => size, + Err(e) => { + log::error!("Failed to parse file size: {}", e); + return None; + } + }; + + // Convert unit to multiplier + let multiplier = match unit { + "bytes" => 1, + "KB" => 1024, + "MB" => 1024 * 1024, + "GB" => 1024 * 1024 * 1024, + _ => { + log::error!("Unknown unit: {}", unit); + return None; + } + }; + + Some(size * multiplier) +} + #[cfg(test)] mod tests { use anyhow::Result; @@ -114,19 +187,31 @@ mod tests { OP_DOCUMENT_COUNT_TOTAL .get_metric_with_label_values(&[])? .get(), - 1 + 4 ); assert_eq!( OP_DOCUMENT_COUNT_PER_VAULT .get_metric_with_label_values(&["36vhq4xz3r6hnemzadk33evi4a"])? .get(), - 1 + 4 ); assert_eq!( OP_DOCUMENT_COUNT_PER_TAG .get_metric_with_label_values(&["test"])? .get(), - 1 + 4 + ); + assert_eq!( + OP_DOCUMENT_FILE_SIZE_PER_VAULT + .get_metric_with_label_values(&["36vhq4xz3r6hnemzadk33evi4a"])? + .get(), + 10494986 + ); + assert_eq!( + OP_DOCUMENT_FILE_SIZE_PER_TAG + .get_metric_with_label_values(&["test"])? + .get(), + 10494986 ); Ok(()) diff --git a/tests/fixtures/document.json b/tests/fixtures/document.json index e40f6f2..a705d55 100644 --- a/tests/fixtures/document.json +++ b/tests/fixtures/document.json @@ -1,11 +1,58 @@ [ { - "id": "lg4ukcbfhd54tgb3ixzmv7lrcu", + "id": "vg5ia5rapo2rqixgg7tmmo47ri", + "title": "Tiny File", + "tags": [ + "test" + ], + "version": 12, + "vault": { + "id": "36vhq4xz3r6hnemzadk33evi4a", + "name": "" + }, + "overview.ainfo": "10 bytes", + "last_edited_by": "K3MAYGGYRZA2XN2AMQ5ADZJ6VI", + "created_at": "2024-09-23T11:17:12Z", + "updated_at": "2024-09-23T11:37:00Z" + }, + { + "id": "ybo3wgmxgq26glyaapd6l432su", "title": "Empty Document", "tags": [ "test" ], - "version": 3, + "version": 2, + "vault": { + "id": "36vhq4xz3r6hnemzadk33evi4a", + "name": "" + }, + "last_edited_by": "K3MAYGGYRZA2XN2AMQ5ADZJ6VI", + "created_at": "2024-09-23T11:35:28Z", + "updated_at": "2024-09-23T11:37:00Z" + }, + { + "id": "bgtzwwcy63c4dhjd2qdq4mytsm", + "title": "Moderate File", + "tags": [ + "test" + ], + "version": 5, + "vault": { + "id": "36vhq4xz3r6hnemzadk33evi4a", + "name": "" + }, + "overview.ainfo": "10 MB", + "last_edited_by": "K3MAYGGYRZA2XN2AMQ5ADZJ6VI", + "created_at": "2024-09-23T11:15:34Z", + "updated_at": "2024-09-23T11:37:00Z" + }, + { + "id": "lg4ukcbfhd54tgb3ixzmv7lrcu", + "title": "Regular File", + "tags": [ + "test" + ], + "version": 5, "vault": { "id": "36vhq4xz3r6hnemzadk33evi4a", "name": "" @@ -13,6 +60,6 @@ "overview.ainfo": "9 KB", "last_edited_by": "K3MAYGGYRZA2XN2AMQ5ADZJ6VI", "created_at": "2024-08-04T04:06:31Z", - "updated_at": "2024-09-03T12:57:17Z" + "updated_at": "2024-09-23T11:37:00Z" } ]