From 9a3efdee836576a7116a159742fc5338fd0749e2 Mon Sep 17 00:00:00 2001 From: Liubov Dmitrieva Date: Thu, 17 Oct 2024 13:31:37 -0700 Subject: [PATCH] add queries count metric per backend Summary: add queries count metric per backend Reviewed By: lmvasquezg Differential Revision: D64405723 fbshipit-source-id: 9cc7f3b4beb3ee0fb5cf693ce0906b594f1bb10f --- .../lib/cas-client/re-cas-common/src/lib.rs | 8 ++--- .../revisionstore/src/scmstore/file/fetch.rs | 12 +++++++ .../lib/revisionstore/src/scmstore/metrics.rs | 32 +++++++++++++++++++ .../revisionstore/src/scmstore/tree/fetch.rs | 12 +++++++ eden/scm/lib/types/src/cas.rs | 8 +++++ 5 files changed, 68 insertions(+), 4 deletions(-) diff --git a/eden/scm/lib/cas-client/re-cas-common/src/lib.rs b/eden/scm/lib/cas-client/re-cas-common/src/lib.rs index 505ad38cfc90c..a7d941da33c1d 100644 --- a/eden/scm/lib/cas-client/re-cas-common/src/lib.rs +++ b/eden/scm/lib/cas-client/re-cas-common/src/lib.rs @@ -78,10 +78,10 @@ macro_rules! re_client { let mut stats = CasFetchedStats::default(); for (backend, dstats) in download_stats_map { match backend { - TStorageBackendType::ZDB => stats.total_bytes_zdb += dstats.bytes as u64, - TStorageBackendType::ZGATEWAY => stats.total_bytes_zgw += dstats.bytes as u64, - TStorageBackendType::MANIFOLD => stats.total_bytes_manifold += dstats.bytes as u64, - TStorageBackendType::HEDWIG => stats.total_bytes_hedwig += dstats.bytes as u64, + TStorageBackendType::ZDB => {stats.total_bytes_zdb += dstats.bytes as u64; stats.queries_zdb += dstats.queries_count as u64} + TStorageBackendType::ZGATEWAY => {stats.total_bytes_zgw += dstats.bytes as u64; stats.queries_zgw += dstats.queries_count as u64} + TStorageBackendType::MANIFOLD => {stats.total_bytes_manifold += dstats.bytes as u64; stats.queries_manifold += dstats.queries_count as u64} + TStorageBackendType::HEDWIG => {stats.total_bytes_hedwig += dstats.bytes as u64; stats.queries_hedwig += dstats.queries_count as u64 } _ => {} } } diff --git a/eden/scm/lib/revisionstore/src/scmstore/file/fetch.rs b/eden/scm/lib/revisionstore/src/scmstore/file/fetch.rs index 931efc862c2aa..133aa3a088b32 100644 --- a/eden/scm/lib/revisionstore/src/scmstore/file/fetch.rs +++ b/eden/scm/lib/revisionstore/src/scmstore/file/fetch.rs @@ -835,6 +835,18 @@ impl FetchState { self.metrics .cas_backend .hedwig_bytes(total_stats.total_bytes_hedwig); + self.metrics + .cas_backend + .zdb_queries(total_stats.queries_zdb); + self.metrics + .cas_backend + .zgw_queries(total_stats.queries_zgw); + self.metrics + .cas_backend + .manifold_queries(total_stats.queries_manifold); + self.metrics + .cas_backend + .hedwig_queries(total_stats.queries_hedwig); } pub(crate) fn fetch_lfs_remote( diff --git a/eden/scm/lib/revisionstore/src/scmstore/metrics.rs b/eden/scm/lib/revisionstore/src/scmstore/metrics.rs index 95f78cfeae27c..373ae2abfca81 100644 --- a/eden/scm/lib/revisionstore/src/scmstore/metrics.rs +++ b/eden/scm/lib/revisionstore/src/scmstore/metrics.rs @@ -227,35 +227,63 @@ pub struct CasBackendMetrics { /// Total number of bytes fetched from the CAS ZippyDb backend zdb_bytes: u64, + /// Total number of queries to the CAS ZippyDb backend + zdb_queries: u64, + /// Total number of bytes fetched from the CAS ZGW backend zgw_bytes: u64, + /// Total number of queries to the CAS ZGW backend + zgw_queries: u64, + /// Total number of bytes fetched from the CAS Manifold backend manifold_bytes: u64, + /// Total number of queries to the CAS Manifold backend + manifold_queries: u64, + /// Total number of bytes fetched from the CAS Hedwig backend hedwig_bytes: u64, + + /// Total number of queries to the CAS Hedwig backend + hedwig_queries: u64, } impl CasBackendMetrics { pub(crate) fn zdb_bytes(&mut self, bytes: u64) { self.zdb_bytes += bytes; } + pub(crate) fn zdb_queries(&mut self, queries: u64) { + self.zdb_queries += queries; + } pub(crate) fn zgw_bytes(&mut self, bytes: u64) { self.zgw_bytes += bytes; } + pub(crate) fn zgw_queries(&mut self, queries: u64) { + self.zgw_queries += queries; + } pub(crate) fn manifold_bytes(&mut self, bytes: u64) { self.manifold_bytes += bytes; } + pub(crate) fn manifold_queries(&mut self, queries: u64) { + self.manifold_queries += queries; + } pub(crate) fn hedwig_bytes(&mut self, bytes: u64) { self.hedwig_bytes += bytes; } + pub(crate) fn hedwig_queries(&mut self, queries: u64) { + self.hedwig_queries += queries; + } pub(crate) fn metrics(&self) -> impl Iterator { [ ("zdb.bytes", self.zdb_bytes as usize), ("zgw.bytes", self.zgw_bytes as usize), ("manifold.bytes", self.manifold_bytes as usize), ("hedwig.bytes", self.hedwig_bytes as usize), + ("zdb.queries", self.zdb_queries as usize), + ("zgw.queries", self.zgw_queries as usize), + ("manifold.queries", self.manifold_queries as usize), + ("hedwig.queries", self.hedwig_queries as usize), ] .into_iter() .filter(|&(_, v)| v != 0) @@ -268,5 +296,9 @@ impl AddAssign for CasBackendMetrics { self.zgw_bytes += rhs.zgw_bytes; self.manifold_bytes += rhs.manifold_bytes; self.hedwig_bytes += rhs.hedwig_bytes; + self.zdb_queries += rhs.zdb_queries; + self.zgw_queries += rhs.zgw_queries; + self.manifold_queries += rhs.manifold_queries; + self.hedwig_queries += rhs.hedwig_queries; } } diff --git a/eden/scm/lib/revisionstore/src/scmstore/tree/fetch.rs b/eden/scm/lib/revisionstore/src/scmstore/tree/fetch.rs index 835d769bf70fd..f98b2626c7e58 100644 --- a/eden/scm/lib/revisionstore/src/scmstore/tree/fetch.rs +++ b/eden/scm/lib/revisionstore/src/scmstore/tree/fetch.rs @@ -308,6 +308,18 @@ impl FetchState { self.metrics .cas_backend .hedwig_bytes(total_stats.total_bytes_hedwig); + self.metrics + .cas_backend + .zdb_queries(total_stats.queries_zdb); + self.metrics + .cas_backend + .zgw_queries(total_stats.queries_zgw); + self.metrics + .cas_backend + .manifold_queries(total_stats.queries_manifold); + self.metrics + .cas_backend + .hedwig_queries(total_stats.queries_hedwig); } } diff --git a/eden/scm/lib/types/src/cas.rs b/eden/scm/lib/types/src/cas.rs index a574a25c756cb..f2aea0c7310dd 100644 --- a/eden/scm/lib/types/src/cas.rs +++ b/eden/scm/lib/types/src/cas.rs @@ -39,6 +39,10 @@ pub struct CasFetchedStats { pub total_bytes_zgw: u64, pub total_bytes_manifold: u64, pub total_bytes_hedwig: u64, + pub queries_zdb: u64, + pub queries_zgw: u64, + pub queries_manifold: u64, + pub queries_hedwig: u64, } impl CasFetchedStats { @@ -47,5 +51,9 @@ impl CasFetchedStats { self.total_bytes_zgw += other.total_bytes_zgw; self.total_bytes_manifold += other.total_bytes_manifold; self.total_bytes_hedwig += other.total_bytes_hedwig; + self.queries_zdb += other.queries_zdb; + self.queries_zgw += other.queries_zgw; + self.queries_manifold += other.queries_manifold; + self.queries_hedwig += other.queries_hedwig; } }