Track store metrics by DB column (#6041)

* Track store metrics by DB column

beacon_node/store/src/leveldb_store.rs

@@ -59,16 +59,13 @@ impl<E: EthSpec> LevelDB<E> {
     ) -> Result<(), Error> {
         let column_key = get_key_for_col(col, key);
 
-        metrics::inc_counter(&metrics::DISK_DB_WRITE_COUNT);
-        metrics::inc_counter_by(&metrics::DISK_DB_WRITE_BYTES, val.len() as u64);
-        let timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES);
+        metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[col]);
+        metrics::inc_counter_vec_by(&metrics::DISK_DB_WRITE_BYTES, &[col], val.len() as u64);
+        let _timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES);
 
         self.db
             .put(opts, BytesKey::from_vec(column_key), val)
             .map_err(Into::into)
-            .map(|()| {
-                metrics::stop_timer(timer);
-            })
     }
 
     pub fn keys_iter(&self) -> KeyIterator<BytesKey> {
@@ -94,15 +91,19 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
     fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
         let column_key = get_key_for_col(col, key);
 
-        metrics::inc_counter(&metrics::DISK_DB_READ_COUNT);
+        metrics::inc_counter_vec(&metrics::DISK_DB_READ_COUNT, &[col]);
         let timer = metrics::start_timer(&metrics::DISK_DB_READ_TIMES);
 
         self.db
             .get(self.read_options(), BytesKey::from_vec(column_key))
             .map_err(Into::into)
             .map(|opt| {
                 opt.map(|bytes| {
-                    metrics::inc_counter_by(&metrics::DISK_DB_READ_BYTES, bytes.len() as u64);
+                    metrics::inc_counter_vec_by(
+                        &metrics::DISK_DB_READ_BYTES,
+                        &[col],
+                        bytes.len() as u64,
+                    );
                     metrics::stop_timer(timer);
                     bytes
                 })
@@ -113,7 +114,7 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
     fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error> {
         let column_key = get_key_for_col(col, key);
 
-        metrics::inc_counter(&metrics::DISK_DB_EXISTS_COUNT);
+        metrics::inc_counter_vec(&metrics::DISK_DB_EXISTS_COUNT, &[col]);
 
         self.db
             .get(self.read_options(), BytesKey::from_vec(column_key))
@@ -125,7 +126,7 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
     fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
         let column_key = get_key_for_col(col, key);
 
-        metrics::inc_counter(&metrics::DISK_DB_DELETE_COUNT);
+        metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[col]);
 
         self.db
             .delete(self.write_options(), BytesKey::from_vec(column_key))
@@ -137,14 +138,28 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
         for op in ops_batch {
             match op {
                 KeyValueStoreOp::PutKeyValue(key, value) => {
+                    let col = get_col_from_key(&key).unwrap_or("unknown".to_owned());
+                    metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[&col]);
+                    metrics::inc_counter_vec_by(
+                        &metrics::DISK_DB_WRITE_BYTES,
+                        &[&col],
+                        value.len() as u64,
+                    );
+
                     leveldb_batch.put(BytesKey::from_vec(key), &value);
                 }
 
                 KeyValueStoreOp::DeleteKey(key) => {
+                    let col = get_col_from_key(&key).unwrap_or("unknown".to_owned());
+                    metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[&col]);
+
                     leveldb_batch.delete(BytesKey::from_vec(key));
                 }
             }
         }
+
+        let _timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES);
+
         self.db.write(self.write_options(), &leveldb_batch)?;
         Ok(())
     }

beacon_node/store/src/lib.rs

@@ -143,6 +143,13 @@ pub fn get_key_for_col(column: &str, key: &[u8]) -> Vec<u8> {
     result
 }
 
+pub fn get_col_from_key(key: &[u8]) -> Option<String> {
+    if key.len() < 3 {
+        return None;
+    }
+    String::from_utf8(key[0..3].to_vec()).ok()
+}
+
 #[must_use]
 #[derive(Clone)]
 pub enum KeyValueStoreOp {
@@ -412,4 +419,11 @@ mod tests {
 
         assert!(!store.exists::<StorableThing>(&key).unwrap());
     }
+
+    #[test]
+    fn test_get_col_from_key() {
+        let key = get_key_for_col(DBColumn::BeaconBlock.into(), &[1u8; 32]);
+        let col = get_col_from_key(&key).unwrap();
+        assert_eq!(col, "blk");
+    }
 }

beacon_node/store/src/metrics.rs

@@ -12,21 +12,25 @@ lazy_static! {
         try_create_int_gauge("store_disk_db_size", "Size of the hot on-disk database (bytes)");
     pub static ref FREEZER_DB_SIZE: Result<IntGauge> =
         try_create_int_gauge("store_freezer_db_size", "Size of the on-disk freezer database (bytes)");
-    pub static ref DISK_DB_WRITE_BYTES: Result<IntCounter> = try_create_int_counter(
+    pub static ref DISK_DB_WRITE_BYTES: Result<IntCounterVec> = try_create_int_counter_vec(
         "store_disk_db_write_bytes_total",
-        "Number of bytes attempted to be written to the hot on-disk DB"
+        "Number of bytes attempted to be written to the hot on-disk DB",
+        &["col"],
     );
-    pub static ref DISK_DB_READ_BYTES: Result<IntCounter> = try_create_int_counter(
+    pub static ref DISK_DB_READ_BYTES: Result<IntCounterVec> = try_create_int_counter_vec(
         "store_disk_db_read_bytes_total",
-        "Number of bytes read from the hot on-disk DB"
+        "Number of bytes read from the hot on-disk DB",
+        &["col"],
     );
-    pub static ref DISK_DB_READ_COUNT: Result<IntCounter> = try_create_int_counter(
+    pub static ref DISK_DB_READ_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
         "store_disk_db_read_count_total",
-        "Total number of reads to the hot on-disk DB"
+        "Total number of reads to the hot on-disk DB",
+        &["col"],
     );
-    pub static ref DISK_DB_WRITE_COUNT: Result<IntCounter> = try_create_int_counter(
+    pub static ref DISK_DB_WRITE_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
         "store_disk_db_write_count_total",
-        "Total number of writes to the hot on-disk DB"
+        "Total number of writes to the hot on-disk DB",
+        &["col"],
     );
     pub static ref DISK_DB_READ_TIMES: Result<Histogram> = try_create_histogram(
         "store_disk_db_read_seconds",
@@ -36,13 +40,15 @@ lazy_static! {
         "store_disk_db_write_seconds",
         "Time taken to write bytes to store."
     );
-    pub static ref DISK_DB_EXISTS_COUNT: Result<IntCounter> = try_create_int_counter(
+    pub static ref DISK_DB_EXISTS_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
         "store_disk_db_exists_count_total",
-        "Total number of checks if a key is in the hot on-disk DB"
+        "Total number of checks if a key is in the hot on-disk DB",
+        &["col"],
     );
-    pub static ref DISK_DB_DELETE_COUNT: Result<IntCounter> = try_create_int_counter(
+    pub static ref DISK_DB_DELETE_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
         "store_disk_db_delete_count_total",
-        "Total number of deletions from the hot on-disk DB"
+        "Total number of deletions from the hot on-disk DB",
+        &["col"],
     );
     /*
      * Beacon State