Add metrics for trie log pruner (hyperledger#7352)

---------

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Next release
 
+### Upcoming Breaking Changes
+- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version.
+- --Xbonsai-limit-trie-logs-enabled is deprecated, use --bonsai-limit-trie-logs-enabled instead
+- --Xbonsai-trie-logs-pruning-window-size is deprecated, use --bonsai-trie-logs-pruning-window-size instead
+- `besu storage x-trie-log` subcommand is deprecated, use `besu storage trie-log` instead
+
 ### Breaking Changes
 - Remove deprecated sync modes (X_SNAP and X_CHECKPOINT). Use SNAP and CHECKPOINT instead [#7309](https://github.com/hyperledger/besu/pull/7309)
 - Remove PKI-backed QBFT (deprecated in 24.5.1) Other forms of QBFT remain unchanged. [#7293](https://github.com/hyperledger/besu/pull/7293)
@@ -15,7 +21,8 @@
 - Implement gnark-crypto for eip-2537 [#7316](https://github.com/hyperledger/besu/pull/7316)
 - Improve blob size transaction selector [#7312](https://github.com/hyperledger/besu/pull/7312)
 - Added EIP-7702 [#7237](https://github.com/hyperledger/besu/pull/7237)
-- implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262)
+- Implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262)
+- Add trie log pruner metrics [#7352](https://github.com/hyperledger/besu/pull/7352)
 
 ### Bug fixes
 - Fix `eth_call` deserialization to correctly ignore unknown fields in the transaction object. [#7323](https://github.com/hyperledger/besu/pull/7323)

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -793,7 +793,8 @@ private TrieLogPruner createTrieLogPruner(
             scheduler::executeServiceTask,
             dataStorageConfiguration.getBonsaiMaxLayersToLoad(),
             dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize(),
-            isProofOfStake);
+            isProofOfStake,
+            metricsSystem);
     trieLogPruner.initialize();
 
     return trieLogPruner;

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java

@@ -19,6 +19,9 @@
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage;
+import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.plugin.services.MetricsSystem;
+import org.hyperledger.besu.plugin.services.metrics.Counter;
 import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
 
 import java.util.Comparator;
@@ -45,6 +48,9 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver {
   private final Consumer<Runnable> executeAsync;
   private final long numBlocksToRetain;
   private final boolean requireFinalizedBlock;
+  private final Counter addedToPruneQueueCounter;
+  private final Counter prunedFromQueueCounter;
+  private final Counter prunedOrphanCounter;
 
   private final Multimap<Long, Hash> trieLogBlocksAndForksByDescendingBlockNumber =
       TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder());
@@ -55,14 +61,26 @@ public TrieLogPruner(
       final Consumer<Runnable> executeAsync,
       final long numBlocksToRetain,
       final int pruningLimit,
-      final boolean requireFinalizedBlock) {
+      final boolean requireFinalizedBlock,
+      final MetricsSystem metricsSystem) {
     this.rootWorldStateStorage = rootWorldStateStorage;
     this.blockchain = blockchain;
     this.executeAsync = executeAsync;
     this.numBlocksToRetain = numBlocksToRetain;
     this.pruningLimit = pruningLimit;
     this.loadingLimit = pruningLimit; // same as pruningLimit for now
     this.requireFinalizedBlock = requireFinalizedBlock;
+    this.addedToPruneQueueCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.PRUNER,
+            "trie_log_added_to_prune_queue",
+            "trie log added to prune queue");
+    this.prunedFromQueueCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.PRUNER, "trie_log_pruned_from_queue", "trie log pruned from queue");
+    this.prunedOrphanCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.PRUNER, "trie_log_pruned_orphan", "trie log pruned orphan");
   }
 
   public int initialize() {
@@ -88,6 +106,7 @@ private int preloadQueue() {
               // prune orphaned blocks (sometimes created during block production)
               rootWorldStateStorage.pruneTrieLog(blockHash);
               orphansPruned.getAndIncrement();
+              prunedOrphanCounter.inc();
             }
           });
       LOG.atDebug().log("Pruned {} orphaned trie logs from database...", orphansPruned.intValue());
@@ -106,6 +125,7 @@ public synchronized void addToPruneQueue(final long blockNumber, final Hash bloc
         .addArgument(blockHash)
         .log();
     trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash);
+    addedToPruneQueueCounter.inc();
   }
 
   public synchronized int pruneFromQueue() {
@@ -155,6 +175,7 @@ public synchronized int pruneFromQueue() {
         });
 
     wasPruned.keySet().forEach(trieLogBlocksAndForksByDescendingBlockNumber::removeAll);
+    prunedFromQueueCounter.inc(wasPruned.size());
 
     LOG.atTrace()
         .setMessage("pruned {} trie logs for blocks {}")

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java

@@ -29,6 +29,7 @@
 import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
+import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 
 import java.util.Optional;
 import java.util.function.Consumer;
@@ -72,7 +73,8 @@ public void initialize_preloads_queue_and_prunes_orphaned_blocks() {
 
     // When
     TrieLogPruner trieLogPruner =
-        new TrieLogPruner(worldState, blockchain, executeAsync, 3, loadingLimit, false);
+        new TrieLogPruner(
+            worldState, blockchain, executeAsync, 3, loadingLimit, false, new NoOpMetricsSystem());
     trieLogPruner.initialize();
 
     // Then
@@ -92,7 +94,13 @@ public void trieLogs_pruned_in_reverse_order_within_pruning_window() {
     // requireFinalizedBlock = false means this is not a PoS chain
     TrieLogPruner trieLogPruner =
         new TrieLogPruner(
-            worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, false);
+            worldState,
+            blockchain,
+            executeAsync,
+            blocksToRetain,
+            pruningWindowSize,
+            false,
+            new NoOpMetricsSystem());
 
     trieLogPruner.addToPruneQueue(0, key(0)); // older block outside prune window
     trieLogPruner.addToPruneQueue(1, key(1)); // block inside the prune window
@@ -201,7 +209,13 @@ public void skip_pruning_when_finalized_block_required_but_not_present() {
     when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight);
     TrieLogPruner trieLogPruner =
         new TrieLogPruner(
-            worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true);
+            worldState,
+            blockchain,
+            executeAsync,
+            blocksToRetain,
+            pruningWindowSize,
+            true,
+            new NoOpMetricsSystem());
 
     trieLogPruner.addToPruneQueue(1, key(1));
     trieLogPruner.addToPruneQueue(2, key(2));
@@ -240,7 +254,8 @@ public void onTrieLogAdded_should_prune() {
     // Given
     final TriggerableConsumer triggerableConsumer = new TriggerableConsumer();
     TrieLogPruner trieLogPruner =
-        new TrieLogPruner(worldState, blockchain, triggerableConsumer, 0, 1, false);
+        new TrieLogPruner(
+            worldState, blockchain, triggerableConsumer, 0, 1, false, new NoOpMetricsSystem());
     assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0);
 
     final TrieLogLayer layer = new TrieLogLayer();
@@ -261,7 +276,8 @@ public void onTrieLogAdded_should_prune() {
   public void onTrieLogAdded_should_not_prune_when_no_blockNumber() {
     // Given
     TrieLogPruner trieLogPruner =
-        new TrieLogPruner(worldState, blockchain, executeAsync, 0, 1, false);
+        new TrieLogPruner(
+            worldState, blockchain, executeAsync, 0, 1, false, new NoOpMetricsSystem());
     assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0);
 
     final TrieLogLayer layer = new TrieLogLayer();
@@ -289,7 +305,13 @@ private TrieLogPruner setupPrunerAndFinalizedBlock(
     when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight);
     TrieLogPruner trieLogPruner =
         new TrieLogPruner(
-            worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true);
+            worldState,
+            blockchain,
+            executeAsync,
+            blocksToRetain,
+            pruningWindowSize,
+            true,
+            new NoOpMetricsSystem());
 
     trieLogPruner.addToPruneQueue(1, key(1));
     trieLogPruner.addToPruneQueue(2, key(2));