Update metrics (#525)

- Adds additional metrics from utils package
- Migrate docs to separate markdown file

README.md

@@ -97,11 +97,7 @@ For testing purposes, chainbridge provides 5 test keys. The can be used with `--
 
 ## Metrics
 
-Basic metrics and a health status check can be enabled with the `--metrics` flag (default port `8001`, use `--metricsPort` to specify).
-
-The endpoint `/health` will return the current block height and a timestamp of when it was processed. If the timestamp is at least 120 seconds old an error will be returned.
-
-Prometheus metrics are served on `/metrics`.
+See [metrics.md](/docs/metrics.md).
 
 # Chain Implementations
 

chains/ethereum/listener.go

@@ -114,6 +114,10 @@ func (l *listener) pollBlocks() error {
 				continue
 			}
 
+			if l.metrics != nil {
+				l.metrics.LatestKnownBlock.Set(float64(latestBlock.Int64()))
+			}
+
 			// Sleep if the difference is less than BlockDelay; (latest - current) < BlockDelay
 			if big.NewInt(0).Sub(latestBlock, currentBlock).Cmp(BlockDelay) == -1 {
 				l.log.Debug("Block not ready, will retry", "target", currentBlock, "latest", latestBlock)
@@ -135,14 +139,17 @@ func (l *listener) pollBlocks() error {
 				l.log.Error("Failed to write latest block to blockstore", "block", currentBlock, "err", err)
 			}
 
-			// Goto next block and reset retry counter
-			currentBlock.Add(currentBlock, big.NewInt(1))
-			l.latestBlock.Height = big.NewInt(0).Set(latestBlock)
-			l.latestBlock.LastUpdated = time.Now()
-			retry = BlockRetryLimit
 			if l.metrics != nil {
 				l.metrics.BlocksProcessed.Inc()
+				l.metrics.LatestProcessedBlock.Set(float64(latestBlock.Int64()))
 			}
+
+			l.latestBlock.Height = big.NewInt(0).Set(latestBlock)
+			l.latestBlock.LastUpdated = time.Now()
+
+			// Goto next block and reset retry counter
+			currentBlock.Add(currentBlock, big.NewInt(1))
+			retry = BlockRetryLimit
 		}
 	}
 }

chains/substrate/listener.go

@@ -131,6 +131,10 @@ func (l *listener) pollBlocks() error {
 				continue
 			}
 
+			if l.metrics != nil {
+				l.metrics.LatestKnownBlock.Set(float64(finalizedHeader.Number))
+			}
+
 			// Sleep if the block we want comes after the most recently finalized block
 			if currentBlock > uint64(finalizedHeader.Number) {
 				l.log.Trace("Block not yet finalized", "target", currentBlock, "latest", finalizedHeader.Number)
@@ -163,13 +167,15 @@ func (l *listener) pollBlocks() error {
 				l.log.Error("Failed to write to blockstore", "err", err)
 			}
 
+			if l.metrics != nil {
+				l.metrics.BlocksProcessed.Inc()
+				l.metrics.LatestProcessedBlock.Set(float64(currentBlock))
+			}
+
 			currentBlock++
 			l.latestBlock.Height = big.NewInt(0).SetUint64(currentBlock)
 			l.latestBlock.LastUpdated = time.Now()
 			retry = BlockRetryLimit
-			if l.metrics != nil {
-				l.metrics.BlocksProcessed.Inc()
-			}
 		}
 	}
 }

docs/metrics.md

@@ -0,0 +1,31 @@
+# Metrics
+
+Basic metrics and a health status check can be enabled with the `--metrics` flag (default port `8001`, use `--metricsPort` to specify).
+
+## Prometheus
+Prometheus metrics are served on `/metrics`. For each chain that exists, this provides:
+- `<chain>_blocks_processed`: the number of blocks processed by the chains listener.
+- `<chain>_latest_processed_block`: most recent block that has been processed by the listener.
+- `<chain>_latest_known_block`: most recent block that exists on the chain.
+- `<chain>_votes_submitted`: number of votes submitted by the relayer.
+
+## Health Check
+The endpoint `/health` will return the current known block height, and a timestamp of when it was first seen for every chain:
+ ```json
+{
+  "chains": [
+    {
+      "chainId": "Number",
+      "height": "Number",
+      "lastUpdated": "Date"
+    }
+  ]
+} 
+```
+
+ If the timestamp is at least 120 seconds old an error will be returned instead:
+```json
+{
+  "error": "String"
+}
+```

go.mod

@@ -4,7 +4,7 @@ go 1.13
 
 require (
 	github.com/ChainSafe/chainbridge-substrate-events v0.0.0-20200715141113-87198532025e
-	github.com/ChainSafe/chainbridge-utils v1.0.2
+	github.com/ChainSafe/chainbridge-utils v1.0.3
 	github.com/ChainSafe/log15 v1.0.0
 	github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 // indirect
 	github.com/btcsuite/btcd v0.20.1-beta // indirect