feat: track syncing status and fetch duties on resynced (#6995)

* feat: track syncing status and fetch duties on synced * Rename scheduling function to runOnResynced * Consider prev offline and syncing to trigger resynced event handlers * Add comment to error handler * Add note about el offline and sycning not considered * Align syncing status logs with existing node is syncing logs * Cleanup * Add ssz support to syncing status api * Align beacon node code to return proper types * Keep track of error in prev syncing status * Print slot in error log * Skip on first slot of epoch since tasks are already scheduled * Update api test data * Fix endpoint tests * await scheduled tasks, mostly relevant for testing * Add unit tests * Move beacon heath metric to syncing status tracker * Add beacon health panel to validator client dashboard * Formatting * Improve info called once assertion * Reset mocks after each test
ChainSafe · Sep 3, 2024 · a7c689f · a7c689f
1 parent 42780e9
commit a7c689f
Show file tree

Hide file tree

Showing 22 changed files with 693 additions and 110 deletions.
diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json
@@ -71,6 +71,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "msg / slot",
@@ -211,10 +212,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "name"
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -231,7 +234,6 @@
         }
       ],
       "title": "Lodestar version",
-      "transformations": [],
       "type": "stat"
     },
     {
@@ -267,10 +269,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "name"
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -321,10 +325,12 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "text": {},
-        "textMode": "name"
+        "textMode": "name",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -359,7 +365,7 @@
       },
       "gridPos": {
         "h": 2,
-        "w": 4,
+        "w": 2,
         "x": 20,
         "y": 0
       },
@@ -376,9 +382,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -395,6 +403,83 @@
       "title": "VC indices",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Ready"
+                },
+                "1": {
+                  "color": "yellow",
+                  "index": 1,
+                  "text": "Syncing"
+                },
+                "2": {
+                  "color": "red",
+                  "index": 2,
+                  "text": "Error"
+                }
+              },
+              "type": "value"
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 2,
+        "x": 22,
+        "y": 0
+      },
+      "id": 47,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "vc_beacon_health",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Beacon health",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -429,9 +514,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -482,9 +569,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -622,7 +711,7 @@
         },
         "showHeader": false
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {
@@ -701,6 +790,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -866,6 +956,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1019,7 +1110,8 @@
         },
         "showValue": "never",
         "tooltip": {
-          "show": true,
+          "mode": "single",
+          "showColorScale": false,
           "yHistogram": false
         },
         "yAxis": {
@@ -1028,7 +1120,7 @@
           "unit": "s"
         }
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "reverseYBuckets": false,
       "targets": [
         {
@@ -1133,7 +1225,8 @@
         },
         "showValue": "never",
         "tooltip": {
-          "show": true,
+          "mode": "single",
+          "showColorScale": false,
           "yHistogram": false
         },
         "yAxis": {
@@ -1142,7 +1235,7 @@
           "unit": "s"
         }
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "reverseYBuckets": false,
       "targets": [
         {
@@ -2344,8 +2437,7 @@
   ],
   "refresh": "10s",
   "revision": 1,
-  "schemaVersion": 38,
-  "style": "dark",
+  "schemaVersion": 39,
   "tags": [
     "lodestar"
   ],

diff --git a/packages/api/src/beacon/routes/node.ts b/packages/api/src/beacon/routes/node.ts
@@ -43,6 +43,22 @@ export const PeerCountType = new ContainerType(
   {jsonCase: "eth2"}
 );
 
+export const SyncingStatusType = new ContainerType(
+  {
+    /** Head slot node is trying to reach */
+    headSlot: ssz.Slot,
+    /** How many slots node needs to process to reach head. 0 if synced. */
+    syncDistance: ssz.Slot,
+    /** Set to true if the node is syncing, false if the node is synced. */
+    isSyncing: ssz.Boolean,
+    /** Set to true if the node is optimistically tracking head. */
+    isOptimistic: ssz.Boolean,
+    /** Set to true if the connected el client is offline */
+    elOffline: ssz.Boolean,
+  },
+  {jsonCase: "eth2"}
+);
+
 export type NetworkIdentity = ValueOf<typeof NetworkIdentityType>;
 
 export type PeerState = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -66,18 +82,7 @@ export type FilterGetPeers = {
   direction?: PeerDirection[];
 };
 
-export type SyncingStatus = {
-  /** Head slot node is trying to reach */
-  headSlot: string;
-  /** How many slots node needs to process to reach head. 0 if synced. */
-  syncDistance: string;
-  /** Set to true if the node is syncing, false if the node is synced. */
-  isSyncing: boolean;
-  /** Set to true if the node is optimistically tracking head. */
-  isOptimistic: boolean;
-  /** Set to true if the connected el client is offline */
-  elOffline: boolean;
-};
+export type SyncingStatus = ValueOf<typeof SyncingStatusType>;
 
 export enum NodeHealth {
   READY = HttpStatusCode.OK,
@@ -243,7 +248,10 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions<Endpo
       url: "/eth/v1/node/syncing",
       method: "GET",
       req: EmptyRequestCodec,
-      resp: JsonOnlyResponseCodec,
+      resp: {
+        data: SyncingStatusType,
+        meta: EmptyMetaCodec,
+      },
     },
     getHealth: {
       url: "/eth/v1/node/health",

diff --git a/packages/api/test/unit/beacon/testData/node.ts b/packages/api/test/unit/beacon/testData/node.ts
@@ -49,7 +49,7 @@ export const testData: GenericServerTestCases<Endpoints> = {
   },
   getSyncingStatus: {
     args: undefined,
-    res: {data: {headSlot: "1", syncDistance: "2", isSyncing: false, isOptimistic: true, elOffline: false}},
+    res: {data: {headSlot: 1, syncDistance: 2, isSyncing: false, isOptimistic: true, elOffline: false}},
   },
   getHealth: {
     args: {syncingStatus: 206},

diff --git a/packages/beacon-node/src/sync/sync.ts b/packages/beacon-node/src/sync/sync.ts
@@ -93,8 +93,8 @@ export class BeaconSync implements IBeaconSync {
     // If we are pre/at genesis, signal ready
     if (currentSlot <= GENESIS_SLOT) {
       return {
-        headSlot: "0",
-        syncDistance: "0",
+        headSlot: 0,
+        syncDistance: 0,
         isSyncing: false,
         isOptimistic: false,
         elOffline,
@@ -107,16 +107,16 @@ export class BeaconSync implements IBeaconSync {
         case SyncState.SyncingHead:
         case SyncState.Stalled:
           return {
-            headSlot: String(head.slot),
-            syncDistance: String(currentSlot - head.slot),
+            headSlot: head.slot,
+            syncDistance: currentSlot - head.slot,
             isSyncing: true,
             isOptimistic: isOptimisticBlock(head),
             elOffline,
           };
         case SyncState.Synced:
           return {
-            headSlot: String(head.slot),
-            syncDistance: "0",
+            headSlot: head.slot,
+            syncDistance: 0,
             isSyncing: false,
             isOptimistic: isOptimisticBlock(head),
             elOffline,

diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts
@@ -1,6 +1,7 @@
 import {describe, beforeAll, afterAll, it, expect, vi} from "vitest";
 import {createBeaconConfig} from "@lodestar/config";
 import {chainConfig as chainConfigDef} from "@lodestar/config/default";
+import {routes} from "@lodestar/api";
 import {ApiClient, getClient} from "@lodestar/api/beacon";
 import {sleep} from "@lodestar/utils";
 import {LogLevel, testLogger} from "../../../../../utils/logger.js";
@@ -46,9 +47,9 @@ describe("beacon node api", function () {
     it("should return valid syncing status", async () => {
       const res = await client.node.getSyncingStatus();
 
-      expect(res.value()).toEqual({
-        headSlot: "0",
-        syncDistance: "0",
+      expect(res.value()).toEqual<routes.node.SyncingStatus>({
+        headSlot: 0,
+        syncDistance: 0,
         isSyncing: false,
         isOptimistic: false,
         elOffline: false,

diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts
@@ -107,8 +107,8 @@ await env.tracker.assert(
 
 await env.tracker.assert("BN Not Synced", async () => {
   const expectedSyncStatus: routes.node.SyncingStatus = {
-    headSlot: "2",
-    syncDistance: "0",
+    headSlot: 2,
+    syncDistance: 0,
     isSyncing: false,
     isOptimistic: false,
     elOffline: false,