Add more logging for debugging

packages/beacon-node/src/chain/historicalState/historicalState.ts

@@ -2,6 +2,7 @@ import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/logger";
 import {BeaconConfig} from "@lodestar/config";
 import {computeEpochAtSlot, PubkeyIndexMap} from "@lodestar/state-transition";
+import {formatBytes} from "@lodestar/utils";
 import {IBeaconDb} from "../../db/interface.js";
 import {HistoricalStateRegenMetrics, IBinaryDiffCodec, StateArchiveStrategy} from "./types.js";
 import {replayBlocks} from "./utils/blockReplay.js";
@@ -99,6 +100,7 @@ export async function putHistoricalState(
       logger.verbose("State stored as snapshot", {
         epoch,
         slot,
+        snapshotSize: formatBytes(stateBytes.byteLength),
       });
       break;
     }
@@ -118,7 +120,8 @@ export async function putHistoricalState(
       logger.verbose("State stored as diff", {
         epoch,
         slot,
-        diffSize: diff.byteLength,
+        baseSize: formatBytes(diffState.byteLength),
+        diffSize: formatBytes(diff.byteLength),
       });
       break;
     }

packages/beacon-node/src/chain/historicalState/utils/diff.ts

@@ -50,7 +50,7 @@ export async function getDiffState(
   const processableDiffs = [...diffSlots];
 
   // Remove the snapshot slot
-  const snapshotSlot = processableDiffs.shift();
+  let snapshotSlot = processableDiffs.shift();
 
   if (skipSlotDiff && processableDiffs[processableDiffs.length - 1] === slot) {
     processableDiffs.pop();
@@ -62,13 +62,22 @@ export async function getDiffState(
     return {diffSlots, diffStateBytes: null};
   }
 
-  const snapshotStateBytes = await getSnapshotStateWithFallback(snapshotSlot, db);
-  if (!snapshotStateBytes) {
+  const snapshot = await getSnapshotStateWithFallback(snapshotSlot, db);
+  if (!snapshot.stateBytes) {
     logger?.error("Missing the snapshot state", {snapshotSlot});
     metrics?.regenErrorCount.inc({reason: RegenErrorType.loadState});
     return {diffStateBytes: null, diffSlots};
   }
 
+  if (snapshot.slot !== snapshotSlot) {
+    // Possibly because of checkpoint sync
+    logger?.warn("Last archived snapshot is not at expected slot", {
+      expectedSnapshotSlot: snapshotSlot,
+      availableSnapshotSlot: snapshot.slot,
+    });
+    snapshotSlot = snapshot.slot;
+  }
+
   // Get all diffs except the first one which was a snapshot layer
   const diffs = await Promise.all(
     processableDiffs.map((s) => {
@@ -103,7 +112,10 @@ export async function getDiffState(
       diffPath: diffSlots.join(","),
       availableDiffs: nonEmptyDiffs.map((d) => d.slot).join(","),
     });
-    const diffState = await replayStateDiffs({diffs: nonEmptyDiffs, snapshotStateBytes}, {codec, logger});
+    const diffState = await replayStateDiffs(
+      {diffs: nonEmptyDiffs, snapshotStateBytes: snapshot.stateBytes},
+      {codec, logger}
+    );
 
     if (diffState.byteLength === 0) {
       throw new Error("Some error during applying diffs");

packages/beacon-node/src/chain/historicalState/utils/snapshot.ts

@@ -1,14 +1,21 @@
 import {Slot} from "@lodestar/types";
 import {IBeaconDb} from "../../../db/index.js";
 
-export async function getSnapshotStateWithFallback(slot: Slot, db: IBeaconDb): Promise<Uint8Array | null> {
+export async function getSnapshotStateWithFallback(
+  slot: Slot,
+  db: IBeaconDb
+): Promise<{stateBytes: Uint8Array | null; slot: Slot}> {
   const state = await db.stateSnapshotArchive.getBinary(slot);
-  if (state) return state;
+  if (state) return {slot, stateBytes: state};
 
   // There is a possibility that node is started with checkpoint and initial snapshot
   // is not persisted on expected slot
   const lastSnapshotSlot = await db.stateSnapshotArchive.lastKey();
-  if (lastSnapshotSlot !== null) return db.stateSnapshotArchive.getBinary(lastSnapshotSlot);
+  if (lastSnapshotSlot !== null)
+    return {
+      slot: lastSnapshotSlot,
+      stateBytes: await db.stateSnapshotArchive.getBinary(lastSnapshotSlot),
+    };
 
-  return null;
+  return {stateBytes: null, slot};
 }