feat: implement db persistent option

packages/beacon-node/src/chain/chain.ts

@@ -78,6 +78,7 @@ import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js";
 import {ShufflingCache} from "./shufflingCache.js";
 import {MemoryCheckpointStateCache} from "./stateCache/memoryCheckpointsCache.js";
 import {FilePersistentApis} from "./stateCache/persistent/file.js";
+import {DbPersistentApis} from "./stateCache/persistent/db.js";
 
 /**
  * Arbitrary constants, blobs should be consumed immediately in the same slot they are produced.
@@ -236,8 +237,9 @@ export class BeaconChain implements IBeaconChain {
     this.index2pubkey = cachedState.epochCtx.index2pubkey;
 
     const stateCache = new StateContextCache(this.opts, {metrics});
-    // TODO: chain option to switch persistent
-    const filePersistent = new FilePersistentApis(CHECKPOINT_STATES_FOLDER);
+    const persistentApis = this.opts.persistCheckpointStatesToFile
+      ? new FilePersistentApis(CHECKPOINT_STATES_FOLDER)
+      : new DbPersistentApis(this.db);
     const checkpointStateCache = this.opts.persistentCheckpointStateCache
       ? new PersistentCheckpointStateCache(
           {
@@ -246,7 +248,7 @@ export class BeaconChain implements IBeaconChain {
             clock,
             shufflingCache: this.shufflingCache,
             getHeadState: this.getHeadState.bind(this),
-            persistentApis: filePersistent,
+            persistentApis,
           },
           this.opts
         )

packages/beacon-node/src/chain/options.ts

@@ -31,7 +31,10 @@ export type IChainOptions = BlockProcessOpts &
     trustedSetup?: string;
     broadcastValidationStrictness?: string;
     minSameMessageSignatureSetsToBatch: number;
+    // TODO: change to n_historical_states
     persistentCheckpointStateCache?: boolean;
+    /** by default persist checkpoint state to db */
+    persistCheckpointStatesToFile?: boolean;
   };
 
 export type BlockProcessOpts = {
@@ -95,6 +98,9 @@ export const defaultChainOptions: IChainOptions = {
   minSameMessageSignatureSetsToBatch: 2,
   // TODO: change to false, leaving here to ease testing
   persistentCheckpointStateCache: true,
+  // TODO: change to false, leaving here to ease testing
+  persistCheckpointStatesToFile: true,
+
   // since Sep 2023, only cache up to 32 states by default. If a big reorg happens it'll load checkpoint state from disk and regen from there.
   // TODO: change to 128, leaving here to ease testing
   maxStates: 32,

packages/beacon-node/src/chain/stateCache/persistent/db.ts

@@ -0,0 +1,38 @@
+import {fromHexString, toHexString} from "@chainsafe/ssz";
+import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
+import {IBeaconDb} from "../../../db/interface.js";
+import {CPStatePersistentApis, PersistentKey} from "./types.js";
+
+/**
+ * Implementation of CPStatePersistentApis using db.
+ */
+export class DbPersistentApis implements CPStatePersistentApis {
+  constructor(private readonly db: IBeaconDb) {
+    void cleanBucket(db);
+  }
+  async write(_: string, state: CachedBeaconStateAllForks): Promise<string> {
+    const root = state.hashTreeRoot();
+    const stateBytes = state.serialize();
+    await this.db.checkpointState.putBinary(root, stateBytes);
+    return toHexString(root);
+  }
+
+  async remove(persistentKey: PersistentKey): Promise<boolean> {
+    await this.db.checkpointState.delete(fromHexString(persistentKey));
+    return true;
+  }
+
+  async read(persistentKey: string): Promise<Uint8Array | null> {
+    return this.db.checkpointState.getBinary(fromHexString(persistentKey));
+  }
+}
+
+/**
+ * Clean all checkpoint state in db at startup time.
+ */
+async function cleanBucket(db: IBeaconDb): Promise<void> {
+  const keys = await db.checkpointState.keys();
+  for (const key of keys) {
+    await db.checkpointState.delete(key);
+  }
+}

packages/beacon-node/src/chain/stateCache/persistent/file.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import {removeFile, writeIfNotExist, ensureDir, readAllFileNames} from "@lodestar/utils";
+import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {CheckpointKey} from "../types.js";
 import {CPStatePersistentApis, PersistentKey} from "./types.js";
 
@@ -9,21 +10,33 @@ import {CPStatePersistentApis, PersistentKey} from "./types.js";
  */
 export class FilePersistentApis implements CPStatePersistentApis {
   constructor(private readonly folderPath: string) {
+    // this is very fast and most of the time we don't need to create folder
+    // state files from previous run will be removed asynchronously
     void ensureEmptyFolder(folderPath);
   }
 
-  async write(checkpointKey: CheckpointKey, bytes: Uint8Array): Promise<PersistentKey> {
+  /**
+   * Writing to file name with `${cp.rootHex}_${cp.epoch}` helps debugging.
+   * This is slow code as it do state serialization which takes 600ms to 900ms on holesky.
+   */
+  async write(checkpointKey: CheckpointKey, state: CachedBeaconStateAllForks): Promise<PersistentKey> {
+    const stateBytes = state.serialize();
     const persistentKey = this.toPersistentKey(checkpointKey);
-    await writeIfNotExist(persistentKey, bytes);
+    await writeIfNotExist(persistentKey, stateBytes);
     return persistentKey;
   }
 
   async remove(persistentKey: PersistentKey): Promise<boolean> {
     return removeFile(persistentKey);
   }
 
-  async read(persistentKey: PersistentKey): Promise<Uint8Array> {
-    return fs.promises.readFile(persistentKey);
+  async read(persistentKey: PersistentKey): Promise<Uint8Array | null> {
+    try {
+      const stateBytes = await fs.promises.readFile(persistentKey);
+      return stateBytes;
+    } catch (_) {
+      return null;
+    }
   }
 
   private toPersistentKey(checkpointKey: CheckpointKey): PersistentKey {

packages/beacon-node/src/chain/stateCache/persistent/types.ts

@@ -1,11 +1,12 @@
+import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {CheckpointKey} from "../types.js";
 
 // With fs implementation, persistentKey is ${CHECKPOINT_STATES_FOLDER/rootHex_epoch}
 export type PersistentKey = string;
 
 // Make this generic to support testing
 export interface CPStatePersistentApis {
-  write: (cpKey: CheckpointKey, bytes: Uint8Array) => Promise<PersistentKey>;
+  write: (cpKey: CheckpointKey, state: CachedBeaconStateAllForks) => Promise<PersistentKey>;
   remove: (persistentKey: PersistentKey) => Promise<boolean>;
-  read: (persistentKey: PersistentKey) => Promise<Uint8Array>;
+  read: (persistentKey: PersistentKey) => Promise<Uint8Array | null>;
 }