diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index a426d575559f..857eba209d56 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -106,6 +106,7 @@ "@chainsafe/ssz": "^0.17.1", "@chainsafe/threads": "^1.11.1", "@chainsafe/pubkey-index-map": "2.0.0", + "@chainsafe/xdelta3-node": "^1.0.2", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^10.0.1", "@fastify/cors": "^10.0.1", @@ -150,8 +151,6 @@ "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", "uint8arraylist": "^2.4.7", - "xdelta3-wasm": "^1.0.0", - "vcdiff-wasm": "^1.0.10", "xxhash-wasm": "1.0.2" }, "devDependencies": { diff --git a/packages/beacon-node/src/chain/historicalState/historicalState.ts b/packages/beacon-node/src/chain/historicalState/historicalState.ts index 2e2653b062af..4b9b45e6612a 100644 --- a/packages/beacon-node/src/chain/historicalState/historicalState.ts +++ b/packages/beacon-node/src/chain/historicalState/historicalState.ts @@ -8,10 +8,10 @@ import {IBeaconDb} from "../../db/interface.js"; import {HistoricalStateRegenMetrics, IBinaryDiffCodec, StateArchiveStrategy} from "./types.js"; import {replayBlocks} from "./utils/blockReplay.js"; import {DiffLayers} from "./diffLayers.js"; -import {BinaryDiffVCDiffCodec} from "./utils/binaryDiffVCDiffCodec.js"; +import {BinaryDiffXDelta3Codec} from "./utils/binaryDiffXDelta3Codec.js"; import {getDiffState} from "./utils/diff.js"; -export const codec: IBinaryDiffCodec = new BinaryDiffVCDiffCodec(); +export const codec: IBinaryDiffCodec = new BinaryDiffXDelta3Codec(); export async function getHistoricalState( {slot}: {slot: Slot}, diff --git a/packages/beacon-node/src/chain/historicalState/utils/binaryDiffVCDiffCodec.ts b/packages/beacon-node/src/chain/historicalState/utils/binaryDiffVCDiffCodec.ts deleted file mode 100644 index d23db8911a64..000000000000 --- a/packages/beacon-node/src/chain/historicalState/utils/binaryDiffVCDiffCodec.ts +++ /dev/null @@ -1,28 +0,0 @@ -import vcdiff from "vcdiff-wasm"; -import {IBinaryDiffCodec} from "../types.js"; - -export class BinaryDiffVCDiffCodec implements IBinaryDiffCodec { - private encoder!: (s: Uint8Array, i: Uint8Array) => Uint8Array; - private decoder!: (s: Uint8Array, i: Uint8Array) => Uint8Array; - private isInitialized: boolean = false; - - async init(): Promise { - const vc = await vcdiff(); - - this.encoder = vc.encoder; - this.decoder = vc.decoder; - this.isInitialized = true; - } - - get initialized(): boolean { - return this.isInitialized; - } - - compute(base: Uint8Array, changed: Uint8Array): Uint8Array { - return this.encoder(base, changed); - } - - apply(base: Uint8Array, delta: Uint8Array): Uint8Array { - return this.decoder(base, delta); - } -} diff --git a/packages/beacon-node/src/chain/historicalState/utils/binaryDiffXDelta3Codec.ts b/packages/beacon-node/src/chain/historicalState/utils/binaryDiffXDelta3Codec.ts index 232f87b6a98d..52a7bf45af85 100644 --- a/packages/beacon-node/src/chain/historicalState/utils/binaryDiffXDelta3Codec.ts +++ b/packages/beacon-node/src/chain/historicalState/utils/binaryDiffXDelta3Codec.ts @@ -1,11 +1,10 @@ -import {init, xd3_encode_memory, xd3_decode_memory, xd3_smatch_cfg} from "xdelta3-wasm"; +import {encodeSync, decodeSync} from "@chainsafe/xdelta3-node"; import {IBinaryDiffCodec} from "../types.js"; export class BinaryDiffXDelta3Codec implements IBinaryDiffCodec { private isInitialized: boolean = false; async init(): Promise { - await init(); this.isInitialized = true; } @@ -14,23 +13,18 @@ export class BinaryDiffXDelta3Codec implements IBinaryDiffCodec { } compute(base: Uint8Array, changed: Uint8Array): Uint8Array { - // The max size of a diff can be if input is empty and source is full state - // TODO: Try to optimize a way to calculate max output size to reduce memory consumption - const delta = xd3_encode_memory(changed, base, 1024 * 1024, xd3_smatch_cfg.FAST); - if (delta.str === "SUCCESS") { - return delta.output; + try { + return encodeSync(base, changed); + } catch (err) { + throw new Error(`Can not compute binary diff error=${(err as Error).message}`); } - - throw new Error(`Can not compute binary diff error=${delta.str}`); } apply(base: Uint8Array, delta: Uint8Array): Uint8Array { - const orig = xd3_decode_memory(delta, base, 9999); - - if (orig.str === "SUCCESS") { - return orig.output; + try { + return decodeSync(base, delta); + } catch (err) { + throw new Error(`Can not apply binary diff patch error=${(err as Error).message}`); } - - throw new Error("Can not apply binary diff patch"); } } diff --git a/packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffCodec.test.ts b/packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffXDelta3Codec.test.ts similarity index 95% rename from packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffCodec.test.ts rename to packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffXDelta3Codec.test.ts index f169bdfacf53..59cc33a5150e 100644 --- a/packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffCodec.test.ts +++ b/packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffXDelta3Codec.test.ts @@ -4,7 +4,7 @@ import {describe, it, expect, beforeAll} from "vitest"; import {BeaconState, Epoch, phase0, RootHex, Slot, ssz} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; -import {BinaryDiffVCDiffCodec} from "../../../../../src/chain/historicalState/utils/binaryDiffVCDiffCodec.js"; +import {BinaryDiffXDelta3Codec} from "../../../../../src/chain/historicalState/utils/binaryDiffXDelta3Codec.js"; import {generateState} from "../../../../utils/state.js"; import {IBinaryDiffCodec} from "../../../../../src/chain/historicalState/types.js"; @@ -16,8 +16,8 @@ const testsCases: {title: string; base: () => Uint8Array; changed: () => Uint8Ar }, { title: "Array of numbers", - base: () => Uint8Array.from([10, 11, 12]), - changed: () => Uint8Array.from([10, 11, 12, 14, 15]), + base: () => Uint8Array.from([10, 11, 12, 13, 14, 15]), + changed: () => Uint8Array.from([10, 11, 12, 14, 15, 16, 17, 18]), }, { title: "An attestation", @@ -69,7 +69,7 @@ describe("BinaryDiffCodec", () => { let multiDiffData: Record; beforeAll(async () => { - codec = new BinaryDiffVCDiffCodec(); + codec = new BinaryDiffXDelta3Codec(); await codec.init(); multiDiffData = { diff --git a/packages/beacon-node/test/unit/chain/historicalState/utils/diff.test.ts b/packages/beacon-node/test/unit/chain/historicalState/utils/diff.test.ts index 8b1abcd800a3..211c8b4c870e 100644 --- a/packages/beacon-node/test/unit/chain/historicalState/utils/diff.test.ts +++ b/packages/beacon-node/test/unit/chain/historicalState/utils/diff.test.ts @@ -6,7 +6,7 @@ import {getMockedBeaconDb} from "../../../../mocks/mockedBeaconDb.js"; import {getMockedLogger} from "../../../../mocks/loggerMock.js"; import {getDiffState} from "../../../../../src/chain/historicalState/utils/diff.js"; import {IBinaryDiffCodec} from "../../../../../src/chain/historicalState/types.js"; -import {BinaryDiffVCDiffCodec} from "../../../../../src/chain/historicalState/utils/binaryDiffVCDiffCodec.js"; +import {BinaryDiffXDelta3Codec} from "../../../../../src/chain/historicalState/utils/binaryDiffXDelta3Codec.js"; describe("historicalState/util", () => { let db: IBeaconDb; @@ -18,7 +18,7 @@ describe("historicalState/util", () => { db = getMockedBeaconDb(); logger = getMockedLogger(); diffLayers = new DiffLayers(); - codec = new BinaryDiffVCDiffCodec(); + codec = new BinaryDiffXDelta3Codec(); vi.spyOn(codec, "apply"); vi.spyOn(codec, "compute"); diff --git a/yarn.lock b/yarn.lock index b2350c8ae21a..7bed282f1838 100644 --- a/yarn.lock +++ b/yarn.lock @@ -650,6 +650,54 @@ optionalDependencies: tiny-worker ">= 2" +"@chainsafe/xdelta3-node-darwin-arm64@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-darwin-arm64/-/xdelta3-node-darwin-arm64-1.0.2.tgz#fa1891f29e7e44d0e4149f4b8f5731b333345db1" + integrity sha512-sgfhUZg87b/7JpBWLIPKv2tXzamLZNzEZnRDyCIR48r/VLZbslHSNz8m4qNWcsbflt81X9G4QxTz/gqdpSNxHA== + +"@chainsafe/xdelta3-node-darwin-x64@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-darwin-x64/-/xdelta3-node-darwin-x64-1.0.2.tgz#bda8f2eed2544ffa90cb06e90f767b96f8f4ea64" + integrity sha512-mOwZTgMqujBTl7gkYX+yoVL4eiM1NvS8cvb9fpLv4kvc9e9T0yMYP1QieDHuLEc2xk27ddZYAzYIXO9c8iDbDQ== + +"@chainsafe/xdelta3-node-linux-arm64-gnu@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-arm64-gnu/-/xdelta3-node-linux-arm64-gnu-1.0.2.tgz#0c27c53eb110a27323a59a93d143453c9dc5f445" + integrity sha512-/K5oZoMmf1WvcKQAL4lozU7kVxQ+NFtjBOsWQNm61weHRkiv4hBx7Fo7fQ71LIESu7Kyvs44zogrViR9bHn31A== + +"@chainsafe/xdelta3-node-linux-arm64-musl@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-arm64-musl/-/xdelta3-node-linux-arm64-musl-1.0.2.tgz#c2bb58531a8b57c00f8929952b391dd445b10cc7" + integrity sha512-GuAgRIUXZFGmAUz/bDAOEuFIexU4EcObIuoqijJWWGmdJYuLQsFkyEtr3YpANQJ0RT6EKYgK6AVHczzuG917AQ== + +"@chainsafe/xdelta3-node-linux-x64-gnu@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-x64-gnu/-/xdelta3-node-linux-x64-gnu-1.0.2.tgz#570184846e0a0ab4216458b73ee54aae3f26bc53" + integrity sha512-sL/UjocpJHLI1W+rqUxaOmlP4+nUUjQQCJuaReV1nDErgeQuUfPhYU8wQYRbwA+uCVR2eP2D64V5yehpFyNtcA== + +"@chainsafe/xdelta3-node-linux-x64-musl@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-x64-musl/-/xdelta3-node-linux-x64-musl-1.0.2.tgz#8d2dffdc44b1b733a6e67a80a54cc9eb73007d35" + integrity sha512-iwl17R2731jCVNfNJFLd3dXw7g4B0wR4AQYf/hUtG6xfxcF3fF75zuTfDbgXq3rH6x0vJv7Bn5Vtm+7w1PQtWg== + +"@chainsafe/xdelta3-node-win32-x64-msvc@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-win32-x64-msvc/-/xdelta3-node-win32-x64-msvc-1.0.2.tgz#f86cc56b0c18dbeb713ecc7b29032a0f5b4d1e4e" + integrity sha512-aRsLH7Pxq+hVvhCXwYWmlvtTa07f1btKSejK64yOlptIVrL0PEqlI0ocIYFyHPd1oDsfgBW6645/572ASlYOTw== + +"@chainsafe/xdelta3-node@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node/-/xdelta3-node-1.0.2.tgz#e0892e1e8a6f1ef72a4dac1c76abdecfead56277" + integrity sha512-zVuznkGTJ2tq1BJktRtc54zyPBZIHKoAf0cQWyoXs0dCju8PimKS5T8I7uwSRV36OkejxCIzFzYOqAlrnNnKGg== + optionalDependencies: + "@chainsafe/xdelta3-node-darwin-arm64" "1.0.2" + "@chainsafe/xdelta3-node-darwin-x64" "1.0.2" + "@chainsafe/xdelta3-node-linux-arm64-gnu" "1.0.2" + "@chainsafe/xdelta3-node-linux-arm64-musl" "1.0.2" + "@chainsafe/xdelta3-node-linux-x64-gnu" "1.0.2" + "@chainsafe/xdelta3-node-linux-x64-musl" "1.0.2" + "@chainsafe/xdelta3-node-win32-x64-msvc" "1.0.2" + "@colors/colors@1.5.0": version "1.5.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" @@ -13152,11 +13200,6 @@ validator@^13.7.0: resolved "https://registry.yarnpkg.com/validator/-/validator-13.11.0.tgz#23ab3fd59290c61248364eabf4067f04955fbb1b" integrity sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ== -vcdiff-wasm@^1.0.10: - version "1.0.10" - resolved "https://registry.yarnpkg.com/vcdiff-wasm/-/vcdiff-wasm-1.0.10.tgz#e58c53f39fcfd04c5facac6854081b1269fd6080" - integrity sha512-07siWwxXYClqAihEtoDYE18f5Z/lsX1ZQnJTVQ3k193bzjKCyOgja7jJBNzzr7jeKBGyF1Byc5CQf8tjPbUHKQ== - vite-node@2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-2.0.4.tgz#5600cc9f0d9c3ff9a64050c6858e7e1b62fb3fcd" @@ -13845,11 +13888,6 @@ ws@^8.18.0: resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== -xdelta3-wasm@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/xdelta3-wasm/-/xdelta3-wasm-1.0.0.tgz#43b6ef9e9649830e25713914e40324217f8565fd" - integrity sha512-vhS28BhVaE3S/PGG1KQIwjBVqJecuS5Sdh82UAZysbnYaU93KS6l8ZPtsqonNZMeuyFgrGW9D44hh2lwQCsidA== - xml-name-validator@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-5.0.0.tgz#82be9b957f7afdacf961e5980f1bf227c0bf7673"