From 0c088c85893979f52135a15a2bbf00220ae951ab Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 29 Jan 2024 20:42:54 -0500 Subject: [PATCH 1/8] feat: use snappy for larger gossip payloads Benchmarks show `snappy` is faster at compressing and uncompressing larger payloads than `snappyjs`. --- packages/beacon-node/package.json | 1 + .../src/network/gossip/encoding.ts | 21 +++++++++++-- .../test/perf/network/gossip/snappy.test.ts | 31 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 packages/beacon-node/test/perf/network/gossip/snappy.test.ts diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 174ed8cbb125..00d2f5bd0b95 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -148,6 +148,7 @@ "multiformats": "^11.0.1", "prom-client": "^15.1.0", "qs": "^6.11.1", + "snappy": "^7.2.2", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.17.12", diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index cccb0807b101..da627635a5d6 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -1,4 +1,5 @@ -import {compress, uncompress} from "snappyjs"; +import * as snappyjs from "snappyjs"; +import * as snappy from "snappy"; import xxhashFactory from "xxhash-wasm"; import {Message} from "@libp2p/interface"; import {digest} from "@chainsafe/as-sha256"; @@ -61,6 +62,11 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 return Buffer.from(digest(Buffer.concat(vec))).subarray(0, 20); } +/** Snappyjs is faster at compressing small buffers */ +const SNAPPYJS_COMPRESS_THRESHOLD = 200; +/** Snappyjs is faster at uncompressing small buffers */ +const SNAPPYJS_UNCOMPRESS_THRESHOLD = 500; + export class DataTransformSnappy { constructor( private readonly gossipTopicCache: GossipTopicCache, @@ -74,7 +80,12 @@ export class DataTransformSnappy { * - `outboundTransform()`: compress snappy payload */ inboundTransform(topicStr: string, data: Uint8Array): Uint8Array { - const uncompressedData = uncompress(data, this.maxSizePerMessage); + let uncompressedData: Uint8Array; + if (data.length < SNAPPYJS_UNCOMPRESS_THRESHOLD) { + uncompressedData = snappyjs.uncompress(data, this.maxSizePerMessage); + } else { + uncompressedData = snappy.uncompressSync(Buffer.from(data)) as Uint8Array; + } // check uncompressed data length before we extract beacon block root, slot or // attestation data at later steps @@ -101,6 +112,10 @@ export class DataTransformSnappy { throw Error(`ssz_snappy encoded data length ${length} > ${this.maxSizePerMessage}`); } // No need to parse topic, everything is snappy compressed - return compress(data); + if (data.length < SNAPPYJS_COMPRESS_THRESHOLD) { + return snappyjs.compress(data); + } else { + return snappy.compressSync(Buffer.from(data)); + } } } diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts new file mode 100644 index 000000000000..6fa9583122c7 --- /dev/null +++ b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts @@ -0,0 +1,31 @@ +import {randomBytes} from "node:crypto"; +import * as snappyjs from "snappyjs"; +import * as snappy from "snappy"; +import {itBench} from "@dapplion/benchmark"; + +describe("network / gossip / snappy", () => { + for (const msgLen of [200, 500, 1000, 10000, 100000]) { + const uncompressed = randomBytes(msgLen); + const compressed = snappyjs.compress(uncompressed); + + itBench({ + id: `snappyjs compress ${msgLen} bytes`, + fn: () => void snappyjs.compress(uncompressed), + }); + + itBench({ + id: `snappyjs uncompress ${msgLen} bytes`, + fn: () => void snappyjs.uncompress(compressed), + }); + + itBench({ + id: `snappy compress ${msgLen} bytes`, + fn: () => void snappy.compressSync(uncompressed), + }); + + itBench({ + id: `snappy uncompress ${msgLen} bytes`, + fn: () => void snappy.uncompressSync(compressed), + }); + } +}); From 68891ee8b987c59889c5ff4b4199a4ef83a58ddd Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 30 Jan 2024 01:59:39 -0500 Subject: [PATCH 2/8] chore: tweak snappyjs uncompress threshold --- packages/beacon-node/src/network/gossip/encoding.ts | 2 +- packages/beacon-node/test/perf/network/gossip/snappy.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index da627635a5d6..3e2b09883c0a 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -65,7 +65,7 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 /** Snappyjs is faster at compressing small buffers */ const SNAPPYJS_COMPRESS_THRESHOLD = 200; /** Snappyjs is faster at uncompressing small buffers */ -const SNAPPYJS_UNCOMPRESS_THRESHOLD = 500; +const SNAPPYJS_UNCOMPRESS_THRESHOLD = 400; export class DataTransformSnappy { constructor( diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts index 6fa9583122c7..120a14e060aa 100644 --- a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts +++ b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts @@ -4,7 +4,7 @@ import * as snappy from "snappy"; import {itBench} from "@dapplion/benchmark"; describe("network / gossip / snappy", () => { - for (const msgLen of [200, 500, 1000, 10000, 100000]) { + for (const msgLen of [100, 200, 300, 400, 500, 1000, 10000, 100000]) { const uncompressed = randomBytes(msgLen); const compressed = snappyjs.compress(uncompressed); From 41d4baef4b40b3a9e6e836042aeaa75fae324b96 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 27 Feb 2024 12:36:41 -0500 Subject: [PATCH 3/8] chore: add run factor to snappy benchmark --- .../test/perf/network/gossip/snappy.test.ts | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts index 120a14e060aa..564215a11e5b 100644 --- a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts +++ b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts @@ -7,25 +7,46 @@ describe("network / gossip / snappy", () => { for (const msgLen of [100, 200, 300, 400, 500, 1000, 10000, 100000]) { const uncompressed = randomBytes(msgLen); const compressed = snappyjs.compress(uncompressed); + const RUNS_FACTOR = 1000; itBench({ id: `snappyjs compress ${msgLen} bytes`, - fn: () => void snappyjs.compress(uncompressed), + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappyjs.compress(uncompressed); + } + }, }); itBench({ id: `snappyjs uncompress ${msgLen} bytes`, - fn: () => void snappyjs.uncompress(compressed), + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappyjs.uncompress(compressed); + } + }, }); itBench({ id: `snappy compress ${msgLen} bytes`, - fn: () => void snappy.compressSync(uncompressed), + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappy.compressSync(uncompressed); + } + }, }); itBench({ id: `snappy uncompress ${msgLen} bytes`, - fn: () => void snappy.uncompressSync(compressed), + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappy.uncompressSync(compressed); + } + }, }); } }); From 864d36130f774894f615bb730dcf69134c7e9997 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 27 Feb 2024 12:39:54 -0500 Subject: [PATCH 4/8] chore: adjust snappyjs thresholds --- packages/beacon-node/src/network/gossip/encoding.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 3e2b09883c0a..794a2102957b 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -63,9 +63,9 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 } /** Snappyjs is faster at compressing small buffers */ -const SNAPPYJS_COMPRESS_THRESHOLD = 200; +const SNAPPYJS_COMPRESS_THRESHOLD = 100; /** Snappyjs is faster at uncompressing small buffers */ -const SNAPPYJS_UNCOMPRESS_THRESHOLD = 400; +const SNAPPYJS_UNCOMPRESS_THRESHOLD = 200; export class DataTransformSnappy { constructor( From 53622cd9714d6028120844544ad29d9c676f67e3 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 29 Feb 2024 16:50:20 -0500 Subject: [PATCH 5/8] chore: tweak perf output --- .../test/perf/network/gossip/snappy.test.ts | 91 ++++++++++--------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts index 564215a11e5b..9eb2d21ea6b5 100644 --- a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts +++ b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts @@ -4,49 +4,58 @@ import * as snappy from "snappy"; import {itBench} from "@dapplion/benchmark"; describe("network / gossip / snappy", () => { - for (const msgLen of [100, 200, 300, 400, 500, 1000, 10000, 100000]) { - const uncompressed = randomBytes(msgLen); - const compressed = snappyjs.compress(uncompressed); - const RUNS_FACTOR = 1000; + const msgLens = [100, 200, 300, 400, 500, 1000, 10000, 100000]; + describe("compress", () => { + for (const msgLen of msgLens) { + const uncompressed = randomBytes(msgLen); + const RUNS_FACTOR = 1000; - itBench({ - id: `snappyjs compress ${msgLen} bytes`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappyjs.compress(uncompressed); - } - }, - }); + itBench({ + id: `${msgLen} bytes - compress - snappyjs`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappyjs.compress(uncompressed); + } + }, + }); - itBench({ - id: `snappyjs uncompress ${msgLen} bytes`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappyjs.uncompress(compressed); - } - }, - }); + itBench({ + id: `${msgLen} bytes - compress - snappy`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappy.compressSync(uncompressed); + } + }, + }); + } + }); + describe("uncompress", () => { + for (const msgLen of [100, 200, 300, 400, 500, 1000, 10000, 100000]) { + const uncompressed = randomBytes(msgLen); + const compressed = snappyjs.compress(uncompressed); + const RUNS_FACTOR = 1000; - itBench({ - id: `snappy compress ${msgLen} bytes`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappy.compressSync(uncompressed); - } - }, - }); + itBench({ + id: `${msgLen} bytes - uncompress - snappyjs`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappyjs.uncompress(compressed); + } + }, + }); - itBench({ - id: `snappy uncompress ${msgLen} bytes`, - runsFactor: RUNS_FACTOR, - fn: () => { - for (let i = 0; i < RUNS_FACTOR; i++) { - snappy.uncompressSync(compressed); - } - }, - }); - } + itBench({ + id: `${msgLen} bytes - uncompress - snappy`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappy.uncompressSync(compressed); + } + }, + }); + } + }); }); From 96333a9bc086dfeac4aab04f4d229b8f59e2df1a Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 24 Jul 2024 12:00:31 -0400 Subject: [PATCH 6/8] feat: use snappy-wasm --- packages/beacon-node/package.json | 1 + .../src/network/gossip/encoding.ts | 28 +-- .../test/perf/network/gossip/snappy.test.ts | 28 ++- yarn.lock | 230 ++++++++++++------ 4 files changed, 195 insertions(+), 92 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index ce3178ba763a..643fe2b503d4 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -148,6 +148,7 @@ "prom-client": "^15.1.0", "qs": "^6.11.1", "snappy": "^7.2.2", + "snappy-wasm": "^0.3.0", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 0a01f29fe97b..78f9f629fae3 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -1,5 +1,3 @@ -import * as snappyjs from "snappyjs"; -import * as snappy from "snappy"; import xxhashFactory from "xxhash-wasm"; import {Message} from "@libp2p/interface"; import {digest} from "@chainsafe/as-sha256"; @@ -10,6 +8,13 @@ import {ForkName} from "@lodestar/params"; import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js"; import {getGossipSSZType, GossipTopicCache} from "./topic.js"; +// Import snappy wasm (work around loading issues) +// eslint-disable-next-line import/order +import {createRequire} from "node:module"; +const require = createRequire(import.meta.url); +// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment +const snappyWasm = require("snappy-wasm") as typeof import("snappy-wasm"); + // Load WASM const xxhash = await xxhashFactory(); @@ -63,11 +68,6 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8 return digest(Buffer.concat(vec)).subarray(0, 20); } -/** Snappyjs is faster at compressing small buffers */ -const SNAPPYJS_COMPRESS_THRESHOLD = 100; -/** Snappyjs is faster at uncompressing small buffers */ -const SNAPPYJS_UNCOMPRESS_THRESHOLD = 200; - export class DataTransformSnappy implements DataTransform { constructor( private readonly gossipTopicCache: GossipTopicCache, @@ -81,12 +81,8 @@ export class DataTransformSnappy implements DataTransform { * - `outboundTransform()`: compress snappy payload */ inboundTransform(topicStr: string, data: Uint8Array): Uint8Array { - let uncompressedData: Uint8Array; - if (data.length < SNAPPYJS_UNCOMPRESS_THRESHOLD) { - uncompressedData = snappyjs.uncompress(data, this.maxSizePerMessage); - } else { - uncompressedData = snappy.uncompressSync(Buffer.from(data)) as Uint8Array; - } + // TODO use this.maxSizePerMessage here + const uncompressedData = snappyWasm.decompress_raw(data); // check uncompressed data length before we extract beacon block root, slot or // attestation data at later steps @@ -113,10 +109,6 @@ export class DataTransformSnappy implements DataTransform { throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`); } // No need to parse topic, everything is snappy compressed - if (data.length < SNAPPYJS_COMPRESS_THRESHOLD) { - return snappyjs.compress(data); - } else { - return snappy.compressSync(Buffer.from(data)); - } + return snappyWasm.compress_raw(data); } } diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts index 9eb2d21ea6b5..366c087575dd 100644 --- a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts +++ b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts @@ -3,6 +3,12 @@ import * as snappyjs from "snappyjs"; import * as snappy from "snappy"; import {itBench} from "@dapplion/benchmark"; +// eslint-disable-next-line import/order +import {createRequire} from "node:module"; +const require = createRequire(import.meta.url); +// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment +const snappyWasm = require("snappy-wasm") as typeof import("snappy-wasm"); + describe("network / gossip / snappy", () => { const msgLens = [100, 200, 300, 400, 500, 1000, 10000, 100000]; describe("compress", () => { @@ -29,10 +35,20 @@ describe("network / gossip / snappy", () => { } }, }); + + itBench({ + id: `${msgLen} bytes - compress - snappy-wasm`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappyWasm.compress_raw(uncompressed); + } + }, + }); } }); describe("uncompress", () => { - for (const msgLen of [100, 200, 300, 400, 500, 1000, 10000, 100000]) { + for (const msgLen of msgLens) { const uncompressed = randomBytes(msgLen); const compressed = snappyjs.compress(uncompressed); const RUNS_FACTOR = 1000; @@ -56,6 +72,16 @@ describe("network / gossip / snappy", () => { } }, }); + + itBench({ + id: `${msgLen} bytes - uncompress - snappy-wasm`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + snappyWasm.decompress_raw(compressed); + } + }, + }); } }); }); diff --git a/yarn.lock b/yarn.lock index 0f01b9e7e294..d7c465ce9ea2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1555,6 +1555,29 @@ progress-events "^1.0.0" uint8arraylist "^2.4.8" +"@libp2p/interface@^1.2.0", "@libp2p/interface@^1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface/-/interface-1.6.1.tgz#dc8f1a49da7ee81cd068e872bde3445c3b631d45" + integrity sha512-bpkIYTvZhGGc/ajITKvgFpaP8UtPWoSj+xHVrj6zyAN8U/cAqN0IQQt4a7daJr5VZa8B86i4d1iccdG42/mz+g== + dependencies: + "@multiformats/multiaddr" "^12.2.3" + it-pushable "^3.2.3" + it-stream-types "^2.0.1" + multiformats "^13.1.0" + progress-events "^1.0.0" + uint8arraylist "^2.4.8" + +"@libp2p/logger@^4.0.10": + version "4.0.17" + resolved "https://registry.yarnpkg.com/@libp2p/logger/-/logger-4.0.17.tgz#1ae663bb2dec3c7e6b29d2195756f64e5e77d939" + integrity sha512-NPGN27uOXFGuKkxnX39InMvxS0lMenq6/aFqQHN1N0f0S3LaG9RuTcz/VE3qyO1Ik1aAockR6qqCwbfFxJuO0g== + dependencies: + "@libp2p/interface" "^1.6.1" + "@multiformats/multiaddr" "^12.2.3" + debug "^4.3.4" + interface-datastore "^8.2.11" + multiformats "^13.1.0" + "@libp2p/logger@^4.0.11", "@libp2p/logger@^4.0.6": version "4.0.11" resolved "https://registry.yarnpkg.com/@libp2p/logger/-/logger-4.0.11.tgz#671692a0cceee73a0c0bf9b5f05ea14fde05f5e5" @@ -1845,6 +1868,19 @@ uint8-varint "^2.0.1" uint8arrays "^5.0.0" +"@multiformats/multiaddr@^12.2.3": + version "12.3.0" + resolved "https://registry.yarnpkg.com/@multiformats/multiaddr/-/multiaddr-12.3.0.tgz#b1422813446e5cdec4b0f6cba51f93239f390884" + integrity sha512-JQ8Gc/jgucqqvEaDTFN/AvxlYDHEE7lgEWLMYW7hKZkWggER+GvG/tVxUgUxIP8M0vFpvEHKKHE0lKzyMsgi8Q== + dependencies: + "@chainsafe/is-ip" "^2.0.1" + "@chainsafe/netmask" "^2.0.0" + "@libp2p/interface" "^1.0.0" + "@multiformats/dns" "^1.0.3" + multiformats "^13.0.0" + uint8-varint "^2.0.1" + uint8arrays "^5.0.0" + "@napi-rs/snappy-android-arm-eabi@7.2.2": version "7.2.2" resolved "https://registry.yarnpkg.com/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz#85fee3ba198dad4b444b5f12bceebcf72db0d65e" @@ -2499,18 +2535,18 @@ integrity sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ== "@puppeteer/browsers@1.4.6", "@puppeteer/browsers@^1.6.0", "@puppeteer/browsers@^2.1.0": - version "2.2.2" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.2.2.tgz#c43b00a9808370fec3e548779d81d1e0b972e8bb" - integrity sha512-hZ/JhxPIceWaGSEzUZp83/8M49CoxlkuThfTR7t4AoCu5+ZvJ3vktLm60Otww2TXeROB5igiZ8D9oPQh6ckBVg== + version "2.2.4" + resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.2.4.tgz#4307245d881aa5a79743050be66568bad0f6ffbb" + integrity sha512-BdG2qiI1dn89OTUUsx2GZSpUzW+DRffR1wlMJyKxVHYrhnKoELSDxDd+2XImUkuWPEKk76H5FcM/gPFrEK1Tfw== dependencies: - debug "4.3.4" - extract-zip "2.0.1" - progress "2.0.3" - proxy-agent "6.4.0" - semver "7.6.0" - tar-fs "3.0.5" - unbzip2-stream "1.4.3" - yargs "17.7.2" + debug "^4.3.5" + extract-zip "^2.0.1" + progress "^2.0.3" + proxy-agent "^6.4.0" + semver "^7.6.2" + tar-fs "^3.0.6" + unbzip2-stream "^1.4.3" + yargs "^17.7.2" "@rollup/plugin-inject@^5.0.5": version "5.0.5" @@ -5408,6 +5444,13 @@ debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.3.5: + version "4.3.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + decamelize-keys@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9" @@ -6383,7 +6426,7 @@ external-editor@^3.0.3: iconv-lite "^0.4.24" tmp "^0.0.33" -extract-zip@2.0.1, extract-zip@^2.0.1: +extract-zip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== @@ -7407,7 +7450,7 @@ http-proxy-agent@^7.0.0: agent-base "^7.1.0" debug "^4.3.4" -http-proxy-agent@^7.0.2: +http-proxy-agent@^7.0.1, http-proxy-agent@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig== @@ -7453,14 +7496,6 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" -https-proxy-agent@^7.0.0, https-proxy-agent@^7.0.4: - version "7.0.4" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" - integrity sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg== - dependencies: - agent-base "^7.0.2" - debug "4" - https-proxy-agent@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz#e2645b846b90e96c6e6f347fb5b2e41f1590b09b" @@ -7469,6 +7504,22 @@ https-proxy-agent@^7.0.2: agent-base "^7.0.2" debug "4" +https-proxy-agent@^7.0.3, https-proxy-agent@^7.0.5: + version "7.0.5" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz#9e8b5013873299e11fab6fd548405da2d6c602b2" + integrity sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw== + dependencies: + agent-base "^7.0.2" + debug "4" + +https-proxy-agent@^7.0.4: + version "7.0.4" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" + integrity sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg== + dependencies: + agent-base "^7.0.2" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" @@ -7675,16 +7726,19 @@ internal-slot@^1.0.5: has "^1.0.3" side-channel "^1.0.4" +ip-address@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a" + integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g== + dependencies: + jsbn "1.1.0" + sprintf-js "^1.1.3" + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz" integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk= -ip@^1.1.8: - version "1.1.9" - resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.9.tgz#8dfbcc99a754d07f425310b86a99546b1151e396" - integrity sha512-cyRxvOEpNHNtchU3Ln9KC/auJgup87llfQpQ+t5ghoC/UhL16SWzbueiCsdTnWmqAWl7LadfuwhlqmtOaqMHdQ== - ip@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.1.tgz#e8f3595d33a3ea66490204234b77636965307105" @@ -8301,6 +8355,11 @@ js-yaml@^3.10.0: argparse "^1.0.7" esprima "^4.0.0" +jsbn@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" + integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A== + jsdom@^23.0.1: version "23.0.1" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-23.0.1.tgz#ede7ff76e89ca035b11178d200710d8982ebfee0" @@ -9489,11 +9548,16 @@ n12@0.4.0: resolved "https://registry.yarnpkg.com/n12/-/n12-0.4.0.tgz#363058560b435e6857b5e039ed5eab08c5122e5e" integrity sha512-p/hj4zQ8d3pbbFLQuN1K9honUxiDDhueOWyFLw/XgBv+wZCE44bcLH4CIcsolOceJQduh4Jf7m/LfaTxyGmGtQ== -nan@^2.16.0, nan@^2.17.0, nan@^2.19.0: +nan@^2.16.0, nan@^2.17.0: version "2.19.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.19.0.tgz#bb58122ad55a6c5bc973303908d5b16cfdd5a8c0" integrity sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw== +nan@^2.19.0: + version "2.20.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.20.0.tgz#08c5ea813dd54ed16e5bd6505bf42af4f7838ca3" + integrity sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw== + nanoid@3.3.3: version "3.3.3" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" @@ -10259,27 +10323,26 @@ p-waterfall@2.1.1: dependencies: p-reduce "^2.0.0" -pac-proxy-agent@^7.0.0: - version "7.0.1" - resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz#6b9ddc002ec3ff0ba5fdf4a8a21d363bcc612d75" - integrity sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A== +pac-proxy-agent@^7.0.1: + version "7.0.2" + resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz#0fb02496bd9fb8ae7eb11cfd98386daaac442f58" + integrity sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg== dependencies: "@tootallnate/quickjs-emscripten" "^0.23.0" agent-base "^7.0.2" debug "^4.3.4" get-uri "^6.0.1" http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.2" - pac-resolver "^7.0.0" - socks-proxy-agent "^8.0.2" + https-proxy-agent "^7.0.5" + pac-resolver "^7.0.1" + socks-proxy-agent "^8.0.4" -pac-resolver@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.0.tgz#79376f1ca26baf245b96b34c339d79bff25e900c" - integrity sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg== +pac-resolver@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.1.tgz#54675558ea368b64d210fd9c92a640b5f3b8abb6" + integrity sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg== dependencies: degenerator "^5.0.0" - ip "^1.1.8" netmask "^2.0.2" pacote@^15.2.0: @@ -10612,7 +10675,7 @@ progress-events@^1.0.0: resolved "https://registry.yarnpkg.com/progress-events/-/progress-events-1.0.0.tgz#34f5e8fdb5dae3561837b22672d1e02277bb2109" integrity sha512-zIB6QDrSbPfRg+33FZalluFIowkbV5Xh1xSuetjG+rlC5he6u2dc6VQJ0TbMdlN3R1RHdpOqxEFMKTnQ+itUwA== -progress@2.0.3, progress@^2.0.3: +progress@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== @@ -10683,19 +10746,19 @@ proxy-addr@^2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-agent@6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.3.0.tgz#72f7bb20eb06049db79f7f86c49342c34f9ba08d" - integrity sha512-0LdR757eTj/JfuU7TL2YCuAZnxWXu3tkJbg4Oq3geW/qFNT/32T0sp2HnZ9O0lMR4q3vwAt0+xCA8SR0WAD0og== +proxy-agent@^6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.4.0.tgz#b4e2dd51dee2b377748aef8d45604c2d7608652d" + integrity sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ== dependencies: agent-base "^7.0.2" debug "^4.3.4" - http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" + http-proxy-agent "^7.0.1" + https-proxy-agent "^7.0.3" lru-cache "^7.14.1" - pac-proxy-agent "^7.0.0" + pac-proxy-agent "^7.0.1" proxy-from-env "^1.1.0" - socks-proxy-agent "^8.0.1" + socks-proxy-agent "^8.0.2" proxy-from-env@^1.1.0: version "1.1.0" @@ -11376,6 +11439,11 @@ semver@^7.6.0: dependencies: lru-cache "^6.0.0" +semver@^7.6.2: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + serialize-error@^11.0.1: version "11.0.3" resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-11.0.3.tgz#b54f439e15da5b4961340fbbd376b6b04aa52e92" @@ -11542,6 +11610,11 @@ smart-buffer@^4.2.0: resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== +snappy-wasm@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/snappy-wasm/-/snappy-wasm-0.3.0.tgz#cdf096130f8258dc55763a8662d974dd40964e52" + integrity sha512-mT2q7y2K0krf2WdsfR7YZL/vJ+BiTKcvgxIA/gJozPQibWJwV8VOl6TNZLjeUlGH2bqee/570fS/LUkO406Ixg== + snappy@^7.2.2: version "7.2.2" resolved "https://registry.yarnpkg.com/snappy/-/snappy-7.2.2.tgz#dbd9217ae06b651c073856036618c2dc8992ef17" @@ -11584,15 +11657,6 @@ socks-proxy-agent@^7.0.0: debug "^4.3.3" socks "^2.6.2" -socks-proxy-agent@^8.0.1: - version "8.0.3" - resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.3.tgz#6b2da3d77364fde6292e810b496cb70440b9b89d" - integrity sha512-VNegTZKhuGq5vSD6XNKlbqWhyt/40CgoEw8XxD6dhnm8Jq9IEa3nIa4HwnM8XOqU0CdB0BwWVXusqiFXfHB3+A== - dependencies: - agent-base "^7.1.1" - debug "^4.3.4" - socks "^2.7.1" - socks-proxy-agent@^8.0.2: version "8.0.2" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad" @@ -11602,6 +11666,15 @@ socks-proxy-agent@^8.0.2: debug "^4.3.4" socks "^2.7.1" +socks-proxy-agent@^8.0.4: + version "8.0.4" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz#9071dca17af95f483300316f4b063578fa0db08c" + integrity sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw== + dependencies: + agent-base "^7.1.1" + debug "^4.3.4" + socks "^2.8.3" + socks@^2.6.1, socks@^2.6.2, socks@^2.7.1: version "2.7.1" resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.1.tgz#d8e651247178fde79c0663043e07240196857d55" @@ -11610,6 +11683,14 @@ socks@^2.6.1, socks@^2.6.2, socks@^2.7.1: ip "^2.0.0" smart-buffer "^4.2.0" +socks@^2.8.3: + version "2.8.3" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.3.tgz#1ebd0f09c52ba95a09750afe3f3f9f724a800cb5" + integrity sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw== + dependencies: + ip-address "^9.0.5" + smart-buffer "^4.2.0" + sonic-boom@^3.7.0: version "3.8.0" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.8.0.tgz#e442c5c23165df897d77c3c14ef3ca40dec66a66" @@ -11707,6 +11788,11 @@ sprintf-js@^1.1.2: resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz" integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug== +sprintf-js@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" + integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" @@ -12089,7 +12175,7 @@ tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@3.0.4, tar-fs@^3.0.4: +tar-fs@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== @@ -12109,6 +12195,17 @@ tar-fs@^3.0.5: bare-fs "^2.1.1" bare-path "^2.1.0" +tar-fs@^3.0.6: + version "3.0.6" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.6.tgz#eaccd3a67d5672f09ca8e8f9c3d2b89fa173f217" + integrity sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w== + dependencies: + pump "^3.0.0" + tar-stream "^3.1.5" + optionalDependencies: + bare-fs "^2.1.1" + bare-path "^2.1.0" + tar-fs@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" @@ -12662,7 +12759,7 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" -unbzip2-stream@1.4.3: +unbzip2-stream@^1.4.3: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg== @@ -13689,20 +13786,7 @@ yargs@16.2.0, yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@17.7.1: - version "17.7.1" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.1.tgz#34a77645201d1a8fc5213ace787c220eabbd0967" - integrity sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw== - dependencies: - cliui "^8.0.1" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.3" - y18n "^5.0.5" - yargs-parser "^21.1.1" - -yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1: +yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1, yargs@^17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== From 093a4695840f07d86d5ad2efae5d6685b66e22a1 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 5 Aug 2024 12:47:21 -0400 Subject: [PATCH 7/8] feat: use @chainsafe/snappy-wasm with prealloc --- packages/beacon-node/package.json | 2 +- .../src/network/gossip/encoding.ts | 32 +++--- .../test/perf/network/gossip/snappy.test.ts | 107 ++++++++++++++++-- yarn.lock | 10 +- 4 files changed, 124 insertions(+), 27 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 579bc3e34276..94f9e21251f3 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -146,7 +146,7 @@ "prom-client": "^15.1.0", "qs": "^6.11.1", "snappy": "^7.2.2", - "snappy-wasm": "^0.3.0", + "@chainsafe/snappy-wasm": "^0.5.0", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 78f9f629fae3..24b9e1f69ee5 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -3,24 +3,22 @@ import {Message} from "@libp2p/interface"; import {digest} from "@chainsafe/as-sha256"; import {RPC} from "@chainsafe/libp2p-gossipsub/message"; import {DataTransform} from "@chainsafe/libp2p-gossipsub/types"; +import snappyWasm from "@chainsafe/snappy-wasm"; import {intToBytes, toHex} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js"; import {getGossipSSZType, GossipTopicCache} from "./topic.js"; -// Import snappy wasm (work around loading issues) -// eslint-disable-next-line import/order -import {createRequire} from "node:module"; -const require = createRequire(import.meta.url); -// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -const snappyWasm = require("snappy-wasm") as typeof import("snappy-wasm"); - // Load WASM const xxhash = await xxhashFactory(); // Use salt to prevent msgId from being mined for collisions const h64Seed = BigInt(Math.floor(Math.random() * 1e9)); +// create singleton snappy encoder + decoder +const encoder = new snappyWasm.Encoder(); +const decoder = new snappyWasm.Decoder(); + /** * The function used to generate a gossipsub message id * We use the first 8 bytes of SHA256(data) for content addressing @@ -81,12 +79,12 @@ export class DataTransformSnappy implements DataTransform { * - `outboundTransform()`: compress snappy payload */ inboundTransform(topicStr: string, data: Uint8Array): Uint8Array { - // TODO use this.maxSizePerMessage here - const uncompressedData = snappyWasm.decompress_raw(data); + // check uncompressed data length before we actually decompress + const uncompressedDataLength = snappyWasm.decompress_len(data); + if (uncompressedDataLength > this.maxSizePerMessage) { + throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${this.maxSizePerMessage}`); + } - // check uncompressed data length before we extract beacon block root, slot or - // attestation data at later steps - const uncompressedDataLength = uncompressedData.length; const topic = this.gossipTopicCache.getTopic(topicStr); const sszType = getGossipSSZType(topic); @@ -97,6 +95,9 @@ export class DataTransformSnappy implements DataTransform { throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`); } + // Only after saniy length checks, we can decompress the data + const uncompressedData = Buffer.allocUnsafe(uncompressedDataLength); + decoder.decompress_into(data, uncompressedData); return uncompressedData; } @@ -104,11 +105,14 @@ export class DataTransformSnappy implements DataTransform { * Takes the data to be published (a topic and associated data) transforms the data. The * transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers. */ + // No need to parse topic, everything is snappy compressed outboundTransform(_topicStr: string, data: Uint8Array): Uint8Array { if (data.length > this.maxSizePerMessage) { throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`); } - // No need to parse topic, everything is snappy compressed - return snappyWasm.compress_raw(data); + + const compressedData = Buffer.allocUnsafe(snappyWasm.max_compress_len(data.length)); + const compressedLen = encoder.compress_into(data, compressedData); + return compressedData.subarray(0, compressedLen); } } diff --git a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts index 366c087575dd..2a14db1a10af 100644 --- a/packages/beacon-node/test/perf/network/gossip/snappy.test.ts +++ b/packages/beacon-node/test/perf/network/gossip/snappy.test.ts @@ -2,16 +2,85 @@ import {randomBytes} from "node:crypto"; import * as snappyjs from "snappyjs"; import * as snappy from "snappy"; import {itBench} from "@dapplion/benchmark"; +import snappyWasm from "@chainsafe/snappy-wasm"; -// eslint-disable-next-line import/order -import {createRequire} from "node:module"; -const require = createRequire(import.meta.url); -// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -const snappyWasm = require("snappy-wasm") as typeof import("snappy-wasm"); +/* 2024-08-05 - Linux 5.15 x86_64 - Node.js v22.4.1 + + network / gossip / snappy + compress + ✔ 100 bytes - compress - snappyjs 335566.9 ops/s 2.980032 us/op - 685 runs 2.54 s + ✔ 100 bytes - compress - snappy 388610.3 ops/s 2.573272 us/op - 870 runs 2.74 s + ✔ 100 bytes - compress - snappy-wasm 583254.0 ops/s 1.714519 us/op - 476 runs 1.32 s + ✔ 100 bytes - compress - snappy-wasm - prealloc 1586695 ops/s 630.2410 ns/op - 481 runs 0.804 s + ✔ 200 bytes - compress - snappyjs 298272.8 ops/s 3.352636 us/op - 213 runs 1.22 s + ✔ 200 bytes - compress - snappy 419528.0 ops/s 2.383631 us/op - 926 runs 2.71 s + ✔ 200 bytes - compress - snappy-wasm 472468.5 ops/s 2.116543 us/op - 577 runs 1.72 s + ✔ 200 bytes - compress - snappy-wasm - prealloc 1430445 ops/s 699.0830 ns/op - 868 runs 1.11 s + ✔ 300 bytes - compress - snappyjs 265124.9 ops/s 3.771807 us/op - 137 runs 1.02 s + ✔ 300 bytes - compress - snappy 361683.9 ops/s 2.764845 us/op - 1332 runs 4.18 s + ✔ 300 bytes - compress - snappy-wasm 443688.4 ops/s 2.253834 us/op - 859 runs 2.44 s + ✔ 300 bytes - compress - snappy-wasm - prealloc 1213825 ops/s 823.8420 ns/op - 370 runs 0.807 s + ✔ 400 bytes - compress - snappyjs 262168.5 ops/s 3.814341 us/op - 358 runs 1.87 s + ✔ 400 bytes - compress - snappy 382494.9 ops/s 2.614414 us/op - 1562 runs 4.58 s + ✔ 400 bytes - compress - snappy-wasm 406373.2 ops/s 2.460792 us/op - 797 runs 2.46 s + ✔ 400 bytes - compress - snappy-wasm - prealloc 1111715 ops/s 899.5110 ns/op - 450 runs 0.906 s + ✔ 500 bytes - compress - snappyjs 229213.1 ops/s 4.362753 us/op - 359 runs 2.07 s + ✔ 500 bytes - compress - snappy 373695.8 ops/s 2.675973 us/op - 2050 runs 5.99 s + ✔ 500 bytes - compress - snappy-wasm 714917.4 ops/s 1.398763 us/op - 960 runs 1.84 s + ✔ 500 bytes - compress - snappy-wasm - prealloc 1054619 ops/s 948.2100 ns/op - 427 runs 0.907 s + ✔ 1000 bytes - compress - snappyjs 148702.3 ops/s 6.724847 us/op - 171 runs 1.65 s + ✔ 1000 bytes - compress - snappy 423688.1 ops/s 2.360227 us/op - 525 runs 1.74 s + ✔ 1000 bytes - compress - snappy-wasm 524350.6 ops/s 1.907121 us/op - 273 runs 1.03 s + ✔ 1000 bytes - compress - snappy-wasm - prealloc 685191.5 ops/s 1.459446 us/op - 349 runs 1.01 s + ✔ 10000 bytes - compress - snappyjs 21716.92 ops/s 46.04704 us/op - 16 runs 1.24 s + ✔ 10000 bytes - compress - snappy 98051.32 ops/s 10.19874 us/op - 184 runs 2.39 s + ✔ 10000 bytes - compress - snappy-wasm 114681.8 ops/s 8.719783 us/op - 49 runs 0.937 s + ✔ 10000 bytes - compress - snappy-wasm - prealloc 111203.6 ops/s 8.992518 us/op - 49 runs 0.953 s + ✔ 100000 bytes - compress - snappyjs 2947.313 ops/s 339.2921 us/op - 12 runs 4.74 s + ✔ 100000 bytes - compress - snappy 14963.78 ops/s 66.82801 us/op - 70 runs 5.19 s + ✔ 100000 bytes - compress - snappy-wasm 19868.33 ops/s 50.33136 us/op - 14 runs 1.21 s + ✔ 100000 bytes - compress - snappy-wasm - prealloc 24579.34 ops/s 40.68457 us/op - 13 runs 1.06 s + uncompress + ✔ 100 bytes - uncompress - snappyjs 589201.6 ops/s 1.697212 us/op - 242 runs 0.911 s + ✔ 100 bytes - uncompress - snappy 537424.1 ops/s 1.860728 us/op - 220 runs 0.910 s + ✔ 100 bytes - uncompress - snappy-wasm 634966.2 ops/s 1.574887 us/op - 194 runs 0.808 s + ✔ 100 bytes - uncompress - snappy-wasm - prealloc 1846964 ops/s 541.4290 ns/op - 559 runs 0.804 s + ✔ 200 bytes - uncompress - snappyjs 395141.8 ops/s 2.530737 us/op - 281 runs 1.22 s + ✔ 200 bytes - uncompress - snappy 536862.6 ops/s 1.862674 us/op - 274 runs 1.01 s + ✔ 200 bytes - uncompress - snappy-wasm 420251.6 ops/s 2.379527 us/op - 129 runs 0.810 s + ✔ 200 bytes - uncompress - snappy-wasm - prealloc 1746167 ops/s 572.6830 ns/op - 529 runs 0.804 s + ✔ 300 bytes - uncompress - snappyjs 441676.2 ops/s 2.264102 us/op - 898 runs 2.53 s + ✔ 300 bytes - uncompress - snappy 551313.2 ops/s 1.813851 us/op - 336 runs 1.11 s + ✔ 300 bytes - uncompress - snappy-wasm 494773.0 ops/s 2.021129 us/op - 203 runs 0.912 s + ✔ 300 bytes - uncompress - snappy-wasm - prealloc 1528680 ops/s 654.1590 ns/op - 465 runs 0.805 s + ✔ 400 bytes - uncompress - snappyjs 383746.1 ops/s 2.605890 us/op - 235 runs 1.11 s + ✔ 400 bytes - uncompress - snappy 515986.6 ops/s 1.938035 us/op - 158 runs 0.809 s + ✔ 400 bytes - uncompress - snappy-wasm 392947.8 ops/s 2.544867 us/op - 322 runs 1.32 s + ✔ 400 bytes - uncompress - snappy-wasm - prealloc 1425978 ops/s 701.2730 ns/op - 721 runs 1.01 s + ✔ 500 bytes - uncompress - snappyjs 330727.5 ops/s 3.023637 us/op - 173 runs 1.02 s + ✔ 500 bytes - uncompress - snappy 513874.1 ops/s 1.946002 us/op - 157 runs 0.806 s + ✔ 500 bytes - uncompress - snappy-wasm 389263.0 ops/s 2.568957 us/op - 161 runs 0.914 s + ✔ 500 bytes - uncompress - snappy-wasm - prealloc 1330936 ops/s 751.3510 ns/op - 672 runs 1.01 s + ✔ 1000 bytes - uncompress - snappyjs 241393.9 ops/s 4.142606 us/op - 126 runs 1.03 s + ✔ 1000 bytes - uncompress - snappy 491119.6 ops/s 2.036164 us/op - 201 runs 0.911 s + ✔ 1000 bytes - uncompress - snappy-wasm 361794.5 ops/s 2.764000 us/op - 148 runs 0.910 s + ✔ 1000 bytes - uncompress - snappy-wasm - prealloc 959026.5 ops/s 1.042724 us/op - 390 runs 0.909 s + ✔ 10000 bytes - uncompress - snappyjs 40519.03 ops/s 24.67976 us/op - 16 runs 0.913 s + ✔ 10000 bytes - uncompress - snappy 202537.6 ops/s 4.937355 us/op - 796 runs 4.43 s + ✔ 10000 bytes - uncompress - snappy-wasm 165017.6 ops/s 6.059960 us/op - 52 runs 0.822 s + ✔ 10000 bytes - uncompress - snappy-wasm - prealloc 175061.5 ops/s 5.712277 us/op - 130 runs 1.25 s + ✔ 100000 bytes - uncompress - snappyjs 4030.391 ops/s 248.1149 us/op - 12 runs 3.71 s + ✔ 100000 bytes - uncompress - snappy 35459.43 ops/s 28.20124 us/op - 41 runs 1.67 s + ✔ 100000 bytes - uncompress - snappy-wasm 22449.16 ops/s 44.54509 us/op - 13 runs 1.11 s + ✔ 100000 bytes - uncompress - snappy-wasm - prealloc 27169.50 ops/s 36.80598 us/op - 13 runs 0.997 s + +*/ describe("network / gossip / snappy", () => { const msgLens = [100, 200, 300, 400, 500, 1000, 10000, 100000]; describe("compress", () => { + const encoder = new snappyWasm.Encoder(); + for (const msgLen of msgLens) { const uncompressed = randomBytes(msgLen); const RUNS_FACTOR = 1000; @@ -41,13 +110,27 @@ describe("network / gossip / snappy", () => { runsFactor: RUNS_FACTOR, fn: () => { for (let i = 0; i < RUNS_FACTOR; i++) { - snappyWasm.compress_raw(uncompressed); + encoder.compress(uncompressed); + } + }, + }); + + itBench({ + id: `${msgLen} bytes - compress - snappy-wasm - prealloc`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + let out = Buffer.allocUnsafe(snappyWasm.max_compress_len(uncompressed.length)); + const len = encoder.compress_into(uncompressed, out); + out = out.subarray(0, len); } }, }); } }); describe("uncompress", () => { + const decoder = new snappyWasm.Decoder(); + for (const msgLen of msgLens) { const uncompressed = randomBytes(msgLen); const compressed = snappyjs.compress(uncompressed); @@ -78,7 +161,17 @@ describe("network / gossip / snappy", () => { runsFactor: RUNS_FACTOR, fn: () => { for (let i = 0; i < RUNS_FACTOR; i++) { - snappyWasm.decompress_raw(compressed); + decoder.decompress(compressed); + } + }, + }); + + itBench({ + id: `${msgLen} bytes - uncompress - snappy-wasm - prealloc`, + runsFactor: RUNS_FACTOR, + fn: () => { + for (let i = 0; i < RUNS_FACTOR; i++) { + decoder.decompress_into(compressed, Buffer.allocUnsafe(snappyWasm.decompress_len(compressed))); } }, }); diff --git a/yarn.lock b/yarn.lock index 081b02e18416..eadac87bdb9a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -556,6 +556,11 @@ resolved "https://registry.yarnpkg.com/@chainsafe/prometheus-gc-stats/-/prometheus-gc-stats-1.0.2.tgz#585f8f1555251db156d7e50ef8c86dd4f3e78f70" integrity sha512-h3mFKduSX85XMVbOdWOYvx9jNq99jGcRVNyW5goGOqju1CsI+ZJLhu5z4zBb/G+ksL0R4uLVulu/mIMe7Y0rNg== +"@chainsafe/snappy-wasm@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@chainsafe/snappy-wasm/-/snappy-wasm-0.5.0.tgz#067e534341ef746706e2dbf255bd7604c849be73" + integrity sha512-ydXvhr9p+JjvzSSEyi6XExq8pHugFnrk70mk17T6mhDsklPvaXc+8K90G7TJF+u51lxI/fpv7MahrA5ayjFcSA== + "@chainsafe/ssz@^0.11.1": version "0.11.1" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.11.1.tgz#d4aec883af2ec5196ae67b96242c467da20b2476" @@ -11773,11 +11778,6 @@ smart-buffer@^4.2.0: resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== -snappy-wasm@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/snappy-wasm/-/snappy-wasm-0.3.0.tgz#cdf096130f8258dc55763a8662d974dd40964e52" - integrity sha512-mT2q7y2K0krf2WdsfR7YZL/vJ+BiTKcvgxIA/gJozPQibWJwV8VOl6TNZLjeUlGH2bqee/570fS/LUkO406Ixg== - snappy@^7.2.2: version "7.2.2" resolved "https://registry.yarnpkg.com/snappy/-/snappy-7.2.2.tgz#dbd9217ae06b651c073856036618c2dc8992ef17" From 33ca8e74a8caa77c62f0b7d5f4e14b70f09ec65c Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 6 Aug 2024 16:39:51 -0400 Subject: [PATCH 8/8] chore: remove snappy dep --- packages/beacon-node/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 94f9e21251f3..a59ec4440dee 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -145,9 +145,7 @@ "multiformats": "^11.0.1", "prom-client": "^15.1.0", "qs": "^6.11.1", - "snappy": "^7.2.2", "@chainsafe/snappy-wasm": "^0.5.0", - "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", "uint8arraylist": "^2.4.7",