Skip to content

Commit

Permalink
chore: update discv5 and varint (#5895)
Browse files Browse the repository at this point in the history
* chore: update discv5 and deps

* chore: remove varint dependency

* fix: fix stray import

* chore: consistent naming
  • Loading branch information
wemeetagain authored Aug 18, 2023
1 parent f9c7107 commit 56ee70c
Show file tree
Hide file tree
Showing 10 changed files with 69 additions and 125 deletions.
5 changes: 2 additions & 3 deletions packages/beacon-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
"@chainsafe/as-sha256": "^0.3.1",
"@chainsafe/bls": "7.1.1",
"@chainsafe/blst": "^0.2.9",
"@chainsafe/discv5": "^5.0.0",
"@chainsafe/discv5": "^5.1.0",
"@chainsafe/libp2p-gossipsub": "^10.0.0",
"@chainsafe/libp2p-noise": "^13.0.0",
"@chainsafe/persistent-merkle-tree": "^0.5.0",
Expand Down Expand Up @@ -148,9 +148,9 @@
"snappyjs": "^0.7.0",
"strict-event-emitter-types": "^2.0.0",
"systeminformation": "^5.17.12",
"uint8-varint": "^2.0.1",
"uint8arraylist": "^2.4.3",
"uint8arrays": "^4.0.3",
"varint": "^6.0.0",
"xxhash-wasm": "1.0.2"
},
"devDependencies": {
Expand All @@ -159,7 +159,6 @@
"@types/qs": "^6.9.7",
"@types/supertest": "^2.0.12",
"@types/tmp": "^0.2.3",
"@types/varint": "^6.0.1",
"eventsource": "^2.0.2",
"leveldown": "^6.1.1",
"rewiremock": "^3.14.5",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {Connection} from "@libp2p/interface/connection";
import {CustomEvent} from "@libp2p/interfaces/events";
import {CustomEvent} from "@libp2p/interface/events";
import sinon from "sinon";
import {expect} from "chai";
import {BitArray} from "@chainsafe/ssz";
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
"@chainsafe/bls-keygen": "^0.3.0",
"@chainsafe/bls-keystore": "^2.0.0",
"@chainsafe/blst": "^0.2.9",
"@chainsafe/discv5": "^5.0.0",
"@chainsafe/discv5": "^5.1.0",
"@chainsafe/ssz": "^0.10.2",
"@chainsafe/threads": "^1.11.1",
"@libp2p/crypto": "^2.0.2",
Expand Down
4 changes: 2 additions & 2 deletions packages/reqresp/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@
"it-pipe": "^3.0.1",
"snappy": "^7.2.2",
"snappyjs": "^0.7.0",
"uint8arraylist": "^2.4.3",
"varint": "^6.0.0"
"uint8-varint": "^2.0.1",
"uint8arraylist": "^2.4.3"
},
"devDependencies": {
"@lodestar/logger": "^1.10.0",
Expand Down
15 changes: 4 additions & 11 deletions packages/reqresp/src/encodingStrategies/sszSnappy/decode.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import varint from "varint";
import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint";
import {Uint8ArrayList} from "uint8arraylist";
import {BufferedSource} from "../../utils/index.js";
import {TypeSizes} from "../../types.js";
Expand Down Expand Up @@ -32,23 +32,16 @@ export async function readSszSnappyHeader(bufferedSource: BufferedSource, type:
continue;
}

// Use Number.MAX_SAFE_INTEGER to guard against this check https://github.com/chrisdickinson/varint/pull/20
// On varint v6 if the number is > Number.MAX_SAFE_INTEGER `varint.decode` throws.
// Since MAX_VARINT_BYTES = 10, this will always be the case for the condition below.
// The check for MAX_VARINT_BYTES is kept for completeness
let sszDataLength: number;
try {
sszDataLength = varint.decode(buffer.slice());
sszDataLength = varintDecode(buffer.subarray());
} catch (e) {
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
}

// MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
// Check for varintBytes > 0 to guard against NaN, or 0 values
const varintBytes = varint.decode.bytes;
if (varintBytes === undefined || varintBytes > MAX_VARINT_BYTES || !(varintBytes > 0)) {
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes ?? 0});
}
// encodingLength function only returns 1-8 inclusive
const varintBytes = varintEncodingLength(sszDataLength);
buffer.consume(varintBytes);

// MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
Expand Down
4 changes: 2 additions & 2 deletions packages/reqresp/src/encodingStrategies/sszSnappy/encode.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import varint from "varint";
import {encode as varintEncode} from "uint8-varint";
import {encodeSnappy} from "./snappyFrames/compress.js";

/**
Expand All @@ -15,7 +15,7 @@ export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => A
*/
export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator<Buffer> {
// MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
yield Buffer.from(varint.encode(bytes.length));
yield Buffer.from(varintEncode(bytes.length));

// By first computing and writing the SSZ byte length, the SSZ encoder can then directly
// write the chunk contents to the stream. Snappy writer compresses frame by frame
Expand Down
6 changes: 3 additions & 3 deletions packages/reqresp/test/fixtures/encodingStrategies.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import {fileURLToPath} from "node:url";
import varint from "varint";
import {encode as varintEncode} from "uint8-varint";
import {ssz} from "@lodestar/types";
import {ForkName} from "@lodestar/params";
import {SszSnappyErrorCode} from "../../src/encodingStrategies/sszSnappy/errors.js";
Expand Down Expand Up @@ -72,12 +72,12 @@ export const encodingStrategiesDecodingErrorCases: {
id: "if it read more than maxEncodedLen",
type: ssz.phase0.Ping,
error: SszSnappyErrorCode.TOO_MUCH_BYTES_READ,
chunks: [Buffer.from(varint.encode(ssz.phase0.Ping.minSize)), Buffer.alloc(100)],
chunks: [Buffer.from(varintEncode(ssz.phase0.Ping.minSize)), Buffer.alloc(100)],
},
{
id: "if failed ssz snappy input malformed",
type: ssz.phase0.Status,
error: SszSnappyErrorCode.DECOMPRESSOR_ERROR,
chunks: [Buffer.from(varint.encode(ssz.phase0.Status.minSize)), Buffer.from("wrong snappy data")],
chunks: [Buffer.from(varintEncode(ssz.phase0.Status.minSize)), Buffer.from("wrong snappy data")],
},
];
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import chai, {expect} from "chai";
import chaiAsPromised from "chai-as-promised";
import {Uint8ArrayList} from "uint8arraylist";
import varint from "varint";
import {encode as varintEncode} from "uint8-varint";
import {readSszSnappyPayload} from "../../../../src/encodingStrategies/sszSnappy/index.js";
import {BufferedSource} from "../../../../src/utils/index.js";
import {
Expand All @@ -25,7 +25,7 @@ describe("encodingStrategies / sszSnappy / decode", () => {
describe("mainnet cases", () => {
for (const {id, payload, type: serializer, streamedBody} of encodingStrategiesMainnetTestCases) {
const bodySize = payload.data.length;
const streamedBytes = new Uint8ArrayList(Buffer.concat([Buffer.from(varint.encode(bodySize)), streamedBody]));
const streamedBytes = new Uint8ArrayList(Buffer.concat([Buffer.from(varintEncode(bodySize)), streamedBody]));

it(id, async () => {
const bufferedSource = new BufferedSource(arrToSource([streamedBytes]));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {expect} from "chai";
import all from "it-all";
import {pipe} from "it-pipe";
import varint from "varint";
import {encode as varintEncode} from "uint8-varint";
import {writeSszSnappyPayload} from "../../../../src/encodingStrategies/sszSnappy/encode.js";
import {encodingStrategiesMainnetTestCases, encodingStrategiesTestCases} from "../../../fixtures/index.js";
import {expectEqualByteChunks} from "../../../utils/index.js";
Expand All @@ -24,7 +24,7 @@ describe("encodingStrategies / sszSnappy / encode", () => {

const encodedChunks = await pipe(writeSszSnappyPayload(payload.data), all);
const encodedStream = Buffer.concat(encodedChunks as Uint8Array[]);
const expectedStreamed = Buffer.concat([Buffer.from(varint.encode(bodySize)), streamedBody]);
const expectedStreamed = Buffer.concat([Buffer.from(varintEncode(bodySize)), streamedBody]);
expect(encodedStream).to.be.deep.equal(expectedStreamed);
});
}
Expand Down
Loading

0 comments on commit 56ee70c

Please sign in to comment.