add comments, fix for blindDeserialize (#504)

core/base/src/utils/amount.ts

@@ -170,10 +170,11 @@ export function display(amount: Amount, precision?: number): string {
 export function whole(amount: Amount): number {
  return Number(display(amount));
 }
+
 /**
- *
- * @param amount
- * @param decimals
+ * fmt formats a bigint amount to a string with the given number of decimals
+ * @param amount bigint amount
+ * @param decimals number of decimals
  */
 export function fmt(amount: bigint, decimals: number): string {
  return display(fromBaseUnits(amount, decimals));

core/base/src/utils/encoding.ts

@@ -10,8 +10,11 @@ const isHexRegex = /^(?:0x)?[0-9a-fA-F]+$/;
 
 /** Base16/Hex encoding and decoding utilities */
 export const hex = {
+ /** check if a string is valid hex */
  valid: (input: string) => isHexRegex.test(input),
+ /** decode a hex string to Uint8Array */
  decode: (input: string) => base16.decode(stripPrefix("0x", input).toUpperCase()),
+ /** encode a string or Uint8Array to hex */
  encode: (input: string | Uint8Array, prefix: boolean = false) => {
  input = typeof input === "string" ? bytes.encode(input) : input;
  return (prefix ? "0x" : "") + base16.encode(input).toLowerCase();
@@ -24,33 +27,44 @@ const isB64Regex = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}
 
 /** Base64 encoding and decoding utilities */
 export const b64 = {
+ /** check if a string is valid base64 */
  valid: (input: string) => isB64Regex.test(input),
+ /** decode a base64 string to Uint8Array */
  decode: base64.decode,
+ /** encode a string or Uint8Array to base64 */
  encode: (input: string | Uint8Array) =>
  base64.encode(typeof input === "string" ? bytes.encode(input) : input),
 };
 
 /** Base58 encoding and decoding utilities */
 export const b58 = {
+ /** decode a base58 string to Uint8Array */
  decode: base58.decode,
+ /** encode a string or Uint8Array to base58 */
  encode: (input: string | Uint8Array) =>
  base58.encode(typeof input === "string" ? bytes.encode(input) : input),
 };
 
 /** BigInt encoding and decoding utilities */
 export const bignum = {
+ /** decode a hex string or bytes to a bigint */
  decode: (input: string | Uint8Array) => {
  if (typeof input !== "string") input = hex.encode(input, true);
  if (input === "" || input === "0x") return 0n;
  return BigInt(input);
  },
+ /** encode a bigint as a hex string */
  encode: (input: bigint, prefix: boolean = false) => bignum.toString(input, prefix),
+ /** convert a bigint to a hexstring */
  toString: (input: bigint, prefix: boolean = false) => {
  let str = input.toString(16);
  str = str.length % 2 === 1 ? (str = "0" + str) : str;
  if (prefix) return "0x" + str;
  return str;
  },
+ /** convert a bigint or number to bytes, 
+ * optionally specify length, left padded with 0s to length
+ */
  toBytes: (input: bigint | number, length?: number) => {
  const b = hex.decode(bignum.toString(typeof input === "number" ? BigInt(input) : input));
  if (!length) return b;
@@ -60,14 +74,19 @@ export const bignum = {
 
 /** Uint8Array encoding and decoding utilities */
 export const bytes = {
+ /** encode a string to Uint8Array */
  encode: (value: string): Uint8Array => new TextEncoder().encode(value),
+ /** decode a Uint8Array to string */
  decode: (value: Uint8Array): string => new TextDecoder().decode(value),
+ /** compare two Uint8Arrays for equality */
  equals: (lhs: Uint8Array, rhs: Uint8Array): boolean =>
  lhs.length === rhs.length && lhs.every((v, i) => v === rhs[i]),
+ /** pad a Uint8Array to a given length, optionally specifying padding direction */
  zpad: (arr: Uint8Array, length: number, padStart: boolean = true): Uint8Array =>
  padStart
  ? bytes.concat(new Uint8Array(length - arr.length), arr)
  : bytes.concat(arr, new Uint8Array(length - arr.length)),
+ /** concatenate multiple Uint8Arrays into a single Uint8Array */
  concat: (...args: Uint8Array[]): Uint8Array => {
  const length = args.reduce((acc, curr) => acc + curr.length, 0);
  const result = new Uint8Array(length);

core/base/src/utils/layout/discriminate.ts

@@ -298,6 +298,8 @@ function buildAscendingBounds(sortedBounds: readonly (readonly [Bounds, LayoutIn
 function generateLayoutDiscriminator(
  layouts: readonly Layout[]
 ): [boolean, (encoded: BytesType) => readonly LayoutIndex[]] {
+ //for debug output:
+ // const candStr = (candidate: Bitset) => candidate.toString(2).padStart(layouts.length, '0');
 
  if (layouts.length === 0)
  throw new Error("Cannot discriminate empty set of layouts");
@@ -357,6 +359,11 @@ function generateLayoutDiscriminator(
  for (let j = 0; j < serialized.length; ++j)
  fixedKnownBytes[offset + j]!.push([serialized[j]!, i]);
 
+ //debug output:
+ // console.log("fixedKnownBytes:",
+ // fixedKnownBytes.map((v, i) => v.length > 0 ? [i, v] : undefined).filter(v => v !== undefined)
+ // );
+
  let bestBytes = [];
  for (const [bytePos, fixedKnownByte] of fixedKnownBytes.entries()) {
  //the number of layouts with a given size is an upper bound on the discriminatory power of
@@ -376,7 +383,7 @@ function generateLayoutDiscriminator(
  distinctValues.set(byteVal, distinctValues.get(byteVal)! | 1n << BigInt(candidate));
  }
 
- let power = count(lwba);
+ let power = layouts.length - Math.max(count(anyValueLayouts), count(outOfBoundsLayouts));
  for (const layoutsWithValue of distinctValues.values()) {
  //if we find the byte value associated with this set of layouts, we can eliminate
  // all other layouts that don't have this value at this position and all layouts
@@ -385,6 +392,17 @@ function generateLayoutDiscriminator(
  power = Math.min(power, curPower);
  }
 
+ //debug output:
+ // console.log(
+ // "bytePos:", bytePos,
+ // "\npower:", power,
+ // "\nfixedKnownByte:", fixedKnownByte,
+ // "\nlwba:", candStr(lwba),
+ // "\nanyValueLayouts:", candStr(anyValueLayouts),
+ // "\noutOfBoundsLayouts:", candStr(outOfBoundsLayouts),
+ // "\ndistinctValues:", new Map([...distinctValues].map(([k, v]) => [k, candStr(v)]))
+ // );
+
  if (power === 0)
  continue;
 
@@ -521,14 +539,31 @@ function generateLayoutDiscriminator(
  throw new Error("Implementation error in layout discrimination algorithm");
  };
 
+ //debug output:
+ // console.log("strategies:", JSON.stringify(
+ // new Map([...strategies].map(([cands, strat]) => [
+ // candStr(cands),
+ // typeof strat === "string"
+ // ? strat
+ // : [
+ // strat[0], //bytePos
+ // candStr(strat[1]), //outOfBoundsLayouts
+ // new Map([...strat[2]].map(([value, cands]) => [value, candStr(cands)]))
+ // ]
+ // ]
+ // ))
+ // ));
+
  return [distinguishable, (encoded: BytesType) => {
  let candidates = allLayouts;
 
- for (
- let strategy = strategies.get(candidates)!;
- strategy !== "indistinguishable";
- strategy = strategies.get(candidates) ?? findSmallestSuperSetStrategy(candidates)
- ) {
+ let strategy = strategies.get(candidates)!;
+ while (strategy !== "indistinguishable") {
+ //debug output:
+ // console.log(
+ // "applying strategy", strategy,
+ // "\nfor remaining candidates:", candStr(candidates)
+ // );
  if (strategy === "size")
  candidates &= layoutsWithSize(encoded.length);
  else {
@@ -546,9 +581,13 @@ function generateLayoutDiscriminator(
  }
 
  if (count(candidates) <= 1)
- return bitsetToArray(candidates);
+ break;
+
+ strategy = strategies.get(candidates) ?? findSmallestSuperSetStrategy(candidates)
  }
 
+ //debug output:
+ // console.log("final candidates", candStr(candidates));
  return bitsetToArray(candidates);
  }];
 }

core/definitions/__tests__/blindDeserialize.ts

@@ -0,0 +1,34 @@
+import { encoding } from "@wormhole-foundation/sdk-base";
+import {
+ blindDeserializePayload,
+ deserialize,
+ deserializePayload,
+ exhaustiveDeserialize,
+} from "../src/index.js";
+import "../src/protocols/index.js";
+
+const cases = [
+ "AQAAAAQNAEI6Ol1ax5h7zmg2cv9pxdznRm1grHS9RL1JLj/I8HJ4Akb1nwXc+zMcOFdAiMASnEwUYpxyBwgJF5QHMS0hXwQBAvTwT11h4/z9dsCEW1pDNaDon0B9aQ6o9/S3zjUnCcuEEgxSwp5hXRjUJ4lds48rpDSfAlMzF/RB3x5+p1NxBDcBA5CMs0VxzRxgAxB+zy9Hn263MEEn5c98Lky8604/RI56C/O/mtoZrvOHtM3ln0yEapeqBcNMvl5L0CpuJ4xtWFsABCVTI/T6ou8+EHP4LC6PBCt/yjEr/QEVJsMx21eFT0y2Z86cBQCw6LmA1ER179Z9WO69FyGPmtHxxHovLi3+sZUABrnxQ0eXTkt7OjiHx/yj3rE6xqzwHqdGQzUBN5SgWCFhTxxfA9Kmt3hlJ6bRnfpR9QMeLxc5tlZCfQGODIQrra8BBwa71PjV61JJaipwAjA/pUsG9fI1qeX5CQknohKbpGUMUb1MZixB8YUMGOsQbBidNh67BwHe0kX7ofh2Y1hYxp0ACEsz46+CAuELDC3Q8jRarQLW20cAWWsRmjjXxSyqOrEDUdSHdJHe1dvzRL1LgD7gsOX7cGuuY/6USFFhPl7cCUAACXm7gAjTn194YTNUdWnZtgNyP+V02tr9a5kcM1xb6D7AcCMW6NbUnjby66L1RycPkyXGoITkGjXvsVxJFxcotzcBCrpiwyBJNx+XM1GEGAmbYYd5Fw6y69L0q9RTk5oNtOmIE2ssvW+/ZMbaLPB3fWf37fYzulNL0YU/u7+JpE+eSuIADSYc1vb3lV+P1rFBgVMlWnnpgpE3UJPq5ZbRHsiP4aLCQUAGkQMl/rtyAyh64fZGA9eRhRRWV76KiwY9CMaWMEYADmhp08yb3cBPvzpDlc1MQ52UKIgHU94cd1dmP27xJbrSJZiwSxL2HiJZdtHZ1EnEp/LBnIrzrTT5w/qwLa7T/0UBD9BzsGb9ekj0TrRLdYdN4PgAzBPV/M8XfFzW2Ex4hhg/BAnd4pH32FN8dGwueGciKr6/z5/ORV8UTCgUDcgTziEBEu7MNJ5xy2J5OWK9ZsKX4UnC29zaTCGIwfx/9bKctOAhIDLn+IIDEwPWQszd2mx4z1IeT0AhHs9Jpuf80uJGuu4BZi0+vA2QAAAADgAAAAAAAAAAAAAAAHlt/2108+JwYLcSVf5Re/sjyT7tAAAAAAACudkBAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAX14QAAAAAAAAAAAAAAAAAwCqqObIj/o0KDlxPJ+rZCDx1bMIAAg9A62uZJ8TprduI7DEsB7ndr+gKgbGMF8Hnc05bod8YAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+ "AQAAAAQNAOk1OzNZ9DfkOsrjn2jpzz01k8MMqxEeS+NtyMPySJw6OhFAvz76Buk+O/hec/ifJ3m1y0joY1DUeru6sJ+PMF0BAunCoXhMBfluALeB3duNBsgwG12ZfuxOH33FPgWq98qnTh9xFUd6z3DPXb7CJA4oFYm9nYEZtfVTkYs6WEjYCc4BA+ZpLzYmZ46XLxRtHdggukDaK+gHiVyMxoL1BOb0OmCGIwV07bMMk1pG5B0IQwRDf68WDhe3PYQDjuvze9Gkw0gABANQu3VOyPj9tWTbC5jWglfzwgQ98JOau0liKfwF/GYiRV3r+7JPiTXhmmk6J+j7b8WGNidGqrRsomcTD2V1/TsABtmKBYXbmuSds3OWE9ZxubU8cw2h9gv6SHxeTpBWIwHPaiHggfWEznJcUkWByjWZ+nOA+kAAFjhoAlyjsBZkgRMBB7FezvEAnCYCneStFT3AolKVifA8mIBfiKxQpiX9mJkTGatmhMpi8Tp+YXgXYMIk3wCxALg/ZwMu3x76KkIEiIwBCCrfe4hZexgLJMZcReLKvvVfJmcThyEEk9aF/sM5pqaRBuAzG0jKvyN9oHYdq7p9qSzOYuJMbKneJ1ERAOdiWrEACVfoGUdlV4p3etUt32Lr8zUf1NtEuE47UtIUBbMRTZxgNqdP3wZIkrOFoOj5k6+2XTVNIRqTxiEMJohjvVtVms4ADSkByD3p1QM1A8GeGslv+wfsTDhq6cI0MRiGouS277uiXMTZ9MHPZ6VVyIb4gppeZm+A7xzJqp6584oPV+LtqsYAD9yLQKItaBTEWXpPzlExi38ztxCe2Soio+udkOIcbq7mdMx/UWvSpQf04/jhNInBpUEgI4GbSlxrqke9ue61p6cAECaPI5R1x7avvXlWP3hx0V08Jz40kqtf6x7M8ZSb+2gRGQTYdRaY5ZtVzOyq0Nz7+F2XYVZ0tGskcTIABikvWIwBEZJGZDxtl0iB1oyM9H3W0ErJ8Sjdjf7pKbrorQ8JnyqmBWws0zWI2Y4wXdadA5kg8kH6vpvsk8JaJ9vvdsbaEGkBEtVhl/Dg4KMluxuPch+EOwEJ8eY+FFm0jxZeHW65zwX6G3QvqlBsGsFbWeXsWrOXdIhhfSI3uUO3WrU3hb08RCIBZi0/JAABaM8AFczO6yk0j3G90i/+9DoqGcH1teF8XMpUEVKRIBgmcq3lAAAAAAAB50YAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADv68CkAAAAAAAAAAAAAAAAisdqUcyVDZgi1ouD/hrZezLNWA0ABAAAAAAAAAAAAAAAAP5/vLIafffsHZUHsJ1mYOEqD6EWAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+];
+
+describe("Blind Deserialize", function () {
+ it("Should deserialize a blind message", function () {
+ for (const c of cases) {
+ const decoded = encoding.b64.decode(c);
+ const vaa = deserialize("Uint8Array", decoded);
+ const actual = deserializePayload("TokenBridge:Transfer", vaa.payload);
+ expect(actual).toBeDefined();
+
+ console.time("exhaustive");
+ const result = exhaustiveDeserialize(vaa.payload);
+ expect(result).toHaveLength(1);
+ console.timeEnd("exhaustive");
+
+ console.time("blind");
+ const blind = blindDeserializePayload(vaa.payload);
+ expect(blind).toHaveLength(1);
+ console.timeEnd("blind");
+ }
+ });
+});

core/definitions/src/vaa/create.ts

@@ -25,6 +25,13 @@ type DynamicProperties<PL extends PayloadLiteral> = LayoutToType<
  DynamicItemsOfLayout<[...typeof baseLayout, PayloadLiteralToPayloadItemLayout<PL>]>
 >;
 
+/**
+ * Create a VAA from a payload literal and a set of dynamic properties.
+ * @param payloadLiteral The payload literal to create a VAA for.
+ * @param vaaData The dynamic properties to include in the VAA.
+ * @returns A VAA with the given payload literal and dynamic properties.
+ * @throws If the dynamic properties do not match the payload literal.
+ */
 export function createVAA<PL extends PayloadLiteral>(
  payloadLiteral: PL,
  vaaData: DynamicProperties<PL>,

core/definitions/src/vaa/functions.ts

@@ -38,6 +38,12 @@ export function payloadLiteralToPayloadItemLayout<PL extends PayloadLiteral>(pay
  } as PayloadLiteralToPayloadItemLayout<PL>;
 }
 
+/**
+ * serialize a VAA to a Uint8Array
+ * @param vaa the VAA to serialize
+ * @returns a Uint8Array representation of the VAA
+ * @throws if the VAA is not valid
+ */
 export function serialize<PL extends PayloadLiteral>(vaa: VAA<PL>): Uint8Array {
  const layout = [
  ...baseLayout,
@@ -46,6 +52,13 @@ export function serialize<PL extends PayloadLiteral>(vaa: VAA<PL>): Uint8Array {
  return serializeLayout(layout, vaa as unknown as LayoutToType<typeof layout>);
 }
 
+/**
+ * serialize a VAA payload to a Uint8Array
+ *
+ * @param payloadLiteral The payload literal to use for serialization
+ * @param payload The dynamic properties to include in the payload
+ * @returns a Uint8Array representation of the VAA Payload
+ */
 export function serializePayload<PL extends PayloadLiteral>(
  payloadLiteral: PL,
  payload: Payload<PL>,
@@ -136,6 +149,15 @@ export function payloadDiscriminator<
 
 type ExtractLiteral<T> = T extends PayloadDiscriminator<infer LL> ? LL : T;
 
+/**
+ * deserialize a VAA from a Uint8Array
+ *
+ * @param payloadDet The payload literal or discriminator to use for deserialization
+ * @param data the data to deserialize
+ * @returns a VAA object with the given payload literal or discriminator
+ * @throws if the data is not a valid VAA
+ */
+
 export function deserialize<T extends PayloadLiteral | PayloadDiscriminator>(
  payloadDet: T,
  data: Byteish,
@@ -188,6 +210,15 @@ type DeserializePayloadReturn<T> = T extends infer PL extends PayloadLiteral
  ? DeserializedPair<LL>
  : never;
 
+/**
+ * deserialize a payload from a Uint8Array
+ *
+ * @param payloadDet the payload literal or discriminator to use for deserialization
+ * @param data the data to deserialize
+ * @param offset the offset to start deserializing from
+ * @returns the deserialized payload
+ * @throws if the data is not a valid payload
+ */
 export function deserializePayload<T extends PayloadLiteral | PayloadDiscriminator>(
  payloadDet: T,
  data: Byteish,
@@ -211,6 +242,42 @@ export function deserializePayload<T extends PayloadLiteral | PayloadDiscriminat
  })() as DeserializePayloadReturn<T>;
 }
 
+/**
+ * Attempt to deserialize a payload from a Uint8Array using all registered layouts
+ *
+ * @param data the data to deserialize
+ * @returns an array of all possible deserialized payloads
+ * @throws if the data is not a valid payload
+ */
+export const exhaustiveDeserialize = (() => {
+ const rebuildDiscrimininator = () => {
+ const layoutLiterals = Array.from(payloadFactory.keys());
+ const layouts = layoutLiterals.map((l) => payloadFactory.get(l)!);
+ return [layoutLiterals, layoutDiscriminator(layouts, true)] as const;
+ };
+
+ let layoutLiterals = [] as LayoutLiteral[];
+
+ return (data: Byteish): readonly DeserializedPair[] => {
+ if (payloadFactory.size !== layoutLiterals.length) [layoutLiterals] = rebuildDiscrimininator();
+
+ const candidates = layoutLiterals;
+ return candidates.reduce((acc, literal) => {
+ try {
+ acc.push([literal, deserializePayload(literal!, data)] as DeserializedPair);
+ } catch {}
+ return acc;
+ }, [] as DeserializedPair[]);
+ };
+})();
+
+/**
+ * Blindly deserialize a payload from a Uint8Array
+ *
+ * @param data the data to deserialize
+ * @returns an array of all possible deserialized payloads
+ * @throws if the data is not a valid payload
+ */
 export const blindDeserializePayload = (() => {
  const rebuildDiscrimininator = () => {
  const layoutLiterals = Array.from(payloadFactory.keys());
@@ -226,7 +293,6 @@ export const blindDeserializePayload = (() => {
  [layoutLiterals, discriminator] = rebuildDiscrimininator();
 
  if (typeof data === "string") data = encoding.hex.decode(data);
-
  const candidates = discriminator(data).map((c) => layoutLiterals[c]);
  return candidates.reduce((acc, literal) => {
  try {