diff --git a/packages/ssz/src/index.ts b/packages/ssz/src/index.ts index cba6b83d..6001704f 100644 --- a/packages/ssz/src/index.ts +++ b/packages/ssz/src/index.ts @@ -16,6 +16,7 @@ export {VectorBasicType} from "./type/vectorBasic"; export {VectorCompositeType} from "./type/vectorComposite"; export {ListUintNum64Type} from "./type/listUintNum64"; export {StableContainerType} from "./type/stableContainer"; +export {SimpleVariantType} from "./type/simpleVariant"; // Base types export {ArrayType} from "./type/array"; diff --git a/packages/ssz/src/type/simpleVariant.ts b/packages/ssz/src/type/simpleVariant.ts new file mode 100644 index 00000000..1da9aa5a --- /dev/null +++ b/packages/ssz/src/type/simpleVariant.ts @@ -0,0 +1,611 @@ +import { + Node, + getNodesAtDepth, + subtreeFillToContents, + Tree, + Gindex, + toGindex, + concatGindices, + getNode, + BranchNode, + zeroNode, +} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; +import {Require} from "../util/types"; +import {namedClass} from "../util/named"; +import {Type, ValueOf} from "./abstract"; +import {CompositeType, ByteViews, CompositeTypeAny} from "./composite"; +import { + getContainerTreeViewClass, + ValueOfFields, + FieldEntry, + ContainerTreeViewType, + ContainerTreeViewTypeConstructor, +} from "../view/simpleVariant"; +import { + getContainerTreeViewDUClass, + ContainerTreeViewDUType, + ContainerTreeViewDUTypeConstructor, +} from "../viewDU/simpleVariant"; +import {Case} from "../util/strings"; +import {BitArray} from "../value/bitArray"; +import {getActiveFields, mixInActiveFields, setActiveFields} from "./stableContainer"; +import {zeroHash} from "../util/zeros"; +/* eslint-disable @typescript-eslint/member-ordering */ + +type BytesRange = {start: number; end: number}; + +/** + * SimpleVariant: ordered heterogeneous collection of values that inherits merkleization from a base stable container + * + * Limitations: + * - No reordering of fields for merkleization + * - No optional fields + */ + +export type SimpleVariantOptions> = { + typeName?: string; + jsonCase?: KeyCase; + casingMap?: CasingMap; + cachePermanentRootStruct?: boolean; + getContainerTreeViewClass?: typeof getContainerTreeViewClass; + getContainerTreeViewDUClass?: typeof getContainerTreeViewDUClass; +}; + +export type KeyCase = + | "eth2" + | "snake" + | "constant" + | "camel" + | "header" + //Same as squish + | "pascal"; + +type CasingMap> = Partial<{[K in keyof Fields]: string}>; + +/** + * Container: ordered heterogeneous collection of values + * - Notation: Custom name per instance + */ +export class SimpleVariantType>> extends CompositeType< + ValueOfFields, + ContainerTreeViewType, + ContainerTreeViewDUType +> { + readonly typeName: string; + readonly depth: number; + readonly maxChunkCount: number; + readonly fixedSize: number | null; + readonly minSize: number; + readonly maxSize: number; + readonly isList = false; + readonly isViewMutable = true; + readonly activeFields: BitArray; + + // Precomputed data for faster serdes + readonly fieldsEntries: FieldEntry[]; + /** End of fixed section of serialized Container */ + readonly fixedEnd: number; + protected readonly fieldsGindex: Record; + protected readonly jsonKeyToFieldName: Record; + protected readonly isFixedLen: boolean[]; + protected readonly fieldRangesFixedLen: BytesRange[]; + /** Offsets position relative to start of serialized Container. Length may not equal field count. */ + protected readonly variableOffsetsPosition: number[]; + + /** Cached TreeView constuctor with custom prototype for this Type's properties */ + protected readonly TreeView: ContainerTreeViewTypeConstructor; + protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; + + constructor(readonly fields: Fields, activeFields: BitArray, readonly opts?: SimpleVariantOptions) { + super(opts?.cachePermanentRootStruct); + + // Render detailed typeName. Consumers should overwrite since it can get long + this.typeName = opts?.typeName ?? renderContainerTypeName(fields); + + if (activeFields.getTrueBitIndexes().length !== Object.keys(fields).length) { + throw new Error("activeFields must have the same number of true bits as fields"); + } + + this.activeFields = activeFields; + this.maxChunkCount = this.activeFields.bitLen; + this.depth = maxChunksToDepth(this.maxChunkCount) + 1; + + // Precalculated data for faster serdes + this.fieldsEntries = []; + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + for (let i = 0, fieldIx = 0; i < this.activeFields.bitLen; i++) { + if (!this.activeFields.get(i)) { + continue; + } + + const fieldName = fieldNames[fieldIx++]; + this.fieldsEntries.push({ + fieldName, + fieldType: this.fields[fieldName], + jsonKey: precomputeJsonKey(fieldName, opts?.casingMap, opts?.jsonCase), + gindex: toGindex(this.depth, BigInt(i)), + chunkIndex: i, + }); + } + + if (this.fieldsEntries.length === 0) { + throw Error("Container must have > 0 fields"); + } + + // Precalculate for Proofs API + this.fieldsGindex = {} as Record; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, chunkIndex} = this.fieldsEntries[i]; + this.fieldsGindex[fieldName] = toGindex(this.depth, BigInt(chunkIndex)); + } + + // To resolve JSON paths in fieldName notation and jsonKey notation + this.jsonKeyToFieldName = {}; + for (const {fieldName, jsonKey} of this.fieldsEntries) { + this.jsonKeyToFieldName[jsonKey] = fieldName; + } + + const {minLen, maxLen, fixedSize} = precomputeSizes(fields); + this.minSize = minLen; + this.maxSize = maxLen; + this.fixedSize = fixedSize; + + const {isFixedLen, fieldRangesFixedLen, variableOffsetsPosition, fixedEnd} = precomputeSerdesData(fields); + this.isFixedLen = isFixedLen; + this.fieldRangesFixedLen = fieldRangesFixedLen; + this.variableOffsetsPosition = variableOffsetsPosition; + this.fixedEnd = fixedEnd; + + // TODO: This options are necessary for ContainerNodeStruct to override this. + // Refactor this constructor to allow customization without pollutin the options + this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); + this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); + } + + static named>>( + fields: Fields, + activeFields: BitArray, + opts: Require, "typeName"> + ): SimpleVariantType { + return new (namedClass(SimpleVariantType, opts.typeName))(fields, activeFields, opts); + } + + defaultValue(): ValueOfFields { + const value = {} as ValueOfFields; + for (const {fieldName, fieldType} of this.fieldsEntries) { + value[fieldName] = fieldType.defaultValue() as ValueOf; + } + return value; + } + + getView(tree: Tree): ContainerTreeViewType { + return new this.TreeView(this, tree); + } + + getViewDU(node: Node, cache?: unknown): ContainerTreeViewDUType { + return new this.TreeViewDU(this, node, cache); + } + + cacheOfViewDU(view: ContainerTreeViewDUType): unknown { + return view.cache; + } + + commitView(view: ContainerTreeViewType): Node { + return view.node; + } + + commitViewDU(view: ContainerTreeViewDUType): Node { + view.commit(); + return view.node; + } + + // Serialization + deserialization + // ------------------------------- + // Containers can mix fixed length and variable length data. + // + // Fixed part Variable part + // [field1 offset][field2 data ][field1 data ] + // [0x000000c] [0xaabbaabbaabbaabb][0xffffffffffffffffffffffff] + + value_serializedSize(value: ValueOfFields): number { + let totalSize = 0; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + // Offset (4 bytes) + size + totalSize += + fieldType.fixedSize === null ? 4 + fieldType.value_serializedSize(value[fieldName]) : fieldType.fixedSize; + } + return totalSize; + } + + value_serializeToBytes(output: ByteViews, offset: number, value: ValueOfFields): number { + let fixedIndex = offset; + let variableIndex = offset + this.fixedEnd; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + if (fieldType.fixedSize === null) { + // write offset + output.dataView.setUint32(fixedIndex, variableIndex - offset, true); + fixedIndex += 4; + // write serialized element to variable section + variableIndex = fieldType.value_serializeToBytes(output, variableIndex, value[fieldName]); + } else { + fixedIndex = fieldType.value_serializeToBytes(output, fixedIndex, value[fieldName]); + } + } + return variableIndex; + } + + value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields { + const fieldRanges = this.getFieldRanges(data.dataView, start, end); + const value = {} as {[K in keyof Fields]: unknown}; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + const fieldRange = fieldRanges[i]; + value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + return value as ValueOfFields; + } + + tree_serializedSize(node: Node): number { + let totalSize = 0; + const nodes = getNodesAtDepth(node, this.depth, 0, this.fieldsEntries.length) as Node[]; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType} = this.fieldsEntries[i]; + const node = nodes[i]; + // Offset (4 bytes) + size + totalSize += fieldType.fixedSize === null ? 4 + fieldType.tree_serializedSize(node) : fieldType.fixedSize; + } + return totalSize; + } + + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { + let fixedIndex = offset; + let variableIndex = offset + this.fixedEnd; + + const nodes = getNodesAtDepth(node, this.depth, 0, this.activeFields.bitLen); + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, chunkIndex} = this.fieldsEntries[i]; + const node = nodes[chunkIndex]; + + if (fieldType.fixedSize === null) { + // write offset + output.dataView.setUint32(fixedIndex, variableIndex - offset, true); + fixedIndex += 4; + // write serialized element to variable section + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node); + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node); + } + } + return variableIndex; + } + + tree_deserializeFromBytes(data: ByteViews, start: number, end: number): Node { + const fieldRanges = this.getFieldRanges(data.dataView, start, end); + const nodes = new Array(this.activeFields.bitLen).fill(zeroNode(0)); + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldType, chunkIndex} = this.fieldsEntries[i]; + const fieldRange = fieldRanges[i]; + nodes[chunkIndex] = fieldType.tree_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + } + + const root = new BranchNode(subtreeFillToContents(nodes, this.depth - 1), zeroNode(0)); + return setActiveFields(root, this.activeFields); + } + + // Merkleization + + hashTreeRoot(value: ValueOfFields): Uint8Array { + const root = super.hashTreeRoot(value); + return mixInActiveFields(root, this.activeFields); + } + + protected getRoots(struct: ValueOfFields): Uint8Array[] { + const roots = new Array(this.activeFields.bitLen).fill(zeroHash(0)); + + // already asserted that # of active fields in bitvector === # of fields + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, chunkIndex} = this.fieldsEntries[i]; + roots[chunkIndex] = fieldType.hashTreeRoot(struct[fieldName]); + } + + return roots; + } + + // Proofs + + // getPropertyGindex + // getPropertyType + // tree_getLeafGindices + /** INTERNAL METHOD: For view's API, create proof from a tree */ + + getPropertyGindex(prop: string): Gindex | null { + const gindex = this.fieldsGindex[prop] ?? this.fieldsGindex[this.jsonKeyToFieldName[prop]]; + if (gindex === undefined) throw Error(`Unknown container property ${prop}`); + return gindex; + } + + getPropertyType(prop: string): Type { + const type = this.fields[prop] ?? this.fields[this.jsonKeyToFieldName[prop]]; + if (type === undefined) throw Error(`Unknown container property ${prop}`); + return type; + } + + getIndexProperty(index: number): string | null { + if (index >= this.fieldsEntries.length) { + return null; + } + return this.fieldsEntries[index].fieldName as string; + } + + tree_getLeafGindices(rootGindex: Gindex, rootNode?: Node): Gindex[] { + const gindices: Gindex[] = []; + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + const fieldGindex = this.fieldsGindex[fieldName]; + const fieldGindexFromRoot = concatGindices([rootGindex, fieldGindex]); + + if (fieldType.isBasic) { + gindices.push(fieldGindexFromRoot); + } else { + const compositeType = fieldType as unknown as CompositeTypeAny; + if (fieldType.fixedSize === null) { + if (!rootNode) { + throw new Error("variable type requires tree argument to get leaves"); + } + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot, getNode(rootNode, fieldGindex))); + } else { + gindices.push(...compositeType.tree_getLeafGindices(fieldGindexFromRoot)); + } + } + } + + return gindices; + } + + // JSON + + fromJson(json: unknown): ValueOfFields { + if (typeof json !== "object") { + throw Error("JSON must be of type object"); + } + if (json === null) { + throw Error("JSON must not be null"); + } + + const value = {} as ValueOfFields; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, jsonKey} = this.fieldsEntries[i]; + const jsonValue = (json as Record)[jsonKey]; + if (jsonValue === undefined) { + throw Error(`JSON expected key ${jsonKey} is undefined`); + } + value[fieldName] = fieldType.fromJson(jsonValue) as ValueOf; + } + + return value; + } + + toJson(value: ValueOfFields): Record { + const json: Record = {}; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType, jsonKey} = this.fieldsEntries[i]; + json[jsonKey] = fieldType.toJson(value[fieldName]); + } + + return json; + } + + clone(value: ValueOfFields): ValueOfFields { + const newValue = {} as ValueOfFields; + + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + newValue[fieldName] = fieldType.clone(value[fieldName]) as ValueOf; + } + + return newValue; + } + + equals(a: ValueOfFields, b: ValueOfFields): boolean { + for (let i = 0; i < this.fieldsEntries.length; i++) { + const {fieldName, fieldType} = this.fieldsEntries[i]; + if (!fieldType.equals(a[fieldName], b[fieldName])) { + return false; + } + } + + return true; + } + + /** + * Deserializer helper: Returns the bytes ranges of all fields, both variable and fixed size. + * Fields may not be contiguous in the serialized bytes, so the returned ranges are [start, end]. + * - For fixed size fields re-uses the pre-computed values this.fieldRangesFixedLen + * - For variable size fields does a first pass over the fixed section to read offsets + */ + getFieldRanges(data: DataView, start: number, end: number): BytesRange[] { + if (this.variableOffsetsPosition.length === 0) { + // Validate fixed length container + const size = end - start; + if (size !== this.fixedEnd) { + throw Error(`${this.typeName} size ${size} not equal fixed size ${this.fixedEnd}`); + } + + return this.fieldRangesFixedLen; + } + + // Read offsets in one pass + const offsets = readVariableOffsets(data, start, end, this.fixedEnd, this.variableOffsetsPosition); + offsets.push(end - start); // The offsets are relative to the start + + // Merge fieldRangesFixedLen + offsets in one array + let variableIdx = 0; + let fixedIdx = 0; + const fieldRanges = new Array(this.isFixedLen.length); + + for (let i = 0; i < this.isFixedLen.length; i++) { + if (this.isFixedLen[i]) { + // push from fixLen ranges ++ + fieldRanges[i] = this.fieldRangesFixedLen[fixedIdx++]; + } else { + // push from varLen ranges ++ + fieldRanges[i] = {start: offsets[variableIdx], end: offsets[variableIdx + 1]}; + variableIdx++; + } + } + return fieldRanges; + } +} + +/** + * Returns the byte ranges of all variable size fields. + */ +function readVariableOffsets( + data: DataView, + start: number, + end: number, + fixedEnd: number, + variableOffsetsPosition: number[] +): number[] { + // Since variable-sized values can be interspersed with fixed-sized values, we precalculate + // the offset indices so we can more easily deserialize the fields in once pass first we get the fixed sizes + // Note: `fixedSizes[i] = null` if that field has variable length + + const size = end - start; + + // with the fixed sizes, we can read the offsets, and store for our single pass + const offsets = new Array(variableOffsetsPosition.length); + for (let i = 0; i < variableOffsetsPosition.length; i++) { + const offset = data.getUint32(start + variableOffsetsPosition[i], true); + + // Validate offsets. If the list is empty the offset points to the end of the buffer, offset == size + if (offset > size) { + throw new Error(`Offset out of bounds ${offset} > ${size}`); + } + if (i === 0) { + if (offset !== fixedEnd) { + throw new Error(`First offset must equal to fixedEnd ${offset} != ${fixedEnd}`); + } + } else { + if (offset < offsets[i - 1]) { + throw new Error(`Offsets must be increasing ${offset} < ${offsets[i - 1]}`); + } + } + + offsets[i] = offset; + } + + return offsets; +} + +/** + * Precompute fixed and variable offsets position for faster deserialization. + * @returns Does a single pass over all fields and returns: + * - isFixedLen: If field index [i] is fixed length + * - fieldRangesFixedLen: For fields with fixed length, their range of bytes + * - variableOffsetsPosition: Position of the 4 bytes offset for variable size fields + * - fixedEnd: End of the fixed size range + * - + */ +function precomputeSerdesData(fields: Record>): { + isFixedLen: boolean[]; + fieldRangesFixedLen: BytesRange[]; + variableOffsetsPosition: number[]; + fixedEnd: number; +} { + const isFixedLen: boolean[] = []; + const fieldRangesFixedLen: BytesRange[] = []; + const variableOffsetsPosition: number[] = []; + let pointerFixed = 0; + + for (const fieldType of Object.values(fields)) { + isFixedLen.push(fieldType.fixedSize !== null); + if (fieldType.fixedSize === null) { + // Variable length + variableOffsetsPosition.push(pointerFixed); + pointerFixed += 4; + } else { + fieldRangesFixedLen.push({start: pointerFixed, end: pointerFixed + fieldType.fixedSize}); + pointerFixed += fieldType.fixedSize; + } + } + + return { + isFixedLen, + fieldRangesFixedLen, + variableOffsetsPosition, + fixedEnd: pointerFixed, + }; +} + +/** + * Precompute sizes of the Container doing one pass over fields + */ +function precomputeSizes(fields: Record>): { + minLen: number; + maxLen: number; + fixedSize: number | null; +} { + let minLen = 0; + let maxLen = 0; + let fixedSize: number | null = 0; + + for (const fieldType of Object.values(fields)) { + minLen += fieldType.minSize; + maxLen += fieldType.maxSize; + + if (fieldType.fixedSize === null) { + // +4 for the offset + minLen += 4; + maxLen += 4; + fixedSize = null; + } else if (fixedSize !== null) { + fixedSize += fieldType.fixedSize; + } + } + return {minLen, maxLen, fixedSize}; +} + +/** + * Compute the JSON key for each fieldName. There will exist a single JSON representation for each type. + * To transform JSON payloads to a casing that is different from the type's defined use external tooling. + */ +export function precomputeJsonKey>>( + fieldName: keyof Fields, + casingMap?: CasingMap, + jsonCase?: KeyCase +): string { + if (casingMap) { + const keyFromCaseMap = casingMap[fieldName]; + if (keyFromCaseMap === undefined) { + throw Error(`casingMap[${fieldName}] not defined`); + } + return keyFromCaseMap as string; + } else if (jsonCase) { + return Case[jsonCase](fieldName as string); + } else { + return fieldName as string; + } +} + +/** + * Render field typeNames for a detailed typeName of this Container + */ +export function renderContainerTypeName>>( + fields: Fields, + prefix = "Container" +): string { + const fieldNames = Object.keys(fields) as (keyof Fields)[]; + const fieldTypeNames = fieldNames.map((fieldName) => `${fieldName}: ${fields[fieldName].typeName}`).join(", "); + return `${prefix}({${fieldTypeNames}})`; +} diff --git a/packages/ssz/src/view/simpleVariant.ts b/packages/ssz/src/view/simpleVariant.ts new file mode 100644 index 00000000..36afe1a8 --- /dev/null +++ b/packages/ssz/src/view/simpleVariant.ts @@ -0,0 +1,137 @@ +import {getNodeAtDepth, Gindex, LeafNode, Node, toGindexBitstring, Tree} from "@chainsafe/persistent-merkle-tree"; +import {Type, ValueOf} from "../type/abstract"; +import {isBasicType, BasicType} from "../type/basic"; +import {isCompositeType, CompositeType} from "../type/composite"; +import {TreeView} from "./abstract"; +import {BitArray} from "../value/bitArray"; + +export type FieldEntry>> = { + fieldName: keyof Fields; + fieldType: Fields[keyof Fields]; + jsonKey: string; + gindex: Gindex; + // added for simple variant + chunkIndex: number; +}; + +/** Expected API of this View's type. This interface allows to break a recursive dependency between types and views */ +export type ContainerTypeGeneric>> = CompositeType< + ValueOfFields, + ContainerTreeViewType, + unknown +> & { + readonly fields: Fields; + readonly fieldsEntries: FieldEntry[]; + readonly fixedEnd: number; + readonly activeFields: BitArray; +}; + +export type ValueOfFields>> = {[K in keyof Fields]: ValueOf}; + +export type FieldsView>> = { + [K in keyof Fields]: Fields[K] extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TV + : // If basic, return struct value. Will NOT propagate changes upwards + Fields[K] extends BasicType + ? V + : never; +}; + +export type ContainerTreeViewType>> = FieldsView & + TreeView>; +export type ContainerTreeViewTypeConstructor>> = { + new (type: ContainerTypeGeneric, tree: Tree): ContainerTreeViewType; +}; + +/** + * Intented usage: + * + * - Get initial BeaconState from disk. + * - Before applying next block, switch to mutable + * - Get some field, create a view in mutable mode + * - Do modifications of the state in the state transition function + * - When done, commit and apply new root node once to og BeaconState + * - However, keep all the caches and transfer them to the new BeaconState + * + * Questions: + * - Can the child views created in mutable mode switch to not mutable? If so, it seems that it needs to recursively + * iterate the entire data structure and views + * + */ +class ContainerTreeView>> extends TreeView> { + constructor(readonly type: ContainerTypeGeneric, readonly tree: Tree) { + super(); + } + + get node(): Node { + return this.tree.rootNode; + } +} + +export function getContainerTreeViewClass>>( + type: ContainerTypeGeneric +): ContainerTreeViewTypeConstructor { + class CustomContainerTreeView extends ContainerTreeView {} + + // Dynamically define prototype methods + for (let index = 0; index < type.fieldsEntries.length; index++) { + const {fieldName, fieldType, chunkIndex} = type.fieldsEntries[index]; + + // If the field type is basic, the value to get and set will be the actual 'struct' value (i.e. a JS number). + // The view must use the tree_getFromNode() and tree_setToNode() methods to persist the struct data to the node, + // and use the cached views array to store the new node. + if (isBasicType(fieldType)) { + Object.defineProperty(CustomContainerTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + + // TODO: Review the memory cost of this closures + get: function (this: CustomContainerTreeView) { + const leafNode = getNodeAtDepth(this.node, this.type.depth, chunkIndex) as LeafNode; + return fieldType.tree_getFromNode(leafNode); + }, + + set: function (this: CustomContainerTreeView, value) { + const leafNodePrev = getNodeAtDepth(this.node, this.type.depth, chunkIndex) as LeafNode; + const leafNode = leafNodePrev.clone(); + fieldType.tree_setToNode(leafNode, value); + this.tree.setNodeAtDepth(this.type.depth, index, leafNode); + }, + }); + } + + // If the field type is composite, the value to get and set will be another TreeView. The parent TreeView must + // cache the view itself to retain the caches of the child view. To set a value the view must return a node to + // set it to the parent tree in the field gindex. + else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomContainerTreeView.prototype, fieldName, { + configurable: false, + enumerable: true, + + // Returns TreeView of fieldName + get: function (this: CustomContainerTreeView) { + const gindex = toGindexBitstring(this.type.depth, chunkIndex); + return fieldType.getView(this.tree.getSubtree(gindex)); + }, + + // Expects TreeView of fieldName + set: function (this: CustomContainerTreeView, value: unknown) { + const node = fieldType.commitView(value); + this.tree.setNodeAtDepth(this.type.depth, chunkIndex, node); + }, + }); + } + + // Should never happen + else { + /* istanbul ignore next - unreachable code */ + throw Error(`Unknown fieldType ${fieldType.typeName} for fieldName ${fieldName}`); + } + } + + // Change class name + Object.defineProperty(CustomContainerTreeView, "name", {value: type.typeName, writable: false}); + + return CustomContainerTreeView as unknown as ContainerTreeViewTypeConstructor; +} diff --git a/packages/ssz/src/viewDU/simpleVariant.ts b/packages/ssz/src/viewDU/simpleVariant.ts new file mode 100644 index 00000000..e3bd1f0d --- /dev/null +++ b/packages/ssz/src/viewDU/simpleVariant.ts @@ -0,0 +1,264 @@ +import {getNodeAtDepth, LeafNode, Node, setNodesAtDepth} from "@chainsafe/persistent-merkle-tree"; +import {ByteViews, Type} from "../type/abstract"; +import {BasicType, isBasicType} from "../type/basic"; +import {CompositeType, isCompositeType, CompositeTypeAny} from "../type/composite"; +import {ContainerTypeGeneric} from "../view/simpleVariant"; +import {TreeViewDU} from "./abstract"; + +/* eslint-disable @typescript-eslint/member-ordering */ + +export type FieldsViewDU>> = { + [K in keyof Fields]: Fields[K] extends CompositeType + ? // If composite, return view. MAY propagate changes updwards + TVDU + : // If basic, return struct value. Will NOT propagate changes upwards + Fields[K] extends BasicType + ? V + : never; +}; + +export type ContainerTreeViewDUType>> = FieldsViewDU & + TreeViewDU>; +export type ContainerTreeViewDUTypeConstructor>> = { + new (type: ContainerTypeGeneric, node: Node, cache?: unknown): ContainerTreeViewDUType; +}; + +type ContainerTreeViewDUCache = { + nodes: Node[]; + caches: unknown[]; + nodesPopulated: boolean; +}; + +class ContainerTreeViewDU>> extends TreeViewDU< + ContainerTypeGeneric +> { + protected nodes: Node[] = []; + protected caches: unknown[]; + protected readonly nodesChanged = new Set(); + protected readonly viewsChanged = new Map(); + private nodesPopulated: boolean; + + constructor( + readonly type: ContainerTypeGeneric, + protected _rootNode: Node, + cache?: ContainerTreeViewDUCache + ) { + super(); + + if (cache) { + this.nodes = cache.nodes; + this.caches = cache.caches; + this.nodesPopulated = cache.nodesPopulated; + } else { + this.nodes = []; + this.caches = []; + this.nodesPopulated = false; + } + } + + get node(): Node { + return this._rootNode; + } + + get cache(): ContainerTreeViewDUCache { + return { + nodes: this.nodes, + caches: this.caches, + nodesPopulated: this.nodesPopulated, + }; + } + + commit(): void { + if (this.nodesChanged.size === 0 && this.viewsChanged.size === 0) { + return; + } + + const nodesChanged: {index: number; node: Node}[] = []; + + for (const [index, view] of this.viewsChanged) { + const fieldType = this.type.fieldsEntries[index].fieldType as unknown as CompositeTypeAny; + const node = fieldType.commitViewDU(view); + // Set new node in nodes array to ensure data represented in the tree and fast nodes access is equal + this.nodes[index] = node; + nodesChanged.push({index, node}); + + // Cache the view's caches to preserve it's data after 'this.viewsChanged.clear()' + const cache = fieldType.cacheOfViewDU(view); + if (cache) this.caches[index] = cache; + } + + for (const index of this.nodesChanged) { + nodesChanged.push({index, node: this.nodes[index]}); + } + + // TODO: Optimize to loop only once, Numerical sort ascending + const nodesChangedSorted = nodesChanged.sort((a, b) => a.index - b.index); + const indexes = nodesChangedSorted.map((entry) => entry.index); + const nodes = nodesChangedSorted.map((entry) => entry.node); + + this._rootNode = setNodesAtDepth(this._rootNode, this.type.depth, indexes, nodes); + + this.nodesChanged.clear(); + this.viewsChanged.clear(); + } + + protected clearCache(): void { + this.nodes = []; + this.caches = []; + this.nodesPopulated = false; + + // Must clear nodesChanged, otherwise a subsequent commit call will break, because it assumes a node is there + this.nodesChanged.clear(); + + // It's not necessary to clear this.viewsChanged since they have no effect on the cache. + // However preserving _SOME_ caches results in a very unpredictable experience. + this.viewsChanged.clear(); + } + + /** + * Same method to `type/container.ts` that call ViewDU.serializeToBytes() of internal fields. + */ + serializeToBytes(output: ByteViews, offset: number): number { + this.commit(); + + let fixedIndex = offset; + let variableIndex = offset + this.type.fixedEnd; + for (let index = 0; index < this.type.fieldsEntries.length; index++) { + const {fieldType, chunkIndex} = this.type.fieldsEntries[index]; + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex); + this.nodes[index] = node; + } + if (fieldType.fixedSize === null) { + // write offset + output.dataView.setUint32(fixedIndex, variableIndex - offset, true); + fixedIndex += 4; + // write serialized element to variable section + // basic types always have fixedSize + if (isCompositeType(fieldType)) { + const view = fieldType.getViewDU(node, this.caches[index]) as TreeViewDU; + if (view.serializeToBytes !== undefined) { + variableIndex = view.serializeToBytes(output, variableIndex); + } else { + // some types don't define ViewDU as TreeViewDU, like the UnionType, in that case view.serializeToBytes = undefined + variableIndex = fieldType.tree_serializeToBytes(output, variableIndex, node); + } + } + } else { + fixedIndex = fieldType.tree_serializeToBytes(output, fixedIndex, node); + } + } + + return variableIndex; + } +} + +export function getContainerTreeViewDUClass>>( + type: ContainerTypeGeneric +): ContainerTreeViewDUTypeConstructor { + class CustomContainerTreeViewDU extends ContainerTreeViewDU {} + + // Dynamically define prototype methods + for (let index = 0; index < type.fieldsEntries.length; index++) { + const {fieldName, fieldType, chunkIndex} = type.fieldsEntries[index]; + + // If the field type is basic, the value to get and set will be the actual 'struct' value (i.e. a JS number). + // The view must use the tree_getFromNode() and tree_setToNode() methods to persist the struct data to the node, + // and use the cached views array to store the new node. + if (isBasicType(fieldType)) { + Object.defineProperty(CustomContainerTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + + // TODO: Review the memory cost of this closures + get: function (this: CustomContainerTreeViewDU) { + // First walk through the tree to get the root node for that index + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex); + this.nodes[index] = node; + } + + return fieldType.tree_getFromNode(node as LeafNode) as unknown; + }, + + set: function (this: CustomContainerTreeViewDU, value) { + // Create new node if current leafNode is not dirty + let nodeChanged: LeafNode; + if (this.nodesChanged.has(index)) { + // TODO: This assumes that node has already been populated + nodeChanged = this.nodes[index] as LeafNode; + } else { + const nodePrev = (this.nodes[index] ?? + getNodeAtDepth(this._rootNode, this.type.depth, chunkIndex)) as LeafNode; + + nodeChanged = nodePrev.clone(); + // Store the changed node in the nodes cache + this.nodes[index] = nodeChanged; + this.nodesChanged.add(index); + } + + fieldType.tree_setToNode(nodeChanged, value); + }, + }); + } + + // If the field type is composite, the value to get and set will be another TreeView. The parent TreeView must + // cache the view itself to retain the caches of the child view. To set a value the view must return a node to + // set it to the parent tree in the field gindex. + else if (isCompositeType(fieldType)) { + Object.defineProperty(CustomContainerTreeViewDU.prototype, fieldName, { + configurable: false, + enumerable: true, + + // Returns TreeViewDU of fieldName + get: function (this: CustomContainerTreeViewDU) { + const viewChanged = this.viewsChanged.get(index); + if (viewChanged) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return viewChanged; + } + + let node = this.nodes[index]; + if (node === undefined) { + node = getNodeAtDepth(this._rootNode, this.type.depth, index); + this.nodes[index] = node; + } + + // Keep a reference to the new view to call .commit on it latter, only if mutable + const view = fieldType.getViewDU(node, this.caches[index]); + if (fieldType.isViewMutable) { + this.viewsChanged.set(index, view); + } + + // No need to persist the child's view cache since a second get returns this view instance. + // The cache is only persisted on commit where the viewsChanged map is dropped. + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return view; + }, + + // Expects TreeViewDU of fieldName + set: function (this: CustomContainerTreeViewDU, view: unknown) { + // When setting a view: + // - Not necessary to commit node + // - Not necessary to persist cache + // Just keeping a reference to the view in this.viewsChanged ensures consistency + this.viewsChanged.set(index, view); + }, + }); + } + + // Should never happen + else { + /* istanbul ignore next - unreachable code */ + throw Error(`Unknown fieldType ${fieldType.typeName} for fieldName ${fieldName}`); + } + } + + // Change class name + Object.defineProperty(CustomContainerTreeViewDU, "name", {value: type.typeName, writable: false}); + + return CustomContainerTreeViewDU as unknown as ContainerTreeViewDUTypeConstructor; +} diff --git a/packages/ssz/test/unit/byType/runTypeProofTest.ts b/packages/ssz/test/unit/byType/runTypeProofTest.ts index 1496b7ca..dd501498 100644 --- a/packages/ssz/test/unit/byType/runTypeProofTest.ts +++ b/packages/ssz/test/unit/byType/runTypeProofTest.ts @@ -1,6 +1,15 @@ import {Node} from "@chainsafe/persistent-merkle-tree"; import {expect} from "chai"; -import {BitArray, ContainerType, fromHexString, JsonPath, OptionalType, StableContainerType, Type} from "../../../src"; +import { + BitArray, + ContainerType, + fromHexString, + JsonPath, + OptionalType, + SimpleVariantType, + StableContainerType, + Type, +} from "../../../src"; import {CompositeTypeAny, isCompositeType} from "../../../src/type/composite"; import {ArrayBasicTreeView} from "../../../src/view/arrayBasic"; import {RootHex} from "../../lodestarTypes"; @@ -111,7 +120,7 @@ function getJsonPathView(type: Type, view: unknown, jsonPath: JsonPath) if (typeof jsonProp === "number") { view = (view as ArrayBasicTreeView).get(jsonProp); } else if (typeof jsonProp === "string") { - if (type instanceof ContainerType || type instanceof StableContainerType) { + if (type instanceof ContainerType || type instanceof StableContainerType || type instanceof SimpleVariantType) { // Coerce jsonProp to a fieldName. JSON paths may be in JSON notation or fieldName notation const fieldName = (type as ContainerType>>)["jsonKeyToFieldName"][jsonProp] ?? jsonProp; @@ -143,7 +152,7 @@ function getJsonPathValue(type: Type, json: unknown, jsonPath: JsonPath if (typeof jsonProp === "number") { json = (json as unknown[])[jsonProp]; } else if (typeof jsonProp === "string") { - if (type instanceof ContainerType || type instanceof StableContainerType) { + if (type instanceof ContainerType || type instanceof StableContainerType || type instanceof SimpleVariantType) { if ((type as ContainerType>>)["jsonKeyToFieldName"][jsonProp] === undefined) { throw Error(`Unknown jsonProp ${jsonProp} for type ${type.typeName}`); } diff --git a/packages/ssz/test/unit/byType/simpleVariant/valid.test.ts b/packages/ssz/test/unit/byType/simpleVariant/valid.test.ts new file mode 100644 index 00000000..4ea7725d --- /dev/null +++ b/packages/ssz/test/unit/byType/simpleVariant/valid.test.ts @@ -0,0 +1,49 @@ +import {BitArray, SimpleVariantType, UintNumberType, hash64} from "../../../../src"; +import {runTypeTestValid} from "../runTypeTestValid"; + +// taken from eip spec tests + +const uint16 = new UintNumberType(2); +const byteType = new UintNumberType(1); + +const Circle1 = new SimpleVariantType( + { + side: uint16, + color: byteType, + }, + BitArray.fromBoolArray([true, true, false, false]) +); + +const Square1 = new SimpleVariantType( + { + color: byteType, + radius: uint16, + }, + BitArray.fromBoolArray([false, true, true, false]) +); + +runTypeTestValid({ + type: Circle1, + defaultValue: {side: 0, color: 0}, + values: [ + { + id: "circle1-0", + serialized: "0x420001", + json: {side: 0x42, color: 1}, + root: "0xbfdb6fda9d02805e640c0f5767b8d1bb9ff4211498a5e2d7c0f36e1b88ce57ff", + }, + ], +}); + +runTypeTestValid({ + type: Square1, + defaultValue: {color: 0, radius: 0}, + values: [ + { + id: "square1-0", + serialized: "0x014200", + json: {color: 1, radius: 0x42}, + root: "0xf66d2c38c8d2afbd409e86c529dff728e9a4208215ca20ee44e49c3d11e145d8", + }, + ], +});