diff --git a/geth-utils/gethutil/mpt/trie/stacktrie.go b/geth-utils/gethutil/mpt/trie/stacktrie.go index b1ac92076a3..9e2ff214cf8 100644 --- a/geth-utils/gethutil/mpt/trie/stacktrie.go +++ b/geth-utils/gethutil/mpt/trie/stacktrie.go @@ -642,6 +642,13 @@ func (sp *StackProof) GetNibblesC() [][]byte { return sp.nibblesC } +func (sp *StackProof) GetTypeS() []byte { + return sp.proofSType +} +func (sp *StackProof) GetTypeC() []byte { + return sp.proofCType +} + func isBranch(proofEl []byte) bool { elems, _, _ := rlp.SplitList(proofEl) c, _ := rlp.CountValues(elems) @@ -712,7 +719,7 @@ func (st *StackTrie) UpdateAndGetProof(db ethdb.KeyValueReader, indexBuf, value printProof(proofC, typesC, indexBuf) // fmt.Println(len1, len2) - if len1 >= len2 { + if len1 > len2 { fmt.Println(KeybytesToHex(indexBuf)) } @@ -772,7 +779,7 @@ func (st *StackTrie) UpdateAndGetProofs(db ethdb.KeyValueReader, list types.Deri func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, [][]byte, []uint8, error) { k := KeybytesToHex(key) - // fmt.Println("k", k) + fmt.Println(" k", k) if st.nodeType == emptyNode { return [][]byte{}, nil, []uint8{emptyNode}, nil } @@ -794,15 +801,13 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, [] var proof [][]byte var nodes []*StackTrie c := st - // isHashed := false for i := 0; i < len(k); i++ { - // fmt.Print(k[i], "/", c.nodeType, " | ") + // fmt.Print(" ", k[i], "/", c.nodeType, " | ") proofType = append(proofType, c.nodeType) if c.nodeType == extNode { nodes = append(nodes, c) c = c.children[0] - } else if c.nodeType == branchNode { nodes = append(nodes, c) c = c.children[k[i]] @@ -813,7 +818,6 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, [] nodes = append(nodes, c) break } else if c.nodeType == hashedNode { - // isHashed = true c_rlp, error := db.Get(c.val) if error != nil { panic(error) @@ -836,10 +840,11 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, [] // (`branchChild = [128]` in this case), but it is also of length 1 when `c_rlp` is a leaf. if len(branchChild) == 1 && (branchChild[0] == 128 || branchChild[0] == 0) { // no child at this position (128 is RLP encoding for nil object) - fmt.Println("NOT CHILD") break } c.val = branchChild + // if there are children, the node type should be branch + proofType[i] = branchNode } } @@ -848,18 +853,17 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, [] // to them - which is needed in MPT proof, because we need a proof for each modification (after // the first modification, some nodes are hashed and we cannot add children to the hashed node). - // if !isHashed { lNodes := len(nodes) for i := lNodes - 1; i >= 0; i-- { node := nodes[i] if node.nodeType == leafNode { + nibbles = append(nibbles, []byte{}) rlp, error := db.Get(node.val) if error != nil { // TODO: avoid error when RLP proof = append(proof, node.val) // already have RLP } else { proof = append(proof, rlp) } - nibbles = append(nibbles, nil) } else if node.nodeType == branchNode || node.nodeType == extNode { node.hash(false) @@ -884,21 +888,18 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, [] for i := 0; i < int(numNibbles); i++ { nibble[i] = raw_rlp[i+1] - 16 } - // fmt.Println(" Ext nibble:", numNibbles, nibble, raw_rlp) + // fmt.Println(" Ext nibble:", numNibbles, nibble) nibbles = append(nibbles, nibble) } else { - nibbles = append(nibbles, nil) + nibbles = append(nibbles, []byte{}) } } } - // if isHashed { - // proof = append(proof, hashedNodeProof) - - // } // The proof is now reversed (only for non-hashed), // let's reverse it back to have the leaf at the bottom: slices.Reverse(proof) + slices.Reverse(nibbles) // } return proof, nibbles, proofType, nil diff --git a/geth-utils/gethutil/mpt/witness/gen_witness_transactions_test.go b/geth-utils/gethutil/mpt/witness/gen_witness_transactions_test.go index c7d77cab721..6e2d5bb4b01 100644 --- a/geth-utils/gethutil/mpt/witness/gen_witness_transactions_test.go +++ b/geth-utils/gethutil/mpt/witness/gen_witness_transactions_test.go @@ -120,13 +120,9 @@ func transactionsStackTrieInsertionTemplate(t *testing.T, n int) { } } -func TestTransaction144(t *testing.T) { - txs := makeTransactions(146) - prepareStackTrieWitness("TransactionInsertion", types.Transactions(txs)) -} - func TestTransactionInsertion(t *testing.T) { - txs := makeTransactions(145) + // TODO: check drifted_index at 128 + txs := makeTransactions(130) prepareStackTrieWitness("TransactionInsertion", types.Transactions(txs)) } @@ -198,7 +194,7 @@ func batchedTransactionsStackTrieProofTemplate(n int) { var indexBuf []byte indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(1)) - proofS, _, err := stackTrie.GetProof(db, indexBuf) + proofS, _, _, err := stackTrie.GetProof(db, indexBuf) if err != nil { fmt.Println(err) return diff --git a/geth-utils/gethutil/mpt/witness/prepare_witness.go b/geth-utils/gethutil/mpt/witness/prepare_witness.go index fe19dcdaed4..c51a803b199 100644 --- a/geth-utils/gethutil/mpt/witness/prepare_witness.go +++ b/geth-utils/gethutil/mpt/witness/prepare_witness.go @@ -302,8 +302,11 @@ func prepareStackTrieWitness(testName string, list types.DerivableList) { var nodes []Node // for i, proof := range proofs { i := len(proofs) - 2 + if len(proofs) > 128 { + i = len(proofs) - 1 + } proof := proofs[i] - idx := i + 1 + idx := i //+ 1 nodes = append(nodes, GetStartNode(testName, common.Hash{}, root, 0)) var node []Node if (i <= 0x7f && len(proofs)-1 == i) || i == 127 { @@ -317,6 +320,44 @@ func prepareStackTrieWitness(testName string, list types.DerivableList) { nodes = append(nodes, GetEndNode()) // } StoreNodes(testName, nodes) + + // check + verifyNodeNumber(nodes, proof) +} + +// For quick verification the json data. +// will be removed before merge. +func verifyNodeNumber(nodes []Node, proof trie.StackProof) { + // start and end nodes + nodeNum := len(nodes) - 2 + + proofS := proof.GetProofS() + proofC := proof.GetProofC() + len1 := len(proofS) + len2 := len(proofC) + maxLen := max(len1, len2) + minLen := max(len1, len2) + + if maxLen == minLen+1 { + if nodeNum != maxLen { + fmt.Println("WARNING: node number not matched: nodeNum != maxLen") + } + } else if maxLen == minLen { + // [EXT - BRANCH] -> [BRANCH - LEAF] + typeS := proof.GetTypeS() + typeC := proof.GetTypeC() + if typeS[0] != typeC[0] && nodeNum == maxLen+1 { + fmt.Println("WARNING: node number not matched: typeS[0] != typeC[0] && nodeNum == maxLen+1") + } + } else if maxLen > minLen+1 { + // usually it happens when a new ext. node created + // [BRANCH - BRANCH - LEAF] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] + if nodeNum == maxLen+1 { + fmt.Println("WARNING: node number not matched: typeS[0] != typeC[0] && nodeNum == maxLen+1") + } + } else { + fmt.Println("WHERE AM I??") + } } // prepareWitness obtains the GetProof proof before and after the modification for each @@ -339,20 +380,22 @@ func prepareWitnessSpecial(testName string, trieModifications []TrieModification // For stack trie, we have the following combinations ([proofS] -> [proofC]) // -// -[x] [(empty)] -> [LEAF] --> 1 -// -[] [LEAF] -> [EXT - BRANCH - LEAF] --> 2 -// -[x] [EXT - BRANCH] -> [EXT - BRANCH - LEAF] --> "< 16" -// -[] [BRANCH - BRANCH] -> [BRANCH - BRANCH - LEAF] --> "< 127" -// -[] [BRANCH - LEAF] -> [BRANCH - BRANCH - LEAF] --> 128 -// -[] [BRANCH] -> [BRANCH - LEAF] --> 0 -// -[] [EXT - BRANCH] -> [BRANCH - LEAF] (modified extension) --> 0 under 16 txs -// -[] [BRANCH - BRANCH - LEAF] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] --> 129 -// -[] [BRANCH - BRANCH - EXT - BRANCH] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] --> 130 -// -[] [BRANCH - BRANCH - EXT - BRANCH - HASHED] -> [BRANCH - BRANCH - BRANCH - LEAF] --> 144 -// -[x] [BRANCH - BRANCH - (...BRANCH)] -> [BRANCH - BRANCH - (...BRANCH) - LEAF] --> 146 ~ 176 -// -[] [BRANCH - BRANCH - EXT - BRANCH - LEAF] -> [BRANCH - BRANCH - EXT - BRANCH - HASHED - HASHED] --> 258 -// -[] [BRANCH - BRANCH - EXT - BRANCH - HASHED - HASHED] -> [BRANCH - BRANCH - EXT - BRANCH - HASHED - HASHED] --> 259 -// -[] [BRANCH - BRANCH - EXT - BRANCH - HASHED - HASHED - HASHED] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] --> 512 +// -[o] [(empty)] -> [LEAF] --> 1 +// -[o] [LEAF] -> [EXT - BRANCH - LEAF] --> 2 +// -[o] [EXT - BRANCH] -> [EXT - BRANCH - LEAF] --> "< 16" +// -[M] [EXT - BRANCH] -> [BRANCH - LEAF] --> 16 (modified ext.) +// -[o] [BRANCH - BRANCH] -> [BRANCH - BRANCH - LEAF] --> "< 127" +// -[o] [BRANCH - LEAF] -> [BRANCH - BRANCH - LEAF] --> 129 +// -[o] [BRANCH] -> [BRANCH - LEAF] --> 0 +// -[M] [EXT - BRANCH] -> [BRANCH - LEAF] (modified extension) --> 0 under 16 txs +// -[o] [BRANCH - BRANCH - LEAF] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] --> 129 +// -[o] [BRANCH - BRANCH - EXT - BRANCH] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] --> 130 +// -[H] [BRANCH - BRANCH - EXT - BRANCH - HASHED] -> [BRANCH - BRANCH - BRANCH - LEAF] --> 144 (hashed node) +// -[o] [BRANCH - BRANCH - (...BRANCH)] -> [BRANCH - BRANCH - (...BRANCH) - LEAF] --> 146 ~ 176 +// +// -[] [BRANCH - BRANCH - EXT - BRANCH - LEAF] -> [BRANCH - BRANCH - EXT - BRANCH - BRANCH - HASHED] --> 258 +// -[] [BRANCH - BRANCH - EXT - BRANCH - BRANCH - HASHED] -> [BRANCH - BRANCH - EXT - BRANCH - BRANCH - HASHED] --> 259~ +// -[] [BRANCH - BRANCH - EXT - BRANCH - BRANCH - BRANCH - HASHED] -> [BRANCH - BRANCH - EXT - BRANCH - LEAF] --> 512 // -[] [[BRANCH - BRANCH - EXT - BRANCH - EXT - BRANCH] -> [BRANCH - BRANCH - EXT - BRANCH - EXT - BRANCH - LEAF] --> 514~ func GenerateWitness(txIdx uint, key, value []byte, proof *trie.StackProof) []Node { k := trie.KeybytesToHex(key) @@ -360,18 +403,24 @@ func GenerateWitness(txIdx uint, key, value []byte, proof *trie.StackProof) []No // padding k to 32 bytes kk := make([]byte, 32-len(k)) k = append(k, kk...) - fmt.Println("txIdx", txIdx) + fmt.Println("== txIdx", txIdx) + fmt.Println(" k", k) proofS := proof.GetProofS() proofC := proof.GetProofC() extNibblesS := proof.GetNibblesS() extNibblesC := proof.GetNibblesC() + proofTypeS := proof.GetTypeS() + proofTypeC := proof.GetTypeC() keyIndex := 0 len1 := len(proofS) len2 := len(proofC) minLen := min(len1, len2) + lastProofTypeS := proofTypeS[len1-1] + lastProofTypeC := proofTypeC[len2-1] + var nodes []Node // Empty stack trie if len1 == 0 { @@ -379,55 +428,53 @@ func GenerateWitness(txIdx uint, key, value []byte, proof *trie.StackProof) []No return append(nodes, leafNode) } - upTo := minLen - 1 - is_end_with_leaf := true - isStartWithExt := false + // FIXME: using enum(branch, leaf...) to replace magic numbers + upTo := minLen additionalBranch := true if len1 > 0 { - is_end_with_leaf = isTxLeaf(proofS[len1-1]) - - if isTxExt(proofS[0]) { - isStartWithExt = true + // e.g. [BRANCH - LEAF] --> [BRANCH - BRANCH - LEAF] + // upTo minus 1 means we handle branch placeholder and leaf in `additionalBranch` + if len1 != len2 && lastProofTypeS == lastProofTypeC && lastProofTypeS == 3 { + upTo-- } // The length of proofS and proofC is equal and - // the last element of proofC is an extension or a leaf - if len1 == len2 && (isTxExt(proofC[len2-1]) || isTxLeaf(proofC[len2-1])) { + // the last element of proofS is a hashed node or a leaf + if len1 == len2 && (lastProofTypeS == 3 || lastProofTypeS == 4) { additionalBranch = false } - // isStartWithExt && len1 > 1 is for "[EXT - BRANCH] -> [EXT - BRANCH - LEAF]"" - if (isStartWithExt && len1 > 1) || (len1 == len2 && isTxLeaf(proofS[len1-1]) && isTxLeaf(proofC[len2-1])) { - upTo = minLen + // Special case for the 2nd tx. + // In this case, proofS only contains a leaf node and proofC are [EXT - BRANCH - LEAF]. + // `additionalBranch` can handle the mismatched the order of the type. + if len1 == 1 && lastProofTypeS == 3 { + upTo = 0 } } - isExtension := false - var extListRlpBytes []byte var extValues [][]byte for i := 0; i < 4; i++ { extValues = append(extValues, make([]byte, 34)) } - fmt.Println("upto", upTo) + isExtension := false + fmt.Println("upto", upTo, additionalBranch) for i := 0; i < upTo; i++ { - if !isBranch(proofS[i]) { - fmt.Println("extNibbleS/C", extNibblesS, extNibblesC) - areThereNibbles := len(extNibblesS[i]) != 0 + if proofTypeS[i] != 1 { + fmt.Println("extNibbleS/C", extNibblesS, "|", extNibblesC) + areThereNibbles := len(extNibblesS[i]) != 0 || len(extNibblesC[i]) != 0 if areThereNibbles { // extension node var numberOfNibbles byte isExtension = true - // FIXME: handle the case of proofS(ext), proofC(branch), and it needs to add a branch placeholder at proofS? numberOfNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesS[i], proofS[i], proofC[i]) - keyIndex += int(numberOfNibbles) fmt.Println("Increase keyIdx", keyIndex) continue } - node := prepareTxLeafNode(txIdx+uint(i), proofS[len1-1], proofC[len2-1], k, nil, false, false, false) + node := prepareTxLeafNode(txIdx, proofS[len1-1], proofC[len2-1], k, nil, false, false, false) nodes = append(nodes, node) } else { var extNode1 []byte = nil @@ -441,35 +488,56 @@ func GenerateWitness(txIdx uint, key, value []byte, proof *trie.StackProof) []No proofS[i], proofC[i], extNode1, extNode2, extListRlpBytes, extValues, k[keyIndex], k[keyIndex], false, false, isExtension) - nodes = append(nodes, bNode) + nodes = append(nodes, bNode) keyIndex += 1 - isExtension = false } } // To address the length of proofS and proofC is not equal or the order of the type is matched. if additionalBranch { - fmt.Println("extNibbleS/C", extNibblesS, extNibblesC) - leafRow0 := proofS[0] // To compute the drifted position. - isModifiedExtNode, _, _, bNode := - addBranchAndPlaceholder( - proofS, proofC, extNibblesS[len1-1], extNibblesC[len2-1], - leafRow0, k, keyIndex, is_end_with_leaf) - - if !isStartWithExt { - nodes = append(nodes, bNode) + fmt.Println("additionalBranch: extNibbleS/C", extNibblesS, "|", extNibblesC) + // This is a special case when the number of txs is 2. + // In this case, proofS is a leaf and len1 is 1 but there is no nibbles + var lastExtNibbleS []byte + if len(extNibblesS) != 0 { + lastExtNibbleS = extNibblesS[len1-1] + } + + leafRow0 := proofS[len1-1] // To compute the drifted position. + if len1 > len2 { + leafRow0 = proofC[len2-1] + } + + // In most of cases, proofs are like this [BRANCH - (BRANCH, EXT)] -> [BRANCH - (BRANCH, EXT) - LEAF] + // That means proofC only appends a leaf node (or a hashed node in some cases) to proofS. + // In these cases, we don't need to add a placeholder branch + need_tx_placeholder := false + need_tx_placeholder = len1 == len2-1 && (lastProofTypeS != lastProofTypeC) && (lastProofTypeC == 3) + + isModifiedExtNode := false + if !need_tx_placeholder { + var branchNode Node + is_end_with_leaf := lastProofTypeS == 3 + if len2 < len1 { + is_end_with_leaf = lastProofTypeC == 3 + } + isModifiedExtNode, _, _, branchNode = + addBranchAndPlaceholder( + proofS, proofC, lastExtNibbleS, extNibblesC[len2-1], + leafRow0, k, keyIndex, is_end_with_leaf) + nodes = append(nodes, branchNode) } var leafNode Node // Add a tx leaf after branch placeholder if !isModifiedExtNode { - // if the last element of proofS is a leaf, it should be covered above - if is_end_with_leaf { - return nodes + if need_tx_placeholder { + leafNode = prepareTxLeafAndPlaceholderNode(txIdx, proofC[len2-1], k, false) + } else { + leafNode = prepareTxLeafNode(txIdx, proofS[len1-1], proofC[len2-1], k, nil, isBranch(proofS[len1-1]), false, false) } - leafNode = prepareTxLeafNode(txIdx, proofS[len1-1], proofC[len2-1], k, nil, isBranch(proofS[len1-1]), false, false) } else { fmt.Println("MODIFIED EXT CASE, IGNORE NOW!!") // TODO might not fit our case bcs we have [EXT - BRANCH] --> [BRANCH - LEAF]