From 0a0dd8fd49a3731cc35da3be9f61868e58c08c53 Mon Sep 17 00:00:00 2001 From: Robin Salen Date: Tue, 11 Jun 2024 17:12:45 -0400 Subject: [PATCH 1/2] feat: Pass original txn RLP encodings --- Cargo.lock | 55 +++++--- Cargo.toml | 6 +- eth_test_parser/src/deserialize.rs | 187 +++++++--------------------- eth_test_parser/src/trie_builder.rs | 62 ++++----- 4 files changed, 113 insertions(+), 197 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ee812a..36340ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,9 +681,8 @@ dependencies = [ [[package]] name = "evm_arithmetization" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2af65f5b147f04c94df5df5e21ff9d1632fd357511e183d5e04de059d6fa93" +version = "0.2.0" +source = "git+https://github.com/0xPolygonZero/zk_evm?branch=prep-release#eebbf3bf1417adce62c046032607af20b3dbacc7" dependencies = [ "anyhow", "bytes", @@ -1032,6 +1031,17 @@ dependencies = [ "parity-scale-codec", ] +[[package]] +name = "impl-num-traits" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951641f13f873bff03d4bf19ae8bec531935ac0ac2cc775f84d7edfdcfed3f17" +dependencies = [ + "integer-sqrt", + "num-traits", + "uint", +] + [[package]] name = "impl-rlp" version = "0.3.0" @@ -1084,6 +1094,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "integer-sqrt" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276ec31bcb4a9ee45f58bec6f9ec700ae4cf4f4f8f2fa7e06cb406bd5ffdd770" +dependencies = [ + "num-traits", +] + [[package]] name = "io-lifetimes" version = "1.0.10" @@ -1281,14 +1300,17 @@ dependencies = [ [[package]] name = "mpt_trie" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf6d77f630021e46e127abfa047aebfba78bf207ed3dfd1c4f9e2370f9b60cd" +version = "0.3.0" +source = "git+https://github.com/0xPolygonZero/zk_evm?branch=prep-release#eebbf3bf1417adce62c046032607af20b3dbacc7" dependencies = [ "bytes", "enum-as-inner", "ethereum-types", "hex", + "impl-codec", + "impl-num-traits", + "impl-rlp", + "impl-serde", "keccak-hash 0.10.0", "log", "num", @@ -1560,9 +1582,8 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "plonky2" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b25deb9a4b9c2014c2f99cd36078f30e453d188d0ca8dd4c5ffd1d494b661ac1" +version = "0.2.2" +source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" dependencies = [ "ahash", "anyhow", @@ -1585,9 +1606,8 @@ dependencies = [ [[package]] name = "plonky2_field" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465a741ba134485af571152aab5086457a470aa8893391186cf78dd389694440" +version = "0.2.2" +source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" dependencies = [ "anyhow", "itertools", @@ -1602,8 +1622,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" +source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" dependencies = [ "rayon", ] @@ -1611,8 +1630,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16136f5f3019c1e83035af76cccddd56d789a5e2933306270185c3f99f12259" +source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" [[package]] name = "portable-atomic" @@ -1972,9 +1990,8 @@ dependencies = [ [[package]] name = "starky" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e0a1eec739c7a67cb1c6f916c0b7bf2d281cf2edb35d3db5caa6989090133e" +version = "0.4.0" +source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" dependencies = [ "ahash", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index a72d5d4..27ad1b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,9 @@ serde_cbor = "0.11.2" tokio = { version = "1.28.1" } # zk-evm dependencies -plonky2 = "0.2.0" -mpt_trie = "0.1.1" -evm_arithmetization = "0.1.1" +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2", branch = "main" } +mpt_trie = { git = "https://github.com/0xPolygonZero/zk_evm", branch = "prep-release" } +evm_arithmetization = { git = "https://github.com/0xPolygonZero/zk_evm", branch = "prep-release" } [profile.release] opt-level = 3 diff --git a/eth_test_parser/src/deserialize.rs b/eth_test_parser/src/deserialize.rs index 4239255..c3984ae 100644 --- a/eth_test_parser/src/deserialize.rs +++ b/eth_test_parser/src/deserialize.rs @@ -4,11 +4,10 @@ use anyhow::Result; use bytes::Bytes; use ethereum_types::{Address, H160, H256, U256}; use evm_arithmetization::generation::mpt::transaction_testing::{ - AccessListItemRlp, AccessListTransactionRlp, AddressOption, FeeMarketTransactionRlp, - LegacyTransactionRlp, + AddressOption, LegacyTransactionRlp, }; use hex::FromHex; -use rlp::{Decodable, DecoderError, Encodable, Rlp, RlpStream}; +use rlp::{Decodable, DecoderError, Rlp}; use rlp_derive::RlpDecodable; use serde::de::MapAccess; use serde::{ @@ -109,32 +108,21 @@ pub(crate) struct BlockHeader { // and hence they require a specific handling. #[derive(Clone, Debug, RlpDecodable)] pub struct AccessItemRlp { - pub address: Address, - pub storage_keys: Vec, + _address: Address, + _storage_keys: Vec, } #[derive(Clone, Debug)] -pub struct StorageKey(pub U256); +pub struct StorageKey; impl Decodable for StorageKey { fn decode(rlp: &Rlp) -> Result { - // Decode the key as a `Vec` to deal with badly encoded scalars, - // and then convert back to U256. - let key = rlp.as_val::>()?; - if key.len() == 1 && key[0] == 0x80 { - return Ok(StorageKey(U256::zero())); - } + // We just need to decode the key as a `Vec` + // to deal with badly encoded scalars, but we do + // not care about the result. + let _key = rlp.as_val::>()?; - Ok(StorageKey(U256::from_big_endian(&key))) - } -} - -impl AccessItemRlp { - fn into_regular(self) -> AccessListItemRlp { - AccessListItemRlp { - address: self.address, - storage_keys: self.storage_keys.iter().map(|k| k.0).collect(), - } + Ok(Self) } } @@ -162,129 +150,55 @@ impl Decodable for Transactions { // A custom type-1 txn to handle some edge-cases with the access_list field. #[derive(RlpDecodable, Debug, Clone)] pub struct CustomAccessListTransactionRlp { - pub chain_id: u64, - pub nonce: U256, - pub gas_price: U256, - pub gas: U256, - pub to: AddressOption, - pub value: U256, - pub data: Bytes, - pub access_list: Vec, - pub y_parity: U256, - pub r: U256, - pub s: U256, -} - -impl CustomAccessListTransactionRlp { - fn into_regular(self) -> AccessListTransactionRlp { - AccessListTransactionRlp { - chain_id: self.chain_id, - nonce: self.nonce, - gas_price: self.gas_price, - gas: self.gas, - to: self.to.clone(), - value: self.value, - data: self.data.clone(), - access_list: self - .access_list - .clone() - .into_iter() - .map(|x| x.into_regular()) - .collect(), - y_parity: self.y_parity, - r: self.r, - s: self.s, - } - } + _chain_id: u64, + _nonce: U256, + _gas_price: U256, + _gas: U256, + _to: AddressOption, + _value: U256, + _data: Bytes, + _access_list: Vec, + _y_parity: U256, + _r: U256, + _s: U256, } // A custom type-2 txn to handle some edge-cases with the access_list field. #[derive(RlpDecodable, Debug, Clone)] pub struct CustomFeeMarketTransactionRlp { - pub chain_id: u64, - pub nonce: U256, - pub max_priority_fee_per_gas: U256, - pub max_fee_per_gas: U256, - pub gas: U256, - pub to: AddressOption, - pub value: U256, - pub data: Bytes, - pub access_list: Vec, - pub y_parity: U256, - pub r: U256, - pub s: U256, -} - -impl CustomFeeMarketTransactionRlp { - fn into_regular(self) -> FeeMarketTransactionRlp { - FeeMarketTransactionRlp { - chain_id: self.chain_id, - nonce: self.nonce, - max_priority_fee_per_gas: self.max_priority_fee_per_gas, - max_fee_per_gas: self.max_fee_per_gas, - gas: self.gas, - to: self.to.clone(), - value: self.value, - data: self.data.clone(), - access_list: self - .access_list - .clone() - .into_iter() - .map(|x| x.into_regular()) - .collect(), - y_parity: self.y_parity, - r: self.r, - s: self.s, - } - } + _chain_id: u64, + _nonce: U256, + _max_priority_fee_per_gas: U256, + _max_fee_per_gas: U256, + _gas: U256, + _to: AddressOption, + _value: U256, + _data: Bytes, + _access_list: Vec, + _y_parity: U256, + _r: U256, + _s: U256, } #[derive(Clone, Debug)] -pub enum Transaction { - Legacy(LegacyTransactionRlp), - AccessList(AccessListTransactionRlp), - FeeMarket(FeeMarketTransactionRlp), -} +pub struct Transaction(pub Vec); impl Transaction { fn decode_actual_rlp(bytes: &[u8]) -> Result { let first_byte = bytes.first().ok_or(DecoderError::RlpInvalidLength)?; match *first_byte { 1 => CustomAccessListTransactionRlp::decode(&Rlp::new(&bytes[1..])) - .map(|tx| Transaction::AccessList(tx.into_regular())), + .map(|_| Self(bytes.to_vec())), 2 => CustomFeeMarketTransactionRlp::decode(&Rlp::new(&bytes[1..])) - .map(|tx| Transaction::FeeMarket(tx.into_regular())), - _ => LegacyTransactionRlp::decode(&Rlp::new(bytes)).map(Transaction::Legacy), + .map(|_| Self(bytes.to_vec())), + _ => LegacyTransactionRlp::decode(&Rlp::new(bytes)).map(|_| Self(bytes.to_vec())), } } } -impl Encodable for Transaction { - fn rlp_append(&self, s: &mut RlpStream) { - match self { - Transaction::Legacy(tx) => s.append(tx), - Transaction::AccessList(tx) => { - s.encoder().encode_value(&[0x01]); - s.append(tx) - } - Transaction::FeeMarket(tx) => { - s.encoder().encode_value(&[0x02]); - s.append(tx) - } - }; - } -} - impl Decodable for Transaction { fn decode(rlp: &Rlp) -> Result { - let first_byte = rlp.as_raw().first().ok_or(DecoderError::RlpInvalidLength)?; - let attempt = match *first_byte { - 1 => CustomAccessListTransactionRlp::decode(&Rlp::new(&rlp.as_raw()[1..])) - .map(|tx| Transaction::AccessList(tx.into_regular())), - 2 => CustomFeeMarketTransactionRlp::decode(&Rlp::new(&rlp.as_raw()[1..])) - .map(|tx| Transaction::FeeMarket(tx.into_regular())), - _ => LegacyTransactionRlp::decode(rlp).map(Transaction::Legacy), - }; + let attempt = Transaction::decode_actual_rlp(rlp.as_raw()); // Somes tests have a different format and store the RLP encoding of the // transaction, which needs an additional layer of decoding. @@ -429,27 +343,12 @@ impl<'de> Deserialize<'de> for TestFile { if value.blocks[0].transaction_sequence.is_none() { let test_body = TestBody::from_parsed_json(&value, key.clone()); - // Sanity check: some tests *do not* abide by standard RLP encoding - // rules, therefore causing discrepancies between the "expected" - // encoding of the transaction that is part of the block RLP and used to - // form the transactions trie, and the *regular* encoding computed here - // after deserialization and to be fed to plonky2 zkEVM. + // Ensure that the gas used fits in 32 bits, otherwise the prover will + // abort. + if TryInto::::try_into(test_body.block.block_header.gas_used) + .is_ok() { - let rlp = &value.blocks[0].rlp.0; - let encoded_txn = rlp::encode(&test_body.get_tx()).to_vec(); - // Ensure that the encoding we will provide the zkEVM prover is in - // the block RLP. - if rlp.windows(encoded_txn.len()).any(|c| c == encoded_txn) { - // Finally, ensure that the gas used fits in 32 bits, otherwise - // the prover will abort. - if TryInto::::try_into( - test_body.block.block_header.gas_used, - ) - .is_ok() - { - map.0.insert(key, test_body); - } - } + map.0.insert(key, test_body); } } else { // Some tests deal with malformed transactions that wouldn't be passed diff --git a/eth_test_parser/src/trie_builder.rs b/eth_test_parser/src/trie_builder.rs index 82d2660..d457c64 100644 --- a/eth_test_parser/src/trie_builder.rs +++ b/eth_test_parser/src/trie_builder.rs @@ -18,6 +18,7 @@ use keccak_hash::keccak; use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, PartialTrie}, + utils::TryFromIterator, }; use rlp::Encodable; use rlp_derive::{RlpDecodable, RlpEncodable}; @@ -106,17 +107,19 @@ impl TestBody { self.pre .iter() .map(|(acc_key, pre_acc)| { - let storage_trie = pre_acc - .storage - .iter() - .filter(|(_, v)| !v.is_zero()) - .map(|(k, v)| { - ( - Nibbles::from_h256_be(hash(&u256_to_be_bytes(*k))), - v.rlp_bytes().to_vec(), - ) - }) - .collect(); + let storage_trie = HashedPartialTrie::try_from_iter( + pre_acc + .storage + .iter() + .filter(|(_, v)| !v.is_zero()) + .map(|(k, v)| { + ( + Nibbles::from_h256_be(hash(&u256_to_be_bytes(*k))), + v.rlp_bytes().to_vec(), + ) + }), + ) + .unwrap(); (hash(acc_key.as_bytes()), storage_trie) }) @@ -124,29 +127,26 @@ impl TestBody { } fn get_state_trie(&self, storage_tries: &[(H256, HashedPartialTrie)]) -> HashedPartialTrie { - self.pre - .iter() - .map(|(acc_key, pre_acc)| { - let addr_hash = hash(acc_key.as_bytes()); - let code_hash = hash(&pre_acc.code.0); - let storage_hash = get_storage_hash(&addr_hash, storage_tries); - - let rlp = AccountRlp { - nonce: pre_acc.nonce, - balance: pre_acc.balance, - storage_hash, - code_hash, - } - .rlp_bytes(); - - (Nibbles::from_h256_be(addr_hash), rlp.to_vec()) - }) - .collect() + HashedPartialTrie::try_from_iter(self.pre.iter().map(|(acc_key, pre_acc)| { + let addr_hash = hash(acc_key.as_bytes()); + let code_hash = hash(&pre_acc.code.0); + let storage_hash = get_storage_hash(&addr_hash, storage_tries); + + let rlp = AccountRlp { + nonce: pre_acc.nonce, + balance: pre_acc.balance, + storage_hash, + code_hash, + } + .rlp_bytes(); + + (Nibbles::from_h256_be(addr_hash), rlp.to_vec()) + })) + .unwrap() } pub(crate) fn get_txn_bytes(&self) -> Vec { - let transaction = &self.get_tx(); - rlp::encode(transaction).to_vec() + self.get_tx().0 } } From 4ddf02bf1ff62da12806f795eaa1ce7773e58d37 Mon Sep 17 00:00:00 2001 From: Robin Salen Date: Wed, 12 Jun 2024 16:15:42 -0400 Subject: [PATCH 2/2] Update deps --- Cargo.lock | 21 ++++++++++++++------- Cargo.toml | 6 +++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36340ef..0638362 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -682,7 +682,8 @@ dependencies = [ [[package]] name = "evm_arithmetization" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/zk_evm?branch=prep-release#eebbf3bf1417adce62c046032607af20b3dbacc7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d958d9badfbbd52c003da99378ff08c11f16efd4c4729e0fe2ca2224c46e8e" dependencies = [ "anyhow", "bytes", @@ -1301,7 +1302,8 @@ dependencies = [ [[package]] name = "mpt_trie" version = "0.3.0" -source = "git+https://github.com/0xPolygonZero/zk_evm?branch=prep-release#eebbf3bf1417adce62c046032607af20b3dbacc7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3febfed5b65d35c9b9811733e811a48b0591ed16dc07bee08c213c888b974b8" dependencies = [ "bytes", "enum-as-inner", @@ -1583,7 +1585,8 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f26b090b989aebdeaf6a4eed748c1fbcabf67e7273a22e4e0c877b63846d0f" dependencies = [ "ahash", "anyhow", @@ -1607,7 +1610,8 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a1dca60ad900d81b1fe2df3d0b88d43345988e2935e6709176e96573f4bcf5d" dependencies = [ "anyhow", "itertools", @@ -1622,7 +1626,8 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ff44a90aaca13e10e7ddf8fab815ba1b404c3f7c3ca82aaf11c46beabaa923" dependencies = [ "rayon", ] @@ -1630,7 +1635,8 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16136f5f3019c1e83035af76cccddd56d789a5e2933306270185c3f99f12259" [[package]] name = "portable-atomic" @@ -1991,7 +1997,8 @@ dependencies = [ [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2?branch=main#217b4a272ca0ab3b8ca1b3f79743c2e03a292b05" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8480ca5b8eedf83ad070a780783b4e21a56c6ef66b4c0d1b7520b72bdfda1b" dependencies = [ "ahash", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 27ad1b9..e7807c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,9 @@ serde_cbor = "0.11.2" tokio = { version = "1.28.1" } # zk-evm dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2", branch = "main" } -mpt_trie = { git = "https://github.com/0xPolygonZero/zk_evm", branch = "prep-release" } -evm_arithmetization = { git = "https://github.com/0xPolygonZero/zk_evm", branch = "prep-release" } +plonky2 = "0.2.2" +mpt_trie = "0.3.0" +evm_arithmetization = "0.2.0" [profile.release] opt-level = 3