From 542d64a168c204d29b5bbde88ba8560937d36ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 3 Dec 2024 14:59:32 -0300 Subject: [PATCH 01/15] Remove hash print --- replay/src/benchmark.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/replay/src/benchmark.rs b/replay/src/benchmark.rs index 398c5b4..4ea93ae 100644 --- a/replay/src/benchmark.rs +++ b/replay/src/benchmark.rs @@ -77,11 +77,7 @@ pub fn execute_block_range(block_range_data: &mut Vec) { } in transactions { // Execute each transaction - let _tx_span = info_span!( - "tx execution", - transaction_hash = transaction_hash.to_string(), - ) - .entered(); + let _tx_span = info_span!("tx execution",).entered(); info!("tx execution started"); From c4f0e70b0bbacb607fac3b438d09ea65cfe69d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 3 Dec 2024 14:59:35 -0300 Subject: [PATCH 02/15] Add sleep --- replay/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/replay/src/main.rs b/replay/src/main.rs index ec78bc3..8f0934e 100644 --- a/replay/src/main.rs +++ b/replay/src/main.rs @@ -1,3 +1,6 @@ +use std::thread; +use std::time::Duration; + use blockifier::state::cached_state::CachedState; use blockifier::state::errors::StateError; use blockifier::transaction::objects::{RevertError, TransactionExecutionInfo}; @@ -224,6 +227,10 @@ fn main() { block_range_data }; + // We pause the main thread to differentiate + // caching from benchmarking from within a profiler + thread::sleep(Duration::from_secs(1)); + { let _benchmark_span = info_span!("benchmarking transaction").entered(); let before_execution = Instant::now(); From 9e39ec48fe995e991be6141fe956b5cb378da40c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 3 Dec 2024 14:59:39 -0300 Subject: [PATCH 03/15] Use aggressive opt --- rpc-state-reader/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc-state-reader/src/utils.rs b/rpc-state-reader/src/utils.rs index 81c8308..4fdde86 100644 --- a/rpc-state-reader/src/utils.rs +++ b/rpc-state-reader/src/utils.rs @@ -102,7 +102,7 @@ pub fn get_native_executor(contract: &ContractClass, class_hash: ClassHash) -> A let mut executor = AotContractExecutor::new( &contract.extract_sierra_program().unwrap(), &contract.entry_points_by_type, - OptLevel::Default, + OptLevel::Aggressive, ) .unwrap(); let compilation_time = pre_compilation_instant.elapsed().as_millis(); From f890f96ffc55fb358fc161144ebde028bdf9ced3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 3 Dec 2024 14:59:44 -0300 Subject: [PATCH 04/15] Add benchmark scripts --- scripts/run_benchmark.sh | 12 ++++++++++++ scripts/run_tx_benchmark.sh | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100755 scripts/run_benchmark.sh create mode 100755 scripts/run_tx_benchmark.sh diff --git a/scripts/run_benchmark.sh b/scripts/run_benchmark.sh new file mode 100755 index 0000000..c35165e --- /dev/null +++ b/scripts/run_benchmark.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +START=874000 +END=874009 +NET=mainnet +LAPS=100 + +output="block-$START-$END-$NET.jsonl" + +cargo run --release --features benchmark,structured_logging bench-block-range $START $END $NET $LAPS | tee "native-$output" + +cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-block-range $START $END $NET $LAPS | tee "vm-$output" diff --git a/scripts/run_tx_benchmark.sh b/scripts/run_tx_benchmark.sh new file mode 100755 index 0000000..4d1c5eb --- /dev/null +++ b/scripts/run_tx_benchmark.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +TX=0x01e06dfbd41e559ee5edd313ab95605331873a5aed09bf1c7312456b7aa2a1c7 +BLOCK=291652 +LAPS=10000 +NET=testnet + +output="tx-$TX-$NET.jsonl" + +cargo run --release --features benchmark,structured_logging bench-tx $TX $NET $BLOCK $LAPS | tee "native-$output" + +cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-tx $TX $NET $BLOCK $LAPS | tee "vm-$output" From 3af1964c6a9c5c111bd24aa63cb21c49cd9e61b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Tue, 3 Dec 2024 18:01:11 -0300 Subject: [PATCH 05/15] Update lock --- Cargo.lock | 98 +++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03e2ee9..078d8f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "aquamarine" @@ -655,7 +655,7 @@ dependencies = [ [[package]] name = "blockifier" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "anyhow", "ark-ec", @@ -699,7 +699,7 @@ dependencies = [ [[package]] name = "blockifier_reexecution" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "assert_matches", "blockifier", @@ -1574,9 +1574,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b" dependencies = [ "clap_builder", "clap_derive", @@ -1584,9 +1584,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1" dependencies = [ "anstream", "anstyle", @@ -1682,9 +1682,9 @@ dependencies = [ [[package]] name = "comrak" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949f8e6b02ebac005a8be2df9ec0876cafc83fdb9c510796c37f0fadf92dcd0e" +checksum = "453dcb42e33f7b474d7e0db12e0b8d82802c88f35cf5a1d8c297d0dfcecb154f" dependencies = [ "bon", "caseless", @@ -3295,9 +3295,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -3322,7 +3322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -3333,7 +3333,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] @@ -3389,7 +3389,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "httparse", "itoa", @@ -3453,7 +3453,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "hyper 1.5.1", "pin-project-lite", @@ -3770,7 +3770,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "infra_utils" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "tokio", ] @@ -4497,9 +4497,9 @@ dependencies = [ [[package]] name = "melior" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c375d5d59f6db85fe38d8833be858ea4e00d6e4a0445820628f080789af6006" +checksum = "ee088ab94ababfb7843fbf3c6239a535f371ea23b57c75f26707c14096527874" dependencies = [ "melior-macro", "mlir-sys", @@ -4507,9 +4507,9 @@ dependencies = [ [[package]] name = "melior-macro" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542d875231bf525c40d5428b0256948b6d1c37ecdff629eb03491cfe01982a7c" +checksum = "575063f5ab34758980f9441a3978b4ad8c46e8eaeb5c6cb0e73b985f056d8490" dependencies = [ "comrak", "convert_case 0.6.0", @@ -4539,7 +4539,7 @@ dependencies = [ [[package]] name = "mempool_test_utils" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "assert_matches", "blockifier", @@ -5003,7 +5003,7 @@ dependencies = [ [[package]] name = "papyrus_common" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "base64 0.13.1", "cairo-lang-starknet-classes", @@ -5022,7 +5022,7 @@ dependencies = [ [[package]] name = "papyrus_config" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "clap", "infra_utils", @@ -5037,7 +5037,7 @@ dependencies = [ [[package]] name = "papyrus_execution" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "anyhow", "blockifier", @@ -5060,7 +5060,7 @@ dependencies = [ [[package]] name = "papyrus_network_types" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "libp2p", "serde", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "papyrus_proc_macros" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "quote", "syn 2.0.90", @@ -5079,7 +5079,7 @@ dependencies = [ [[package]] name = "papyrus_rpc" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "anyhow", "async-trait", @@ -5113,7 +5113,7 @@ dependencies = [ [[package]] name = "papyrus_storage" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "byteorder", "cairo-lang-casm", @@ -5158,14 +5158,14 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.7.0" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8781a75c6205af67215f382092b6e0a4ff3734798523e69073d4bcd294ec767b" +checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 1.0.109", ] [[package]] @@ -5828,7 +5828,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -5868,7 +5868,7 @@ checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" dependencies = [ "anyhow", "async-trait", - "http 1.1.0", + "http 1.2.0", "reqwest 0.12.9", "serde", "thiserror", @@ -6950,7 +6950,7 @@ dependencies = [ [[package]] name = "starknet_api" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "bitvec", "cairo-lang-runner", @@ -6976,7 +6976,7 @@ dependencies = [ [[package]] name = "starknet_client" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "cairo-lang-starknet-classes", @@ -7003,7 +7003,7 @@ dependencies = [ [[package]] name = "starknet_gateway" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "axum", @@ -7031,7 +7031,7 @@ dependencies = [ [[package]] name = "starknet_gateway_types" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "axum", @@ -7050,7 +7050,7 @@ dependencies = [ [[package]] name = "starknet_mempool_types" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "papyrus_network_types", @@ -7064,7 +7064,7 @@ dependencies = [ [[package]] name = "starknet_sequencer_infra" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "bincode 1.3.3", @@ -7082,7 +7082,7 @@ dependencies = [ [[package]] name = "starknet_sierra_compile" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#111c6fd7ea6895f6322c15865a6f7fafaf29781e" +source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "cairo-lang-sierra", "cairo-lang-starknet-classes", @@ -7398,9 +7398,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -7419,9 +7419,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -7463,9 +7463,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -7876,9 +7876,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +checksum = "3193f92e105038f98ae68af40c008e3c94f2f046926e0f95e6c835dc6459bac8" dependencies = [ "base64 0.22.1", "flate2", From 525c593c72b463b2e199d944366fd0207341d789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 10:55:33 -0300 Subject: [PATCH 06/15] Update lock --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 078d8f9..091b516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7549,9 +7549,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -7876,9 +7876,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3193f92e105038f98ae68af40c008e3c94f2f046926e0f95e6c835dc6459bac8" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64 0.22.1", "flate2", From 4379d4faa7817a09291a6bbfd67626c36a2137bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 11:56:06 -0300 Subject: [PATCH 07/15] Update deps --- Cargo.lock | 36 ++++++++++++++++++------------------ Cargo.toml | 11 ++++------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 091b516..3ba4c3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -655,7 +655,7 @@ dependencies = [ [[package]] name = "blockifier" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "anyhow", "ark-ec", @@ -699,7 +699,7 @@ dependencies = [ [[package]] name = "blockifier_reexecution" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "assert_matches", "blockifier", @@ -3770,7 +3770,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "infra_utils" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "tokio", ] @@ -4539,7 +4539,7 @@ dependencies = [ [[package]] name = "mempool_test_utils" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "assert_matches", "blockifier", @@ -5003,7 +5003,7 @@ dependencies = [ [[package]] name = "papyrus_common" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "base64 0.13.1", "cairo-lang-starknet-classes", @@ -5022,7 +5022,7 @@ dependencies = [ [[package]] name = "papyrus_config" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "clap", "infra_utils", @@ -5037,7 +5037,7 @@ dependencies = [ [[package]] name = "papyrus_execution" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "anyhow", "blockifier", @@ -5060,7 +5060,7 @@ dependencies = [ [[package]] name = "papyrus_network_types" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "libp2p", "serde", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "papyrus_proc_macros" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "quote", "syn 2.0.90", @@ -5079,7 +5079,7 @@ dependencies = [ [[package]] name = "papyrus_rpc" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "anyhow", "async-trait", @@ -5113,7 +5113,7 @@ dependencies = [ [[package]] name = "papyrus_storage" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "byteorder", "cairo-lang-casm", @@ -6950,7 +6950,7 @@ dependencies = [ [[package]] name = "starknet_api" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "bitvec", "cairo-lang-runner", @@ -6976,7 +6976,7 @@ dependencies = [ [[package]] name = "starknet_client" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "cairo-lang-starknet-classes", @@ -7003,7 +7003,7 @@ dependencies = [ [[package]] name = "starknet_gateway" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "axum", @@ -7031,7 +7031,7 @@ dependencies = [ [[package]] name = "starknet_gateway_types" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "axum", @@ -7050,7 +7050,7 @@ dependencies = [ [[package]] name = "starknet_mempool_types" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "papyrus_network_types", @@ -7064,7 +7064,7 @@ dependencies = [ [[package]] name = "starknet_sequencer_infra" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "async-trait", "bincode 1.3.3", @@ -7082,7 +7082,7 @@ dependencies = [ [[package]] name = "starknet_sierra_compile" version = "0.0.0" -source = "git+https://github.com/lambdaclass/sequencer.git?branch=replay#bfc5b6b5475d359b8fc910516e026d972be5d02f" +source = "git+https://github.com/lambdaclass/sequencer.git?rev=bfc5b6b5475d359b8fc910516e026d972be5d02f#bfc5b6b5475d359b8fc910516e026d972be5d02f" dependencies = [ "cairo-lang-sierra", "cairo-lang-starknet-classes", diff --git a/Cargo.toml b/Cargo.toml index 5bc40ad..a6f7403 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,7 @@ serde_with = "3.11.0" serde = "1.0.197" cairo-native = "0.2.4" # Sequencer Dependencies -starknet_api = { git = "https://github.com/lambdaclass/sequencer.git", branch = "replay" } # main -blockifier = { git = "https://github.com/lambdaclass/sequencer.git", branch = "replay", features = ["cairo_native"] } # main -starknet_gateway = { git = "https://github.com/lambdaclass/sequencer.git", branch = "replay" } # main -blockifier_reexecution = { git = "https://github.com/lambdaclass/sequencer.git", branch = "replay" } # main - -[patch.'https://github.com/lambdaclass/cairo_native'] -cairo-native = "=0.2.4" +starknet_api = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f" } # replay +blockifier = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f", features = ["cairo_native"] } # replay +starknet_gateway = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f" } # replay +blockifier_reexecution = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f" } # replay From 9d9b6fb26a7130c41d65afae8ed09e170d45202f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 12:23:51 -0300 Subject: [PATCH 08/15] Rename --- scripts/{run_benchmark.sh => benchmark_block.sh} | 0 scripts/{run_tx_benchmark.sh => benchmark_tx.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{run_benchmark.sh => benchmark_block.sh} (100%) rename scripts/{run_tx_benchmark.sh => benchmark_tx.sh} (100%) diff --git a/scripts/run_benchmark.sh b/scripts/benchmark_block.sh similarity index 100% rename from scripts/run_benchmark.sh rename to scripts/benchmark_block.sh diff --git a/scripts/run_tx_benchmark.sh b/scripts/benchmark_tx.sh similarity index 100% rename from scripts/run_tx_benchmark.sh rename to scripts/benchmark_tx.sh From a83d791c9b3776f291771d810882a841f371bdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 12:41:30 -0300 Subject: [PATCH 09/15] Improve benchmark_tx script --- scripts/benchmark_tx.sh | 44 ++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/scripts/benchmark_tx.sh b/scripts/benchmark_tx.sh index 4d1c5eb..4e9a6e9 100755 --- a/scripts/benchmark_tx.sh +++ b/scripts/benchmark_tx.sh @@ -1,12 +1,42 @@ #!/usr/bin/env bash -TX=0x01e06dfbd41e559ee5edd313ab95605331873a5aed09bf1c7312456b7aa2a1c7 -BLOCK=291652 -LAPS=10000 -NET=testnet +usage() { +cat < -output="tx-$TX-$NET.jsonl" +Benches a single transaction: +- Saves output to native-- and vm-- +- Prints speedup +EOF +} -cargo run --release --features benchmark,structured_logging bench-tx $TX $NET $BLOCK $LAPS | tee "native-$output" +if ! [ "$#" -ge "4" ]; then + usage + exit 1 +fi -cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-tx $TX $NET $BLOCK $LAPS | tee "vm-$output" +TX=$1 +NET=$2 +BLOCK=$3 +LAPS=$4 + +echo "Benchmarking $NET $TX" + +output="$TX-$NET.jsonl" +native_output="native-$output" +vm_output="vm-$output" + +echo "Executing with Native" +cargo run --release --features benchmark,structured_logging bench-tx "$TX" "$NET" "$BLOCK" "$LAPS" > "$native_output" 2>/dev/null + +native_time=$(tail -n1 "$native_output" | jq .fields.average_run_time) +echo "Average Native time: $native_time" + +echo "Executing with VM" +cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-tx "$TX" "$NET" "$BLOCK" "$LAPS" > "$vm_output" 2>/dev/null + +vm_time=$(tail -n1 "$vm_output" | jq .fields.average_run_time) +echo "Average VM time: $vm_time" + +speedup=$(bc -le "$vm_time/$native_time") +echo "Native Speedup: $speedup" From cb74ab12748aa3e5b252f685b62653c96ae64a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 13:05:56 -0300 Subject: [PATCH 10/15] Add benchmark_txs.sh script --- scripts/benchmark_txs.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 scripts/benchmark_txs.sh diff --git a/scripts/benchmark_txs.sh b/scripts/benchmark_txs.sh new file mode 100755 index 0000000..5284d3b --- /dev/null +++ b/scripts/benchmark_txs.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Calls `benchmark_tx` with a predefined list of transactions + +dir=$(dirname "$0") +benchmark_tx_script="$dir/benchmark_tx.sh" + +cat < Date: Wed, 4 Dec 2024 14:48:47 -0300 Subject: [PATCH 11/15] Make linux compatible --- scripts/benchmark_tx.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/benchmark_tx.sh b/scripts/benchmark_tx.sh index 4e9a6e9..e537287 100755 --- a/scripts/benchmark_tx.sh +++ b/scripts/benchmark_tx.sh @@ -20,8 +20,6 @@ NET=$2 BLOCK=$3 LAPS=$4 -echo "Benchmarking $NET $TX" - output="$TX-$NET.jsonl" native_output="native-$output" vm_output="vm-$output" @@ -38,5 +36,5 @@ cargo run --release --features benchmark,structured_logging,only_cairo_vm bench- vm_time=$(tail -n1 "$vm_output" | jq .fields.average_run_time) echo "Average VM time: $vm_time" -speedup=$(bc -le "$vm_time/$native_time") +speedup=$(bc -l <<< "$vm_time/$native_time") echo "Native Speedup: $speedup" From 69980e38da5ea97bab4afc2020166a7f50af25fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 14:48:53 -0300 Subject: [PATCH 12/15] Use for instead of xargs --- scripts/benchmark_txs.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/benchmark_txs.sh b/scripts/benchmark_txs.sh index 5284d3b..375148c 100755 --- a/scripts/benchmark_txs.sh +++ b/scripts/benchmark_txs.sh @@ -5,8 +5,18 @@ dir=$(dirname "$0") benchmark_tx_script="$dir/benchmark_tx.sh" -cat < Date: Wed, 4 Dec 2024 14:59:14 -0300 Subject: [PATCH 13/15] Print laps --- scripts/benchmark_txs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmark_txs.sh b/scripts/benchmark_txs.sh index 375148c..d640df8 100755 --- a/scripts/benchmark_txs.sh +++ b/scripts/benchmark_txs.sh @@ -15,7 +15,7 @@ for case in "${cases[@]}"; do read -r tx net block laps <<< "$case" echo - echo "Benchmarking $tx from $net" + echo "Benchmarking $tx from $net $laps times" echo $benchmark_tx_script "$tx" "$net" "$block" "$laps" From a37ff1349fcbea9800068ec24893c1fea29f39e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 14:59:23 -0300 Subject: [PATCH 14/15] Show errors --- scripts/benchmark_tx.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/benchmark_tx.sh b/scripts/benchmark_tx.sh index e537287..1ba22c1 100755 --- a/scripts/benchmark_tx.sh +++ b/scripts/benchmark_tx.sh @@ -25,13 +25,13 @@ native_output="native-$output" vm_output="vm-$output" echo "Executing with Native" -cargo run --release --features benchmark,structured_logging bench-tx "$TX" "$NET" "$BLOCK" "$LAPS" > "$native_output" 2>/dev/null +cargo run --release --features benchmark,structured_logging bench-tx "$TX" "$NET" "$BLOCK" "$LAPS" > "$native_output" native_time=$(tail -n1 "$native_output" | jq .fields.average_run_time) echo "Average Native time: $native_time" echo "Executing with VM" -cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-tx "$TX" "$NET" "$BLOCK" "$LAPS" > "$vm_output" 2>/dev/null +cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-tx "$TX" "$NET" "$BLOCK" "$LAPS" > "$vm_output" vm_time=$(tail -n1 "$vm_output" | jq .fields.average_run_time) echo "Average VM time: $vm_time" From 1f5bcb416fd0de2d5a8dd4e2ed107943c0dec469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Gonz=C3=A1lez=20Calder=C3=B3n?= Date: Wed, 4 Dec 2024 14:59:28 -0300 Subject: [PATCH 15/15] Refactor block script --- scripts/benchmark_block.sh | 41 ++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/scripts/benchmark_block.sh b/scripts/benchmark_block.sh index c35165e..c9fc50f 100755 --- a/scripts/benchmark_block.sh +++ b/scripts/benchmark_block.sh @@ -1,12 +1,41 @@ #!/usr/bin/env bash -START=874000 -END=874009 -NET=mainnet -LAPS=100 +usage() { +cat < + +Benches a block range +- Saves output to native--- and vm--- +- Prints speedup +EOF +} + +if ! [ "$#" -ge "4" ]; then + usage + exit 1 +fi + + +START=$1 +END=$2 +NET=$3 +LAPS=$4 output="block-$START-$END-$NET.jsonl" +native_output="native-$output" +vm_output="vm-$output" + +echo "Executing with Native" +cargo run --release --features benchmark,structured_logging bench-block-range "$START" "$END" "$NET" "$LAPS" > "$native_output" + +native_time=$(tail -n1 "$native_output" | jq .fields.average_run_time) +echo "Average Native time: $native_time" + +echo "Executing with VM" +cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-block-range "$START" "$END" "$NET" "$LAPS" > "$vm_output" -cargo run --release --features benchmark,structured_logging bench-block-range $START $END $NET $LAPS | tee "native-$output" +vm_time=$(tail -n1 "$vm_output" | jq .fields.average_run_time) +echo "Average VM time: $vm_time" -cargo run --release --features benchmark,structured_logging,only_cairo_vm bench-block-range $START $END $NET $LAPS | tee "vm-$output" +speedup=$(bc -l <<< "$vm_time/$native_time") +echo "Native Speedup: $speedup"