Merge pull request #102 from lambdaclass/bench-by-call

Save contract execution time

.gitignore

@@ -1,5 +1,6 @@
 debug/
 target/
+**.pyc
 
 .env
 .envrc

Cargo.toml

@@ -14,7 +14,7 @@ serde_with = "3.11.0"
 serde = "1.0.197"
 cairo-native = "0.2.4"
 # Sequencer Dependencies
-starknet_api = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f" } # replay
-blockifier = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f", features = ["cairo_native"] } # replay
-starknet_gateway = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f" } # replay
-blockifier_reexecution = { git = "https://github.com/lambdaclass/sequencer.git", rev = "bfc5b6b5475d359b8fc910516e026d972be5d02f" } # replay
+starknet_api = { git = "https://github.com/lambdaclass/sequencer.git", rev = "2ad9ecad71bd71304b80c36992f41568c82313ad" } # replay
+blockifier = { git = "https://github.com/lambdaclass/sequencer.git", rev = "2ad9ecad71bd71304b80c36992f41568c82313ad", features = ["cairo_native"] } # replay
+starknet_gateway = { git = "https://github.com/lambdaclass/sequencer.git", rev = "2ad9ecad71bd71304b80c36992f41568c82313ad" } # replay
+blockifier_reexecution = { git = "https://github.com/lambdaclass/sequencer.git", rev = "2ad9ecad71bd71304b80c36992f41568c82313ad" } # replay

README.md

@@ -126,17 +126,25 @@ In the `plotting` directory, you can find python scripts to plot relevant inform
 Make sure to erase the `compiled_programs` directory, then run:
 
 ```bash
-cargo run --features benchmark,structured_logging bench-block-range 724000 724000 mainnet 1 | tee native-logs
-cargo run --features benchmark,structured_logging,only_cairo_vm bench-block-range 724000 724000 mainnet 1 | tee vm-logs
+./scripts/benchmark_tx.sh <tx> <net> <block> <laps>
+```
+
+This generates four files:
+- `{native,vm}-data-$tx-$net.json`: Contains the execution time of each contract call
+- `{native,vm}-logs-$tx-$net.json`: Contains the output of running the benchmark
+
+If you want to benchmark a full block, you could run:
+```bash
+./scripts/benchmark_block.sh <block-start> <block-end> <net> <laps>
 ```
 
 Once you have done this, you can use the plotting scripts:
 
+- `python ./plotting/plot_execution_time.py native-data vm-data`: Plots the execution time of Native vs VM, by contract class.
 - `python ./plotting/plot_compilation_memory.py native-logs`: Size of the compiled native libraries, by contract class.
 - `python ./plotting/plot_compilation_memory_corr.py native-logs vm-logs`: Size of the compiled native libraries, by the associated Casm contract size.
 - `python ./plotting/plot_compilation_memory_trend.py native-logs vm-logs`: Size of the compiled native and casm contracts, by the sierra contract size.
 - `python ./plotting/plot_compilation_time.py native-logs`: Native compilation time, by contract class
 - `python ./plotting/plot_compilation_time_trend.py native-logs vm-logs`: Native and Casm compilation time, by the sierra contract size.
-- `python ./plotting/plot_execution_time.py native-logs vm-logs`: Plots the execution time of Native vs VM, by contract class.
 - `python ./plotting/plot_compilation_time_finer.py native-logs`: Native compilation time, with fine-grained stage separation, by contract class.