diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml index e46bc0b898..5472342399 100644 --- a/.github/workflows/papyrus_ci.yml +++ b/.github/workflows/papyrus_ci.yml @@ -75,6 +75,24 @@ jobs: target/release/papyrus_node --base_layer.node_url ${{ secrets.CI_BASE_LAYER_NODE_URL }} & sleep 30 ; kill $! + p2p-sync-e2e-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: Noelware/setup-protoc@1.1.0 + with: + version: ${{env.PROTOC_VERSION}} + + - name: Build node + run: | + cargo build -r -p papyrus_node + + - name: Run p2p sync end-to-end test + run: | + scripts/papyrus/p2p_sync_e2e_test/main.sh ${{ secrets.CI_BASE_LAYER_NODE_URL }} + integration-test: runs-on: starkware-ubuntu-latest-medium steps: diff --git a/.gitignore b/.gitignore index 32bbb4b2fd..5c3eda09fd 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ tmp_venv/* # Python artifacts. scripts/__pycache__ + +# Papyrus p2p sync test artifacts. +scripts/papyrus/p2p_sync_e2e_test/data_client/ +scripts/papyrus/p2p_sync_e2e_test/data_server/ diff --git a/Cargo.lock b/Cargo.lock index 17a971331e..cbb60a7e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6282,7 +6282,6 @@ dependencies = [ "assert_matches", "async-stream", "bytes", - "clap", "deadqueue", "defaultmap", "derive_more", diff --git a/crates/blockifier/src/fee/actual_cost_test.rs b/crates/blockifier/src/fee/actual_cost_test.rs index acddd480c1..7653df2586 100644 --- a/crates/blockifier/src/fee/actual_cost_test.rs +++ b/crates/blockifier/src/fee/actual_cost_test.rs @@ -48,7 +48,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool let versioned_constants = VersionedConstants::default(); let empty_tx_starknet_resources = StarknetResources::default(); let empty_tx_gas_usage_vector = - empty_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + empty_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); assert_eq!(empty_tx_gas_usage_vector, GasVector::default()); // Declare. @@ -72,7 +72,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool let manual_gas_vector = GasVector { l1_gas: code_gas_cost.to_integer(), ..Default::default() }; let declare_gas_usage_vector = - declare_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + declare_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); assert_eq!(manual_gas_vector, declare_gas_usage_vector); } @@ -104,7 +104,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool + deploy_account_tx_starknet_resources.get_state_changes_cost(use_kzg_da); let deploy_account_gas_usage_vector = - deploy_account_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + deploy_account_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); assert_eq!(manual_gas_vector, deploy_account_gas_usage_vector); // L1 handler. @@ -119,7 +119,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool std::iter::empty(), ); let l1_handler_gas_usage_vector = - l1_handler_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + l1_handler_tx_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); // Manual calculation. let message_segment_length = get_message_segment_length(&[], Some(l1_handler_payload_size)); @@ -183,7 +183,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool ); let l2_to_l1_messages_gas_usage_vector = - l2_to_l1_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + l2_to_l1_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); // Manual calculation. let message_segment_length = get_message_segment_length(&l2_to_l1_payload_lengths, None); @@ -226,7 +226,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool ); let storage_writings_gas_usage_vector = - storage_writes_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + storage_writes_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); // Manual calculation. let manual_gas_computation = @@ -252,7 +252,7 @@ fn test_calculate_tx_gas_usage_basic<'a>(#[values(false, true)] use_kzg_da: bool ); let gas_usage_vector = - combined_cases_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da); + combined_cases_starknet_resources.to_gas_vector(&versioned_constants, use_kzg_da, false); // Manual calculation. let fee_balance_discount = match use_kzg_da { @@ -327,12 +327,12 @@ fn test_calculate_tx_gas_usage( ); assert_eq!( - starknet_resources.to_gas_vector(versioned_constants, use_kzg_da), - tx_execution_info - .receipt - .resources - .starknet_resources - .to_gas_vector(versioned_constants, use_kzg_da) + starknet_resources.to_gas_vector(versioned_constants, use_kzg_da, false), + tx_execution_info.receipt.resources.starknet_resources.to_gas_vector( + versioned_constants, + use_kzg_da, + false + ) ); // A tx that changes the account and some other balance in execute. @@ -378,11 +378,11 @@ fn test_calculate_tx_gas_usage( ); assert_eq!( - starknet_resources.to_gas_vector(versioned_constants, use_kzg_da), - tx_execution_info - .receipt - .resources - .starknet_resources - .to_gas_vector(versioned_constants, use_kzg_da) + starknet_resources.to_gas_vector(versioned_constants, use_kzg_da, false), + tx_execution_info.receipt.resources.starknet_resources.to_gas_vector( + versioned_constants, + use_kzg_da, + false + ) ); } diff --git a/crates/blockifier/src/fee/gas_usage_test.rs b/crates/blockifier/src/fee/gas_usage_test.rs index 1d89c913db..a1e65f4dfc 100644 --- a/crates/blockifier/src/fee/gas_usage_test.rs +++ b/crates/blockifier/src/fee/gas_usage_test.rs @@ -41,7 +41,7 @@ fn test_get_event_gas_cost( StarknetResources::new(0, 0, 0, StateChangesCount::default(), None, call_infos_iter); assert_eq!( GasVector::default(), - starknet_resources.to_gas_vector(versioned_constants, use_kzg_da) + starknet_resources.to_gas_vector(versioned_constants, use_kzg_da, false) ); let create_event = |keys_size: usize, data_size: usize| OrderedEvent { @@ -81,7 +81,7 @@ fn test_get_event_gas_cost( ); let starknet_resources = StarknetResources::new(0, 0, 0, StateChangesCount::default(), None, call_infos_iter); - let gas_vector = starknet_resources.to_gas_vector(versioned_constants, use_kzg_da); + let gas_vector = starknet_resources.to_gas_vector(versioned_constants, use_kzg_da, false); assert_eq!(expected, gas_vector); assert_ne!(GasVector::default(), gas_vector) } diff --git a/crates/blockifier/src/transaction/objects.rs b/crates/blockifier/src/transaction/objects.rs index 43b601ca92..a10c98137e 100644 --- a/crates/blockifier/src/transaction/objects.rs +++ b/crates/blockifier/src/transaction/objects.rs @@ -310,12 +310,13 @@ impl StarknetResources { &self, versioned_constants: &VersionedConstants, use_kzg_da: bool, + include_l2_gas: bool, ) -> GasVector { - self.get_calldata_and_signature_cost(versioned_constants) - + self.get_code_cost(versioned_constants) + self.get_calldata_and_signature_cost(versioned_constants, include_l2_gas) + + self.get_code_cost(versioned_constants, include_l2_gas) + self.get_state_changes_cost(use_kzg_da) + self.get_messages_cost() - + self.get_events_cost(versioned_constants) + + self.get_events_cost(versioned_constants, include_l2_gas) } // Returns the gas cost for transaction calldata and transaction signature. Each felt costs a @@ -324,13 +325,18 @@ impl StarknetResources { pub fn get_calldata_and_signature_cost( &self, versioned_constants: &VersionedConstants, + include_l2_gas: bool, ) -> GasVector { - // TODO(Avi, 20/2/2024): Calculate the number of bytes instead of the number of felts. - let total_data_size = u128_from_usize(self.calldata_length + self.signature_length); - let l1_gas = (versioned_constants.l2_resource_gas_costs.gas_per_data_felt - * total_data_size) - .to_integer(); - GasVector::from_l1_gas(l1_gas) + if include_l2_gas { + todo!() + } else { + // TODO(Avi, 20/2/2024): Calculate the number of bytes instead of the number of felts. + let total_data_size = u128_from_usize(self.calldata_length + self.signature_length); + let l1_gas = (versioned_constants.l2_resource_gas_costs.gas_per_data_felt + * total_data_size) + .to_integer(); + GasVector::from_l1_gas(l1_gas) + } } /// Returns an estimation of the gas usage for processing L1<>L2 messages on L1. Accounts for @@ -379,12 +385,20 @@ impl StarknetResources { } /// Returns the gas cost of declared class codes. - pub fn get_code_cost(&self, versioned_constants: &VersionedConstants) -> GasVector { - GasVector::from_l1_gas( - (versioned_constants.l2_resource_gas_costs.gas_per_code_byte - * u128_from_usize(self.code_size)) - .to_integer(), - ) + pub fn get_code_cost( + &self, + versioned_constants: &VersionedConstants, + include_l2_gas: bool, + ) -> GasVector { + if include_l2_gas { + todo!() + } else { + GasVector::from_l1_gas( + (versioned_constants.l2_resource_gas_costs.gas_per_code_byte + * u128_from_usize(self.code_size)) + .to_integer(), + ) + } } /// Returns the gas cost of the transaction's state changes. @@ -394,15 +408,23 @@ impl StarknetResources { } /// Returns the gas cost of the transaction's emmited events. - pub fn get_events_cost(&self, versioned_constants: &VersionedConstants) -> GasVector { - let l2_resource_gas_costs = &versioned_constants.l2_resource_gas_costs; - let (event_key_factor, data_word_cost) = - (l2_resource_gas_costs.event_key_factor, l2_resource_gas_costs.gas_per_data_felt); - let l1_gas: u128 = (data_word_cost - * (event_key_factor * self.total_event_keys + self.total_event_data_size)) - .to_integer(); - - GasVector::from_l1_gas(l1_gas) + pub fn get_events_cost( + &self, + versioned_constants: &VersionedConstants, + include_l2_gas: bool, + ) -> GasVector { + if include_l2_gas { + todo!() + } else { + let l2_resource_gas_costs = &versioned_constants.l2_resource_gas_costs; + let (event_key_factor, data_word_cost) = + (l2_resource_gas_costs.event_key_factor, l2_resource_gas_costs.gas_per_data_felt); + let l1_gas: u128 = (data_word_cost + * (event_key_factor * self.total_event_keys + self.total_event_data_size)) + .to_integer(); + + GasVector::from_l1_gas(l1_gas) + } } pub fn get_onchain_data_segment_length(&self) -> usize { @@ -449,7 +471,7 @@ impl TransactionResources { versioned_constants: &VersionedConstants, use_kzg_da: bool, ) -> TransactionFeeResult { - Ok(self.starknet_resources.to_gas_vector(versioned_constants, use_kzg_da) + Ok(self.starknet_resources.to_gas_vector(versioned_constants, use_kzg_da, false) + calculate_l1_gas_by_vm_usage( versioned_constants, &self.vm_resources, @@ -464,7 +486,7 @@ impl TransactionResources { with_reverted_steps: bool, ) -> ResourcesMapping { let GasVector { l1_gas, l1_data_gas, .. } = - self.starknet_resources.to_gas_vector(versioned_constants, use_kzg_da); + self.starknet_resources.to_gas_vector(versioned_constants, use_kzg_da, false); let mut resources = self.vm_resources.to_resources_mapping(); resources.0.extend(HashMap::from([ ( diff --git a/crates/blockifier/src/transaction/transactions_test.rs b/crates/blockifier/src/transaction/transactions_test.rs index a8198cfa9b..ffd28965a4 100644 --- a/crates/blockifier/src/transaction/transactions_test.rs +++ b/crates/blockifier/src/transaction/transactions_test.rs @@ -1899,11 +1899,11 @@ fn test_l1_handler(#[values(false, true)] use_kzg_da: bool) { }; assert_eq!( expected_gas, - actual_execution_info - .receipt - .resources - .starknet_resources - .to_gas_vector(versioned_constants, use_kzg_da) + actual_execution_info.receipt.resources.starknet_resources.to_gas_vector( + versioned_constants, + use_kzg_da, + false + ) ); let total_gas = expected_tx_resources diff --git a/scripts/papyrus/p2p_sync_e2e_test/client_node_config.json b/scripts/papyrus/p2p_sync_e2e_test/client_node_config.json new file mode 100644 index 0000000000..393ee3bd87 --- /dev/null +++ b/scripts/papyrus/p2p_sync_e2e_test/client_node_config.json @@ -0,0 +1,12 @@ +{ + "sync.#is_none": true, + "network.#is_none": false, + "p2p_sync.#is_none": false, + "storage.db_config.path_prefix": "./scripts/papyrus/p2p_sync_e2e_test/data_client", + "monitoring_gateway.server_address": "127.0.0.1:8082", + "collect_metrics": true, + "rpc.server_address": "127.0.0.1:8083", + "network.tcp_port": 10003, + "network.bootstrap_peer_multiaddr.#is_none": false, + "network.bootstrap_peer_multiaddr": "/ip4/127.0.0.1/tcp/10000/p2p/12D3KooWDFYi71juk6dYWo3UDvqs5gAzGDc124LSvcR5d187Tdvi" +} diff --git a/scripts/papyrus/p2p_sync_e2e_test/main.sh b/scripts/papyrus/p2p_sync_e2e_test/main.sh new file mode 100755 index 0000000000..c61c9fafde --- /dev/null +++ b/scripts/papyrus/p2p_sync_e2e_test/main.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +MONITORING_METRICS_URL="http://localhost:8082/monitoring/metrics" +SLEEP_DURATION_SECONDS=30 + +RED='\033[1;31m' +GREEN='\033[0;32m' +GRAY='\033[0;37m' +NO_COLOR='\033[0m' + +run_nodes_and_process_output() { + local client_command=$1 + local server_command=$2 + + eval "$client_command" & + client_pid=$! + + eval "$server_command" & + server_pid=$! + + echo "Client PID: $client_pid" + echo "Server PID: $server_pid" + + sleep $SLEEP_DURATION_SECONDS + validate_marker papyrus_state_marker + cleanup + echo -e "${GREEN}Test passed successfully.$NO_COLOR" +} + +# Extract the value of the given marker from the monitoring gateway and check it's big enough +validate_marker() { + local marker_name=$1 + + # Run curl and check the state marker + curl_output=$(curl -s -X GET "$MONITORING_METRICS_URL") + + # Extract the numeric value after marker_name + marker_value=$(echo "$curl_output" | grep -oP "$marker_name"' \K\d+') + + if [[ -z "$marker_value" ]]; then + cleanup + echo -e "${RED}Failed to extract $marker_name value from monitoring output. Test failed.$NO_COLOR" + exit 1 + fi + + if (( marker_value < 10 )); then + cleanup + echo -e "${RED}$marker_name is $marker_value which is less than 10. Test failed.$NO_COLOR" + exit 1 + fi + echo -e "${GREEN}$marker_name is $marker_value which is valid.$NO_COLOR" +} + + + +cleanup() { + echo -e "${GRAY}######## Cleaning up... You'll might see an ERROR on the 2nd process that is killed because the connection was closed and when killing it that there's no such process. This is ok. ########$NO_COLOR" + pgrep -P $$ + pkill -P $client_pid + pkill -P $server_pid + kill -KILL "$client_pid" + kill -KILL "$server_pid" +} + +main() { + if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 + fi + + base_layer_node_url=$1 + + rm -rf scripts/papyrus/p2p_sync_e2e_test/data_client scripts/papyrus/p2p_sync_e2e_test/data_server + mkdir scripts/papyrus/p2p_sync_e2e_test/data_client scripts/papyrus/p2p_sync_e2e_test/data_server + + client_node_command="target/release/papyrus_node --base_layer.node_url $base_layer_node_url --config_file scripts/papyrus/p2p_sync_e2e_test/client_node_config.json" + server_node_command="target/release/papyrus_node --base_layer.node_url $base_layer_node_url --config_file scripts/papyrus/p2p_sync_e2e_test/server_node_config.json" + + run_nodes_and_process_output "$client_node_command" "$server_node_command" +} + +main "$@" diff --git a/scripts/papyrus/p2p_sync_e2e_test/server_node_config.json b/scripts/papyrus/p2p_sync_e2e_test/server_node_config.json new file mode 100644 index 0000000000..bb55b3aaa7 --- /dev/null +++ b/scripts/papyrus/p2p_sync_e2e_test/server_node_config.json @@ -0,0 +1,6 @@ +{ + "central.http_headers": "X-Throttling-Bypass:dlaxszinf1zggax2xfbqeujanj603hhry8vli3vma34j", + "network.#is_none": false, + "storage.db_config.path_prefix": "./scripts/papyrus/p2p_sync_e2e_test/data_server", + "network.secret_key": "0xabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" +}