From 877b730235b6d81eb93743e741eeeae892b11907 Mon Sep 17 00:00:00 2001 From: Itay Tsabary Date: Mon, 7 Oct 2024 09:41:35 +0300 Subject: [PATCH] feat(tests-integration): add binaries for integration test commit-id:56c09be1 --- Cargo.lock | 3 + crates/mempool_node/src/main.rs | 3 + crates/tests-integration/Cargo.toml | 3 + .../src/bin/run_test_rpc_state_reader.rs | 52 ++++++++++++ .../src/bin/run_test_tx_generator.rs | 64 +++++++++++++++ .../src/integration_test_config_utils.rs | 80 +++++++++++++++++++ .../src/integration_test_utils.rs | 4 +- crates/tests-integration/src/lib.rs | 1 + 8 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 crates/tests-integration/src/bin/run_test_rpc_state_reader.rs create mode 100644 crates/tests-integration/src/bin/run_test_tx_generator.rs create mode 100644 crates/tests-integration/src/integration_test_config_utils.rs diff --git a/Cargo.lock b/Cargo.lock index a8a82bd6b5..d3f457ccc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10361,6 +10361,7 @@ dependencies = [ name = "starknet_mempool_integration_tests" version = "0.0.0" dependencies = [ + "anyhow", "assert_matches", "axum", "blockifier", @@ -10368,6 +10369,7 @@ dependencies = [ "indexmap 2.6.0", "mempool_test_utils", "papyrus_common", + "papyrus_config", "papyrus_rpc", "papyrus_storage", "pretty_assertions", @@ -10388,6 +10390,7 @@ dependencies = [ "strum 0.25.0", "tempfile", "tokio", + "tracing", ] [[package]] diff --git a/crates/mempool_node/src/main.rs b/crates/mempool_node/src/main.rs index 8df25d4a70..2417836033 100644 --- a/crates/mempool_node/src/main.rs +++ b/crates/mempool_node/src/main.rs @@ -17,12 +17,14 @@ async fn main() -> anyhow::Result<()> { if let Err(ConfigError::CommandInput(clap_err)) = config { clap_err.exit(); } + info!("Finished loading configuration."); let config = config?; if let Err(error) = config_validate(&config) { error!("{}", error); exit(1); } + info!("Finished validating configuration."); // Clients are currently unused, but should not be dropped. let (_clients, servers) = create_node_modules(&config); @@ -30,5 +32,6 @@ async fn main() -> anyhow::Result<()> { info!("Starting components!"); run_component_servers(&config, servers).await?; + // TODO(Tsabary): Add graceful shutdown. Ok(()) } diff --git a/crates/tests-integration/Cargo.toml b/crates/tests-integration/Cargo.toml index d4343cf8c6..c753265a35 100644 --- a/crates/tests-integration/Cargo.toml +++ b/crates/tests-integration/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +anyhow.workspace = true assert_matches.workspace = true axum.workspace = true blockifier.workspace = true @@ -16,6 +17,7 @@ cairo-lang-starknet-classes.workspace = true indexmap.workspace = true mempool_test_utils.workspace = true papyrus_common.workspace = true +papyrus_config.workspace = true papyrus_rpc.workspace = true papyrus_storage = { workspace = true, features = ["testing"] } reqwest.workspace = true @@ -34,6 +36,7 @@ starknet_task_executor.workspace = true strum.workspace = true tempfile.workspace = true tokio.workspace = true +tracing.workspace = true [dev-dependencies] pretty_assertions.workspace = true diff --git a/crates/tests-integration/src/bin/run_test_rpc_state_reader.rs b/crates/tests-integration/src/bin/run_test_rpc_state_reader.rs new file mode 100644 index 0000000000..3ce972a7af --- /dev/null +++ b/crates/tests-integration/src/bin/run_test_rpc_state_reader.rs @@ -0,0 +1,52 @@ +use std::future::pending; + +use anyhow::Ok; +use blockifier::test_utils::contracts::FeatureContract; +use blockifier::test_utils::CairoVersion; +use mempool_test_utils::starknet_api_test_utils::MultiAccountTransactionGenerator; +use starknet_mempool_infra::trace_util::configure_tracing; +use starknet_mempool_integration_tests::integration_test_config_utils::create_config_files_for_node_and_tx_generator; +use starknet_mempool_integration_tests::integration_test_utils::create_config; +use starknet_mempool_integration_tests::state_reader::{ + spawn_test_rpc_state_reader, + StorageTestSetup, +}; +use tracing::info; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + configure_tracing(); + info!("Running integration test setup for the sequencer node."); + + // TODO(Tsabary): Code duplication with the end-to-end test. Refactor to avoid it. + let mut tx_generator: MultiAccountTransactionGenerator = + MultiAccountTransactionGenerator::new(); + + for account in [ + FeatureContract::AccountWithoutValidations(CairoVersion::Cairo1), + FeatureContract::AccountWithoutValidations(CairoVersion::Cairo0), + ] { + tx_generator.register_account_for_flow_test(account); + } + + // Spawn a papyrus rpc server for a papyrus storage reader. + let accounts = tx_generator.accounts(); + let storage_for_test = StorageTestSetup::new(accounts); + + // Spawn a papyrus rpc server for a papyrus storage reader. + let rpc_server_addr = spawn_test_rpc_state_reader(storage_for_test.rpc_storage_reader).await; + + // Derive the configuration for the mempool node. + let config = create_config(rpc_server_addr, storage_for_test.batcher_storage_config).await; + + // Note: the batcher storage file handle is passed as a reference to maintain its ownership in + // this scope, such that the handle is not dropped and the storage is maintained. + create_config_files_for_node_and_tx_generator(config)?; + + // Keep the program running so the rpc state reader server, its storage, and the batcher + // storage, are maintained. + let () = pending().await; + Ok(()) + + // TODO(Tsabary): Find a way to stop the program once the test is done. +} diff --git a/crates/tests-integration/src/bin/run_test_tx_generator.rs b/crates/tests-integration/src/bin/run_test_tx_generator.rs new file mode 100644 index 0000000000..1e79fa1b9c --- /dev/null +++ b/crates/tests-integration/src/bin/run_test_tx_generator.rs @@ -0,0 +1,64 @@ +use std::env::args; +use std::net::SocketAddr; +use std::process::exit; + +use blockifier::test_utils::contracts::FeatureContract; +use blockifier::test_utils::CairoVersion; +use mempool_test_utils::starknet_api_test_utils::MultiAccountTransactionGenerator; +use papyrus_config::validators::config_validate; +use papyrus_config::ConfigError; +use starknet_http_server::config::HttpServerConfig; +use starknet_mempool_infra::trace_util::configure_tracing; +use starknet_mempool_integration_tests::integration_test_utils::HttpTestClient; +use starknet_mempool_node::config::SequencerNodeConfig; +use tracing::{error, info}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + configure_tracing(); + info!("Running integration test transaction generation for the sequencer node."); + + // TODO(Tsabary): Code duplication with the end-to-end test. Refactor to avoid it. + let mut tx_generator: MultiAccountTransactionGenerator = + MultiAccountTransactionGenerator::new(); + + for account in [ + FeatureContract::AccountWithoutValidations(CairoVersion::Cairo1), + FeatureContract::AccountWithoutValidations(CairoVersion::Cairo0), + ] { + tx_generator.register_account_for_flow_test(account); + } + + let config = SequencerNodeConfig::load_and_process(args().collect()); + if let Err(ConfigError::CommandInput(clap_err)) = config { + clap_err.exit(); + } + + let config = config?; + if let Err(error) = config_validate(&config) { + error!("{}", error); + exit(1); + } + + let account0_invoke_nonce1 = tx_generator.account_with_id(0).generate_invoke_with_tip(1); + let account0_invoke_nonce2 = tx_generator.account_with_id(0).generate_invoke_with_tip(1); + let account1_invoke_nonce1 = tx_generator.account_with_id(1).generate_invoke_with_tip(1); + + let HttpServerConfig { ip, port } = config.http_server_config; + let http_test_client = HttpTestClient::new(SocketAddr::from((ip, port))); + + let account0_invoke_nonce1_tx_hash = + http_test_client.assert_add_tx_success(&account0_invoke_nonce1).await; + + let account1_invoke_nonce1_tx_hash = + http_test_client.assert_add_tx_success(&account1_invoke_nonce1).await; + + let account0_invoke_nonce2_tx_hash = + http_test_client.assert_add_tx_success(&account0_invoke_nonce2).await; + + info!("Add tx result: {:?}", account0_invoke_nonce1_tx_hash); + info!("Add tx result: {:?}", account1_invoke_nonce1_tx_hash); + info!("Add tx result: {:?}", account0_invoke_nonce2_tx_hash); + + Ok(()) +} diff --git a/crates/tests-integration/src/integration_test_config_utils.rs b/crates/tests-integration/src/integration_test_config_utils.rs new file mode 100644 index 0000000000..22431cefd8 --- /dev/null +++ b/crates/tests-integration/src/integration_test_config_utils.rs @@ -0,0 +1,80 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Write; + +use serde_json::{json, Value}; +use starknet_mempool_node::config::SequencerNodeConfig; +use tokio::io::Result; +use tracing::info; + +// TODO(Tsabary): Move here all config-related functions from "integration_test_utils.rs". + +const CONFIG_PARAMETERS_PATH: &str = "integration_test_config_changes.json"; +const TX_GEN_CONFIG_PARAMETERS_PATH: &str = "tx_gen_integration_test_config_changes.json"; + +/// Takes a list of config fields and returns a json dictionary with "field name : field value" +/// entries. Note that the prefixed "config." name is removed from the entry key. +macro_rules! config_fields_to_json { + ( $( $expr:expr ),+ ) => { + json!({ + $( + strip_config_prefix(stringify!($expr)): $expr + ),+ + }) + }; +} + +// TODO(Tsabary): Consider wrapping dumped config files in a temp dir. + +/// Returns config files to be supplied for the sequencer node and the transaction generator. Then +/// +/// Sequencer node: +/// cargo run --bin starknet_mempool_node -- --config_file CONFIG_PARAMETERS_PATH +/// Transaction generator: +/// cargo run --bin run_test_tx_generator -- --config_file TX_GEN_CONFIG_PARAMETERS_PATH +pub fn create_config_files_for_node_and_tx_generator( + config: SequencerNodeConfig, +) -> anyhow::Result<()> { + // Create config file for the sequencer node. + let json_data = config_fields_to_json!( + config.rpc_state_reader_config.json_rpc_version, + config.rpc_state_reader_config.url, + config.batcher_config.storage.db_config.path_prefix, + config + .gateway_config + .stateful_tx_validator_config + .chain_info + .fee_token_addresses + .eth_fee_token_address, + config + .gateway_config + .stateful_tx_validator_config + .chain_info + .fee_token_addresses + .strk_fee_token_address + ); + dump_json_data(json_data, CONFIG_PARAMETERS_PATH)?; + + // Create config file for the transaction generator. + let json_data = + config_fields_to_json!(config.http_server_config.ip, config.http_server_config.port); + dump_json_data(json_data, TX_GEN_CONFIG_PARAMETERS_PATH)?; + + Ok(()) +} + +fn dump_json_data(json_data: Value, path: &str) -> Result<()> { + // Serialize the JSON data to a pretty-printed string + let json_string = serde_json::to_string_pretty(&json_data).unwrap(); + + // Write the JSON string to a file + let mut file = File::create(path)?; + file.write_all(json_string.as_bytes())?; + info!("Writing JSON data to: {:?}", path); + + Ok(()) +} + +fn strip_config_prefix(input: &str) -> &str { + input.strip_prefix("config.").unwrap_or(input) +} diff --git a/crates/tests-integration/src/integration_test_utils.rs b/crates/tests-integration/src/integration_test_utils.rs index 854910ddfa..558b151744 100644 --- a/crates/tests-integration/src/integration_test_utils.rs +++ b/crates/tests-integration/src/integration_test_utils.rs @@ -92,7 +92,9 @@ impl HttpTestClient { } } -fn test_rpc_state_reader_config(rpc_server_addr: SocketAddr) -> RpcStateReaderConfig { +// TODO(Tsabary): move all public functions to be at the start of this module. + +pub fn test_rpc_state_reader_config(rpc_server_addr: SocketAddr) -> RpcStateReaderConfig { const RPC_SPEC_VERION: &str = "V0_8"; const JSON_RPC_VERSION: &str = "2.0"; RpcStateReaderConfig { diff --git a/crates/tests-integration/src/lib.rs b/crates/tests-integration/src/lib.rs index d0f04d51cc..5819eff91b 100644 --- a/crates/tests-integration/src/lib.rs +++ b/crates/tests-integration/src/lib.rs @@ -1,3 +1,4 @@ +pub mod integration_test_config_utils; pub mod integration_test_setup; pub mod integration_test_utils; pub mod state_reader;