feat(tests-integration): add binaries for integration test

crates/mempool_node/src/main.rs

@@ -17,18 +17,21 @@ async fn main() -> anyhow::Result<()> {
     if let Err(ConfigError::CommandInput(clap_err)) = config {
         clap_err.exit();
     }
+    info!("Finished loading configuration.");
 
     let config = config?;
     if let Err(error) = config_validate(&config) {
         error!("{}", error);
         exit(1);
     }
+    info!("Finished validating configuration.");
 
     // Clients are currently unused, but should not be dropped.
     let (_clients, servers) = create_node_modules(&config);
 
     info!("Starting components!");
     run_component_servers(&config, servers).await?;
 
+    // TODO(Tsabary): Add graceful shutdown.
     Ok(())
 }

crates/tests-integration/Cargo.toml

@@ -9,13 +9,15 @@ license.workspace = true
 workspace = true
 
 [dependencies]
+anyhow.workspace = true
 assert_matches.workspace = true
 axum.workspace = true
 blockifier.workspace = true
 cairo-lang-starknet-classes.workspace = true
 indexmap.workspace = true
 mempool_test_utils.workspace = true
 papyrus_common.workspace = true
+papyrus_config.workspace = true
 papyrus_rpc.workspace = true
 papyrus_storage = { workspace = true, features = ["testing"] }
 reqwest.workspace = true
@@ -34,6 +36,7 @@ starknet_task_executor.workspace = true
 strum.workspace = true
 tempfile.workspace = true
 tokio.workspace = true
+tracing.workspace = true
 
 [dev-dependencies]
 pretty_assertions.workspace = true

crates/tests-integration/src/bin/run_test_rpc_state_reader.rs

@@ -0,0 +1,52 @@
+use std::future::pending;
+
+use anyhow::Ok;
+use blockifier::test_utils::contracts::FeatureContract;
+use blockifier::test_utils::CairoVersion;
+use mempool_test_utils::starknet_api_test_utils::MultiAccountTransactionGenerator;
+use starknet_mempool_infra::trace_util::configure_tracing;
+use starknet_mempool_integration_tests::integration_test_config_utils::create_config_files_for_node_and_tx_generator;
+use starknet_mempool_integration_tests::integration_test_utils::create_config;
+use starknet_mempool_integration_tests::state_reader::{
+    spawn_test_rpc_state_reader,
+    StorageTestSetup,
+};
+use tracing::info;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    configure_tracing();
+    info!("Running integration test setup for the sequencer node.");
+
+    // TODO(Tsabary): Code duplication with the end-to-end test. Refactor to avoid it.
+    let mut tx_generator: MultiAccountTransactionGenerator =
+        MultiAccountTransactionGenerator::new();
+
+    for account in [
+        FeatureContract::AccountWithoutValidations(CairoVersion::Cairo1),
+        FeatureContract::AccountWithoutValidations(CairoVersion::Cairo0),
+    ] {
+        tx_generator.register_account_for_flow_test(account);
+    }
+
+    // Spawn a papyrus rpc server for a papyrus storage reader.
+    let accounts = tx_generator.accounts();
+    let storage_for_test = StorageTestSetup::new(accounts);
+
+    // Spawn a papyrus rpc server for a papyrus storage reader.
+    let rpc_server_addr = spawn_test_rpc_state_reader(storage_for_test.rpc_storage_reader).await;
+
+    // Derive the configuration for the mempool node.
+    let config = create_config(rpc_server_addr, storage_for_test.batcher_storage_config).await;
+
+    // Note: the batcher storage file handle is passed as a reference to maintain its ownership in
+    // this scope, such that the handle is not dropped and the storage is maintained.
+    create_config_files_for_node_and_tx_generator(config)?;
+
+    // Keep the program running so the rpc state reader server, its storage, and the batcher
+    // storage, are maintained.
+    let () = pending().await;
+    Ok(())
+
+    // TODO(Tsabary): Find a way to stop the program once the test is done.
+}

crates/tests-integration/src/bin/run_test_tx_generator.rs

@@ -0,0 +1,64 @@
+use std::env::args;
+use std::net::SocketAddr;
+use std::process::exit;
+
+use blockifier::test_utils::contracts::FeatureContract;
+use blockifier::test_utils::CairoVersion;
+use mempool_test_utils::starknet_api_test_utils::MultiAccountTransactionGenerator;
+use papyrus_config::validators::config_validate;
+use papyrus_config::ConfigError;
+use starknet_http_server::config::HttpServerConfig;
+use starknet_mempool_infra::trace_util::configure_tracing;
+use starknet_mempool_integration_tests::integration_test_utils::HttpTestClient;
+use starknet_mempool_node::config::SequencerNodeConfig;
+use tracing::{error, info};
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    configure_tracing();
+    info!("Running integration test transaction generation for the sequencer node.");
+
+    // TODO(Tsabary): Code duplication with the end-to-end test. Refactor to avoid it.
+    let mut tx_generator: MultiAccountTransactionGenerator =
+        MultiAccountTransactionGenerator::new();
+
+    for account in [
+        FeatureContract::AccountWithoutValidations(CairoVersion::Cairo1),
+        FeatureContract::AccountWithoutValidations(CairoVersion::Cairo0),
+    ] {
+        tx_generator.register_account_for_flow_test(account);
+    }
+
+    let config = SequencerNodeConfig::load_and_process(args().collect());
+    if let Err(ConfigError::CommandInput(clap_err)) = config {
+        clap_err.exit();
+    }
+
+    let config = config?;
+    if let Err(error) = config_validate(&config) {
+        error!("{}", error);
+        exit(1);
+    }
+
+    let account0_invoke_nonce1 = tx_generator.account_with_id(0).generate_invoke_with_tip(1);
+    let account0_invoke_nonce2 = tx_generator.account_with_id(0).generate_invoke_with_tip(1);
+    let account1_invoke_nonce1 = tx_generator.account_with_id(1).generate_invoke_with_tip(1);
+
+    let HttpServerConfig { ip, port } = config.http_server_config;
+    let http_test_client = HttpTestClient::new(SocketAddr::from((ip, port)));
+
+    let account0_invoke_nonce1_tx_hash =
+        http_test_client.assert_add_tx_success(&account0_invoke_nonce1).await;
+
+    let account1_invoke_nonce1_tx_hash =
+        http_test_client.assert_add_tx_success(&account1_invoke_nonce1).await;
+
+    let account0_invoke_nonce2_tx_hash =
+        http_test_client.assert_add_tx_success(&account0_invoke_nonce2).await;
+
+    info!("Add tx result: {:?}", account0_invoke_nonce1_tx_hash);
+    info!("Add tx result: {:?}", account1_invoke_nonce1_tx_hash);
+    info!("Add tx result: {:?}", account0_invoke_nonce2_tx_hash);
+
+    Ok(())
+}

crates/tests-integration/src/integration_test_config_utils.rs

@@ -0,0 +1,80 @@
+use std::collections::BTreeMap;
+use std::fs::File;
+use std::io::Write;
+
+use serde_json::{json, Value};
+use starknet_mempool_node::config::SequencerNodeConfig;
+use tokio::io::Result;
+use tracing::info;
+
+// TODO(Tsabary): Move here all config-related functions from "integration_test_utils.rs".
+
+const CONFIG_PARAMETERS_PATH: &str = "integration_test_config_changes.json";
+const TX_GEN_CONFIG_PARAMETERS_PATH: &str = "tx_gen_integration_test_config_changes.json";
+
+/// Takes a list of config fields and returns a json dictionary with "field name : field value"
+/// entries. Note that the prefixed "config." name is removed from the entry key.
+macro_rules! config_fields_to_json {
+    ( $( $expr:expr ),+ ) => {
+        json!({
+            $(
+                strip_config_prefix(stringify!($expr)): $expr
+            ),+
+        })
+    };
+}
+
+// TODO(Tsabary): Consider wrapping dumped config files in a temp dir.
+
+/// Returns config files to be supplied for the sequencer node and the transaction generator. Then
+///
+/// Sequencer node:
+/// cargo run --bin starknet_mempool_node -- --config_file CONFIG_PARAMETERS_PATH
+/// Transaction generator:
+/// cargo run --bin run_test_tx_generator -- --config_file TX_GEN_CONFIG_PARAMETERS_PATH
+pub fn create_config_files_for_node_and_tx_generator(
+    config: SequencerNodeConfig,
+) -> anyhow::Result<()> {
+    // Create config file for the sequencer node.
+    let json_data = config_fields_to_json!(
+        config.rpc_state_reader_config.json_rpc_version,
+        config.rpc_state_reader_config.url,
+        config.batcher_config.storage.db_config.path_prefix,
+        config
+            .gateway_config
+            .stateful_tx_validator_config
+            .chain_info
+            .fee_token_addresses
+            .eth_fee_token_address,
+        config
+            .gateway_config
+            .stateful_tx_validator_config
+            .chain_info
+            .fee_token_addresses
+            .strk_fee_token_address
+    );
+    dump_json_data(json_data, CONFIG_PARAMETERS_PATH)?;
+
+    // Create config file for the transaction generator.
+    let json_data =
+        config_fields_to_json!(config.http_server_config.ip, config.http_server_config.port);
+    dump_json_data(json_data, TX_GEN_CONFIG_PARAMETERS_PATH)?;
+
+    Ok(())
+}
+
+fn dump_json_data(json_data: Value, path: &str) -> Result<()> {
+    // Serialize the JSON data to a pretty-printed string
+    let json_string = serde_json::to_string_pretty(&json_data).unwrap();
+
+    // Write the JSON string to a file
+    let mut file = File::create(path)?;
+    file.write_all(json_string.as_bytes())?;
+    info!("Writing JSON data to: {:?}", path);
+
+    Ok(())
+}
+
+fn strip_config_prefix(input: &str) -> &str {
+    input.strip_prefix("config.").unwrap_or(input)
+}

crates/tests-integration/src/integration_test_utils.rs

@@ -92,7 +92,9 @@ impl HttpTestClient {
     }
 }
 
-fn test_rpc_state_reader_config(rpc_server_addr: SocketAddr) -> RpcStateReaderConfig {
+// TODO(Tsabary): move all public functions to be at the start of this module.
+
+pub fn test_rpc_state_reader_config(rpc_server_addr: SocketAddr) -> RpcStateReaderConfig {
     const RPC_SPEC_VERION: &str = "V0_8";
     const JSON_RPC_VERSION: &str = "2.0";
     RpcStateReaderConfig {

crates/tests-integration/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod integration_test_config_utils;
 pub mod integration_test_setup;
 pub mod integration_test_utils;
 pub mod state_reader;