feat(starknet_state_sync): create state sync component

config/sequencer/default_config.json

@@ -944,6 +944,176 @@
     "privacy": "Public",
     "value": ""
   },
+  "state_sync_config.network_config.advertised_multiaddr": {
+    "description": "The external address other peers see this node. If this is set, the node will not try to find out which addresses it has and will write this address as external instead",
+    "privacy": "Public",
+    "value": ""
+  },
+  "state_sync_config.network_config.advertised_multiaddr.#is_none": {
+    "description": "Flag for an optional field.",
+    "privacy": "TemporaryValue",
+    "value": true
+  },
+  "state_sync_config.network_config.bootstrap_peer_multiaddr": {
+    "description": "The multiaddress of the peer node. It should include the peer's id. For more info: https://docs.libp2p.io/concepts/fundamentals/peers/",
+    "privacy": "Public",
+    "value": ""
+  },
+  "state_sync_config.network_config.bootstrap_peer_multiaddr.#is_none": {
+    "description": "Flag for an optional field.",
+    "privacy": "TemporaryValue",
+    "value": true
+  },
+  "state_sync_config.network_config.chain_id": {
+    "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.",
+    "pointer_target": "chain_id",
+    "privacy": "Public"
+  },
+  "state_sync_config.network_config.discovery_config.bootstrap_dial_retry_config.base_delay_millis": {
+    "description": "The base delay in milliseconds for the exponential backoff strategy.",
+    "privacy": "Public",
+    "value": 2
+  },
+  "state_sync_config.network_config.discovery_config.bootstrap_dial_retry_config.factor": {
+    "description": "The factor for the exponential backoff strategy.",
+    "privacy": "Public",
+    "value": 5
+  },
+  "state_sync_config.network_config.discovery_config.bootstrap_dial_retry_config.max_delay_seconds": {
+    "description": "The maximum delay in seconds for the exponential backoff strategy.",
+    "privacy": "Public",
+    "value": 5
+  },
+  "state_sync_config.network_config.discovery_config.heartbeat_interval": {
+    "description": "The interval between each discovery (Kademlia) query in milliseconds.",
+    "privacy": "Public",
+    "value": 100
+  },
+  "state_sync_config.network_config.idle_connection_timeout": {
+    "description": "Amount of time in seconds that a connection with no active sessions will stay alive.",
+    "privacy": "Public",
+    "value": 120
+  },
+  "state_sync_config.network_config.peer_manager_config.malicious_timeout_seconds": {
+    "description": "The duration in seconds a peer is blacklisted after being marked as malicious.",
+    "privacy": "Public",
+    "value": 31536000
+  },
+  "state_sync_config.network_config.peer_manager_config.unstable_timeout_millis": {
+    "description": "The duration in milliseconds a peer blacklisted after being reported as unstable.",
+    "privacy": "Public",
+    "value": 1000
+  },
+  "state_sync_config.network_config.quic_port": {
+    "description": "The port that the node listens on for incoming quic connections.",
+    "privacy": "Public",
+    "value": 10001
+  },
+  "state_sync_config.network_config.secret_key": {
+    "description": "The secret key used for building the peer id. If it's an empty string a random one will be used.",
+    "privacy": "Private",
+    "value": ""
+  },
+  "state_sync_config.network_config.session_timeout": {
+    "description": "Maximal time in seconds that each session can take before failing on timeout.",
+    "privacy": "Public",
+    "value": 120
+  },
+  "state_sync_config.network_config.tcp_port": {
+    "description": "The port that the node listens on for incoming tcp connections.",
+    "privacy": "Public",
+    "value": 10000
+  },
+  "state_sync_config.p2p_sync_client_config.buffer_size": {
+    "description": "Size of the buffer for read from the storage and for incoming responses.",
+    "privacy": "Public",
+    "value": 100000
+  },
+  "state_sync_config.p2p_sync_client_config.num_block_classes_per_query": {
+    "description": "The maximum amount of block's classes to ask from peers in each iteration.",
+    "privacy": "Public",
+    "value": 100
+  },
+  "state_sync_config.p2p_sync_client_config.num_block_state_diffs_per_query": {
+    "description": "The maximum amount of block's state diffs to ask from peers in each iteration.",
+    "privacy": "Public",
+    "value": 100
+  },
+  "state_sync_config.p2p_sync_client_config.num_block_transactions_per_query": {
+    "description": "The maximum amount of blocks to ask their transactions from peers in each iteration.",
+    "privacy": "Public",
+    "value": 100
+  },
+  "state_sync_config.p2p_sync_client_config.num_headers_per_query": {
+    "description": "The maximum amount of headers to ask from peers in each iteration.",
+    "privacy": "Public",
+    "value": 10000
+  },
+  "state_sync_config.p2p_sync_client_config.stop_sync_at_block_number": {
+    "description": "Stops the sync at given block number and closes the node cleanly. Used to run profiling on the node.",
+    "privacy": "Public",
+    "value": 1000
+  },
+  "state_sync_config.p2p_sync_client_config.stop_sync_at_block_number.#is_none": {
+    "description": "Flag for an optional field.",
+    "privacy": "TemporaryValue",
+    "value": true
+  },
+  "state_sync_config.p2p_sync_client_config.wait_period_for_new_data": {
+    "description": "Time in seconds to wait when a query returned with partial data before sending a new query",
+    "privacy": "Public",
+    "value": 5
+  },
+  "state_sync_config.storage_config.db_config.chain_id": {
+    "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.",
+    "pointer_target": "chain_id",
+    "privacy": "Public"
+  },
+  "state_sync_config.storage_config.db_config.enforce_file_exists": {
+    "description": "Whether to enforce that the path exists. If true, `open_env` fails when the mdbx.dat file does not exist.",
+    "privacy": "Public",
+    "value": false
+  },
+  "state_sync_config.storage_config.db_config.growth_step": {
+    "description": "The growth step in bytes, must be greater than zero to allow the database to grow.",
+    "privacy": "Public",
+    "value": 4294967296
+  },
+  "state_sync_config.storage_config.db_config.max_size": {
+    "description": "The maximum size of the node's storage in bytes.",
+    "privacy": "Public",
+    "value": 1099511627776
+  },
+  "state_sync_config.storage_config.db_config.min_size": {
+    "description": "The minimum size of the node's storage in bytes.",
+    "privacy": "Public",
+    "value": 1048576
+  },
+  "state_sync_config.storage_config.db_config.path_prefix": {
+    "description": "Prefix of the path of the node's storage directory, the storage file path will be <path_prefix>/<chain_id>. The path is not created automatically.",
+    "privacy": "Public",
+    "value": "./data"
+  },
+  "state_sync_config.storage_config.mmap_file_config.growth_step": {
+    "description": "The growth step in bytes, must be greater than max_object_size.",
+    "privacy": "Public",
+    "value": 1073741824
+  },
+  "state_sync_config.storage_config.mmap_file_config.max_object_size": {
+    "description": "The maximum size of a single object in the file in bytes",
+    "privacy": "Public",
+    "value": 268435456
+  },
+  "state_sync_config.storage_config.mmap_file_config.max_size": {
+    "description": "The maximum size of a memory mapped file in bytes. Must be greater than growth_step.",
+    "privacy": "Public",
+    "value": 1099511627776
+  },
+  "state_sync_config.storage_config.scope": {
+    "description": "The categories of data saved in storage.",
+    "privacy": "Public",
+    "value": "FullArchive"
+  },
   "strk_fee_token_address": {
     "description": "A required param! Address of the STRK fee token.",
     "param_type": "String",

crates/starknet_integration_tests/Cargo.toml

@@ -36,6 +36,7 @@ starknet_mempool_p2p.workspace = true
 starknet_monitoring_endpoint = { workspace = true, features = ["testing"] }
 starknet_sequencer_infra = { workspace = true, features = ["testing"] }
 starknet_sequencer_node = { workspace = true, features = ["testing"] }
+starknet_state_sync.workspace = true
 starknet_task_executor.workspace = true
 strum.workspace = true
 tempfile.workspace = true

crates/starknet_integration_tests/src/config_utils.rs

@@ -52,6 +52,8 @@ pub(crate) fn dump_config_file_changes(
         config.http_server_config.ip,
         config.http_server_config.port,
         config.consensus_manager_config.consensus_config.start_height,
+        config.state_sync_config.storage_config.db_config.path_prefix,
+        config.state_sync_config.network_config.tcp_port,
     );
     let node_config_path = dump_json_data(json_data, NODE_CONFIG_CHANGES_FILE_PATH, dir);
     assert!(node_config_path.exists(), "File does not exist: {:?}", node_config_path);

crates/starknet_integration_tests/src/flow_test_setup.rs

@@ -28,6 +28,7 @@ pub struct FlowTestSetup {
     // Handlers for the storage files, maintained so the files are not deleted.
     pub batcher_storage_file_handle: TempDir,
     pub rpc_storage_file_handle: TempDir,
+    pub state_sync_storage_file_handle: TempDir,
 
     // Handle of the sequencer node.
     pub sequencer_node_handle: JoinHandle<Result<(), anyhow::Error>>,
@@ -56,9 +57,13 @@ impl FlowTestSetup {
         .await;
 
         // Derive the configuration for the sequencer node.
-        let (config, _required_params, consensus_proposals_channels) =
-            create_config(chain_info, rpc_server_addr, storage_for_test.batcher_storage_config)
-                .await;
+        let (config, _required_params, consensus_proposals_channels) = create_config(
+            chain_info,
+            rpc_server_addr,
+            storage_for_test.batcher_storage_config,
+            storage_for_test.state_sync_storage_config,
+        )
+        .await;
 
         let (_clients, servers) = create_node_modules(&config);
 
@@ -79,6 +84,7 @@ impl FlowTestSetup {
             add_tx_http_client,
             batcher_storage_file_handle: storage_for_test.batcher_storage_handle,
             rpc_storage_file_handle: storage_for_test.rpc_storage_handle,
+            state_sync_storage_file_handle: storage_for_test.state_sync_storage_handle,
             sequencer_node_handle,
             consensus_proposals_channels,
         }

crates/starknet_integration_tests/src/integration_test_setup.rs

@@ -22,6 +22,8 @@ pub struct IntegrationTestSetup {
     pub node_config_path: PathBuf,
     // Storage reader for the batcher.
     pub batcher_storage_config: StorageConfig,
+    // Storage reader for the state sync.
+    pub state_sync_storage_config: StorageConfig,
     // Handlers for the storage and config files, maintained so the files are not deleted. Since
     // these are only maintained to avoid dropping the handlers, private visibility suffices, and
     // as such, the '#[allow(dead_code)]' attributes are used to suppress the warning.
@@ -31,6 +33,8 @@ pub struct IntegrationTestSetup {
     rpc_storage_handle: TempDir,
     #[allow(dead_code)]
     node_config_dir_handle: TempDir,
+    #[allow(dead_code)]
+    state_sync_storage_handle: TempDir,
 }
 
 impl IntegrationTestSetup {
@@ -47,9 +51,13 @@ impl IntegrationTestSetup {
         .await;
 
         // Derive the configuration for the sequencer node.
-        let (config, required_params, _) =
-            create_config(chain_info, rpc_server_addr, storage_for_test.batcher_storage_config)
-                .await;
+        let (config, required_params, _) = create_config(
+            chain_info,
+            rpc_server_addr,
+            storage_for_test.batcher_storage_config,
+            storage_for_test.state_sync_storage_config,
+        )
+        .await;
 
         let node_config_dir_handle = tempdir().unwrap();
         let node_config_path = dump_config_file_changes(
@@ -73,6 +81,8 @@ impl IntegrationTestSetup {
             rpc_storage_handle: storage_for_test.rpc_storage_handle,
             node_config_dir_handle,
             node_config_path,
+            state_sync_storage_handle: storage_for_test.state_sync_storage_handle,
+            state_sync_storage_config: config.state_sync_config.storage_config,
         }
     }
 }

crates/starknet_integration_tests/src/state_reader.rs

@@ -55,6 +55,8 @@ pub struct StorageTestSetup {
     pub rpc_storage_handle: TempDir,
     pub batcher_storage_config: StorageConfig,
     pub batcher_storage_handle: TempDir,
+    pub state_sync_storage_config: StorageConfig,
+    pub state_sync_storage_handle: TempDir,
 }
 
 impl StorageTestSetup {
@@ -67,12 +69,23 @@ impl StorageTestSetup {
                 .scope(StorageScope::StateOnly)
                 .chain_id(chain_info.chain_id.clone())
                 .build();
-        create_test_state(&mut batcher_storage_writer, chain_info, test_defined_accounts);
+        create_test_state(&mut batcher_storage_writer, chain_info, test_defined_accounts.clone());
+        let (
+            (_, mut state_sync_storage_writer),
+            state_sync_storage_config,
+            state_sync_storage_handle,
+        ) = TestStorageBuilder::default()
+            .scope(StorageScope::FullArchive)
+            .chain_id(chain_info.chain_id.clone())
+            .build();
+        create_test_state(&mut state_sync_storage_writer, chain_info, test_defined_accounts);
         Self {
             rpc_storage_reader,
             rpc_storage_handle: rpc_storage_file_handle,
             batcher_storage_config,
             batcher_storage_handle: batcher_storage_file_handle,
+            state_sync_storage_config,
+            state_sync_storage_handle,
         }
     }
 }

crates/starknet_integration_tests/src/utils.rs

@@ -9,6 +9,7 @@ use mempool_test_utils::starknet_api_test_utils::{AccountId, MultiAccountTransac
 use papyrus_consensus::config::ConsensusConfig;
 use papyrus_network::network_manager::test_utils::create_network_configs_connected_to_broadcast_channels;
 use papyrus_network::network_manager::BroadcastTopicChannels;
+use papyrus_network::NetworkConfig;
 use papyrus_protobuf::consensus::{ProposalPart, StreamMessage};
 use papyrus_storage::StorageConfig;
 use starknet_api::block::BlockNumber;
@@ -27,6 +28,7 @@ use starknet_http_server::config::HttpServerConfig;
 use starknet_sequencer_infra::test_utils::get_available_socket;
 use starknet_sequencer_node::config::node_config::SequencerNodeConfig;
 use starknet_sequencer_node::config::test_utils::RequiredParams;
+use starknet_state_sync::config::StateSyncConfig;
 
 pub fn create_chain_info() -> ChainInfo {
     let mut chain_info = ChainInfo::create_for_testing();
@@ -40,6 +42,7 @@ pub async fn create_config(
     chain_info: ChainInfo,
     rpc_server_addr: SocketAddr,
     batcher_storage_config: StorageConfig,
+    state_sync_storage_config: StorageConfig,
 ) -> (SequencerNodeConfig, RequiredParams, BroadcastTopicChannels<StreamMessage<ProposalPart>>) {
     let fee_token_addresses = chain_info.fee_token_addresses.clone();
     let batcher_config = create_batcher_config(batcher_storage_config, chain_info.clone());
@@ -49,13 +52,15 @@ pub async fn create_config(
     let (mut consensus_manager_configs, consensus_proposals_channels) =
         create_consensus_manager_configs_and_channels(1);
     let consensus_manager_config = consensus_manager_configs.pop().unwrap();
+    let state_sync_config = create_state_sync_config(state_sync_storage_config);
     (
         SequencerNodeConfig {
             batcher_config,
             consensus_manager_config,
             gateway_config,
             http_server_config,
             rpc_state_reader_config,
+            state_sync_config,
             ..SequencerNodeConfig::default()
         },
         RequiredParams {
@@ -227,3 +232,15 @@ pub fn create_batcher_config(
         ..Default::default()
     }
 }
+
+pub fn create_state_sync_config(state_sync_storage_config: StorageConfig) -> StateSyncConfig {
+    const STATE_SYNC_NETWORK_CONFIG_TCP_PORT_FOR_TESTING: u16 = 12345;
+    StateSyncConfig {
+        storage_config: state_sync_storage_config,
+        network_config: NetworkConfig {
+            tcp_port: STATE_SYNC_NETWORK_CONFIG_TCP_PORT_FOR_TESTING,
+            ..Default::default()
+        },
+        ..Default::default()
+    }
+}