diff --git a/.github/scripts/release/build-deb.sh b/.github/scripts/release/build-deb.sh index 6cb833f98a4e4..8dce621bb4def 100755 --- a/.github/scripts/release/build-deb.sh +++ b/.github/scripts/release/build-deb.sh @@ -9,8 +9,7 @@ cargo install --version 2.7.0 cargo-deb --locked -q echo "Using cargo-deb v$(cargo-deb --version)" echo "Building a Debian package for '$PRODUCT' in '$PROFILE' profile" -# we need to start the custom version with a didgit as requires it cargo-deb -cargo deb --profile $PROFILE --no-strip --no-build -p $PRODUCT --deb-version 1-$VERSION +cargo deb --profile $PROFILE --no-strip --no-build -p $PRODUCT --deb-version $VERSION deb=target/debian/$PRODUCT_*_amd64.deb diff --git a/.github/workflows/release-10_rc-automation.yml b/.github/workflows/release-10_rc-automation.yml index 4ec4c05252b34..0be671185c70c 100644 --- a/.github/workflows/release-10_rc-automation.yml +++ b/.github/workflows/release-10_rc-automation.yml @@ -42,7 +42,7 @@ jobs: with: app-id: ${{ vars.RELEASE_AUTOMATION_APP_ID }} private-key: ${{ secrets.RELEASE_AUTOMATION_APP_PRIVATE_KEY }} - owner: paritytech-release + owner: paritytech - name: Checkout sources uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7 diff --git a/.github/workflows/release-branchoff-stable.yml b/.github/workflows/release-branchoff-stable.yml index 4249a274ffd76..adce1b261b71f 100644 --- a/.github/workflows/release-branchoff-stable.yml +++ b/.github/workflows/release-branchoff-stable.yml @@ -58,7 +58,7 @@ jobs: with: app-id: ${{ vars.RELEASE_AUTOMATION_APP_ID }} private-key: ${{ secrets.RELEASE_AUTOMATION_APP_PRIVATE_KEY }} - owner: paritytech-release + owner: paritytech - name: Checkout sources uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7 diff --git a/.github/workflows/release-reusable-rc-buid.yml b/.github/workflows/release-reusable-rc-buid.yml index d76f36e95c8d9..d925839fb84a0 100644 --- a/.github/workflows/release-reusable-rc-buid.yml +++ b/.github/workflows/release-reusable-rc-buid.yml @@ -151,7 +151,9 @@ jobs: - name: Build polkadot deb package shell: bash run: | - . "${GITHUB_WORKSPACE}"/.github/scripts/release/build-deb.sh ${{ inputs.package }} ${{ inputs.release_tag }} + . "${GITHUB_WORKSPACE}"/.github/scripts/common/lib.sh + VERSION=$(get_polkadot_node_version_from_code) + . "${GITHUB_WORKSPACE}"/.github/scripts/release/build-deb.sh ${{ inputs.package }} ${VERSION} - name: Generate artifact attestation uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3 diff --git a/.gitignore b/.gitignore index afa9ed33f4a03..d482876570857 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ artifacts bin/node-template/Cargo.lock nohup.out polkadot_argument_parsing -polkadot.* !docs/sdk/src/polkadot_sdk/polkadot.rs pwasm-alloc/Cargo.lock pwasm-libc/Cargo.lock diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 15d9a5fb5fca2..3dab49a118e5b 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -241,7 +241,7 @@ zombienet-polkadot-functional-0017-sync-backing: --local-dir="${LOCAL_DIR}/functional" --test="0017-sync-backing.zndsl" -.zombienet-polkadot-functional-0018-shared-core-idle-parachain: +zombienet-polkadot-functional-0018-shared-core-idle-parachain: extends: - .zombienet-polkadot-common before_script: diff --git a/Cargo.lock b/Cargo.lock index 539bacd008bff..05f30ad944959 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6764,9 +6764,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -6789,9 +6789,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -6799,15 +6799,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -6817,9 +6817,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -6851,9 +6851,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.37", @@ -6872,15 +6872,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -6894,9 +6894,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -14703,6 +14703,7 @@ dependencies = [ "polkadot-node-metrics", "polkadot-node-primitives", "polkadot-node-subsystem", + "polkadot-node-subsystem-test-helpers", "polkadot-parachain-primitives", "polkadot-primitives", "procfs", @@ -14966,7 +14967,6 @@ dependencies = [ "sp-blockchain", "sp-consensus-babe", "sp-runtime 31.0.1", - "strum 0.26.3", "substrate-prometheus-endpoint", "thiserror", ] @@ -15999,7 +15999,6 @@ dependencies = [ "sc-transaction-pool-api", "serde", "serde_json", - "serial_test", "sp-api 26.0.0", "sp-authority-discovery", "sp-block-builder", @@ -19399,6 +19398,7 @@ dependencies = [ "asynchronous-codec", "bytes", "cid 0.9.0", + "criterion", "either", "fnv", "futures", @@ -19420,6 +19420,7 @@ dependencies = [ "rand", "sc-block-builder", "sc-client-api", + "sc-consensus", "sc-network-common", "sc-network-light", "sc-network-sync", @@ -20648,31 +20649,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serial_test" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" -dependencies = [ - "dashmap", - "futures", - "lazy_static", - "log", - "parking_lot 0.12.3", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" -dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.37", - "syn 2.0.87", -] - [[package]] name = "sha-1" version = "0.9.8" diff --git a/Cargo.toml b/Cargo.toml index 46051453bc5b9..dad503a31fc92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -796,7 +796,7 @@ frame-system-rpc-runtime-api = { path = "substrate/frame/system/rpc/runtime-api" frame-try-runtime = { path = "substrate/frame/try-runtime", default-features = false } fs4 = { version = "0.7.0" } fs_extra = { version = "1.3.0" } -futures = { version = "0.3.30" } +futures = { version = "0.3.31" } futures-channel = { version = "0.3.23" } futures-timer = { version = "3.0.2" } futures-util = { version = "0.3.30", default-features = false } @@ -1214,7 +1214,6 @@ serde-big-array = { version = "0.3.2" } serde_derive = { version = "1.0.117" } serde_json = { version = "1.0.132", default-features = false } serde_yaml = { version = "0.9" } -serial_test = { version = "2.0.0" } sha1 = { version = "0.10.6" } sha2 = { version = "0.10.7", default-features = false } sha3 = { version = "0.10.0", default-features = false } diff --git a/bridges/snowbridge/pallets/inbound-queue-v2/src/mock.rs b/bridges/snowbridge/pallets/inbound-queue-v2/src/mock.rs index fad62628c0f0b..07e0a5564e094 100644 --- a/bridges/snowbridge/pallets/inbound-queue-v2/src/mock.rs +++ b/bridges/snowbridge/pallets/inbound-queue-v2/src/mock.rs @@ -19,11 +19,13 @@ use sp_runtime::{ BuildStorage, FixedU128, MultiSignature, }; use sp_std::{convert::From, default::Default}; -use xcm::{latest::SendXcm, prelude::*}; +use xcm::prelude::*; use xcm_executor::AssetsInHolding; use crate::{self as inbound_queue}; +use xcm::latest::{ROCOCO_GENESIS_HASH, WESTEND_GENESIS_HASH}; + type Block = frame_system::mocking::MockBlock; frame_support::construct_runtime!( @@ -113,8 +115,8 @@ parameter_types! { pub const InitialFund: u128 = 1_000_000_000_000; pub const InboundQueuePalletInstance: u8 = 80; pub UniversalLocation: InteriorLocation = - [GlobalConsensus(Westend), Parachain(1002)].into(); - pub AssetHubFromEthereum: Location = Location::new(1,[GlobalConsensus(Westend),Parachain(1000)]); + [GlobalConsensus(ByGenesis(WESTEND_GENESIS_HASH)), Parachain(1002)].into(); + pub AssetHubFromEthereum: Location = Location::new(1,[GlobalConsensus(ByGenesis(WESTEND_GENESIS_HASH)),Parachain(1000)]); } #[cfg(feature = "runtime-benchmarks")] diff --git a/cumulus/client/service/src/lib.rs b/cumulus/client/service/src/lib.rs index 25b8ee10a931e..ae83f2ade3f6a 100644 --- a/cumulus/client/service/src/lib.rs +++ b/cumulus/client/service/src/lib.rs @@ -40,10 +40,7 @@ use sc_consensus::{ use sc_network::{config::SyncMode, service::traits::NetworkService, NetworkBackend}; use sc_network_sync::SyncingService; use sc_network_transactions::TransactionsHandlerController; -use sc_service::{ - build_polkadot_syncing_strategy, Configuration, NetworkStarter, SpawnTaskHandle, TaskManager, - WarpSyncConfig, -}; +use sc_service::{Configuration, NetworkStarter, SpawnTaskHandle, TaskManager, WarpSyncConfig}; use sc_telemetry::{log, TelemetryWorkerHandle}; use sc_utils::mpsc::TracingUnboundedSender; use sp_api::ProvideRuntimeApi; @@ -429,7 +426,7 @@ pub struct BuildNetworkParams< pub async fn build_network<'a, Block, Client, RCInterface, IQ, Network>( BuildNetworkParams { parachain_config, - mut net_config, + net_config, client, transaction_pool, para_id, @@ -500,16 +497,6 @@ where parachain_config.prometheus_config.as_ref().map(|config| &config.registry), ); - let syncing_strategy = build_polkadot_syncing_strategy( - parachain_config.protocol_id(), - parachain_config.chain_spec.fork_id(), - &mut net_config, - warp_sync_config, - client.clone(), - &spawn_handle, - parachain_config.prometheus_config.as_ref().map(|config| &config.registry), - )?; - sc_service::build_network(sc_service::BuildNetworkParams { config: parachain_config, net_config, @@ -518,7 +505,7 @@ where spawn_handle, import_queue, block_announce_validator_builder: Some(Box::new(move |_| block_announce_validator)), - syncing_strategy, + warp_sync_config, block_relay: None, metrics, }) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/genesis_config_presets.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/genesis_config_presets.rs index f440b5a2f421a..824544e3b6879 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/genesis_config_presets.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/genesis_config_presets.rs @@ -18,6 +18,7 @@ use crate::*; use alloc::{vec, vec::Vec}; use cumulus_primitives_core::ParaId; +use frame_support::build_struct_json_patch; use hex_literal::hex; use parachains_common::{AccountId, AuraId}; use sp_core::crypto::UncheckedInto; @@ -35,15 +36,14 @@ fn asset_hub_westend_genesis( endowment: Balance, id: ParaId, ) -> serde_json::Value { - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts.iter().cloned().map(|k| (k, endowment)).collect(), }, - parachain_info: ParachainInfoConfig { parachain_id: id, ..Default::default() }, + parachain_info: ParachainInfoConfig { parachain_id: id }, collator_selection: CollatorSelectionConfig { invulnerables: invulnerables.iter().cloned().map(|(acc, _)| acc).collect(), candidacy_bond: ASSET_HUB_WESTEND_ED * 16, - ..Default::default() }, session: SessionConfig { keys: invulnerables @@ -56,16 +56,9 @@ fn asset_hub_westend_genesis( ) }) .collect(), - ..Default::default() }, - polkadot_xcm: PolkadotXcmConfig { - safe_xcm_version: Some(SAFE_XCM_VERSION), - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + polkadot_xcm: PolkadotXcmConfig { safe_xcm_version: Some(SAFE_XCM_VERSION) }, + }) } /// Encapsulates names of predefined presets. diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs index d155efccd7494..63175222cc26e 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs @@ -2000,7 +2000,7 @@ impl_runtime_apis! { fn fee_asset() -> Result { Ok(Asset { id: AssetId(WestendLocation::get()), - fun: Fungible(1_000_000 * UNITS), + fun: Fungible(1_000 * UNITS), }) } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/genesis_config_presets.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/genesis_config_presets.rs index 1796fde28d36a..679d33f3456a1 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/genesis_config_presets.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/genesis_config_presets.rs @@ -18,10 +18,12 @@ use crate::*; use alloc::{vec, vec::Vec}; use cumulus_primitives_core::ParaId; +use frame_support::build_struct_json_patch; use parachains_common::{AccountId, AuraId}; use sp_genesis_builder::PresetId; use sp_keyring::Sr25519Keyring; use testnet_parachains_constants::rococo::xcm_version::SAFE_XCM_VERSION; +use xcm::latest::WESTEND_GENESIS_HASH; const BRIDGE_HUB_ROCOCO_ED: Balance = ExistentialDeposit::get(); use xcm::latest::WESTEND_GENESIS_HASH; @@ -34,7 +36,7 @@ fn bridge_hub_rococo_genesis( asset_hub_para_id: ParaId, opened_bridges: Vec<(Location, InteriorLocation, Option)>, ) -> serde_json::Value { - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -42,11 +44,10 @@ fn bridge_hub_rococo_genesis( .map(|k| (k, 1u128 << 60)) .collect::>(), }, - parachain_info: ParachainInfoConfig { parachain_id: id, ..Default::default() }, + parachain_info: ParachainInfoConfig { parachain_id: id }, collator_selection: CollatorSelectionConfig { invulnerables: invulnerables.iter().cloned().map(|(acc, _)| acc).collect(), candidacy_bond: BRIDGE_HUB_ROCOCO_ED * 16, - ..Default::default() }, session: SessionConfig { keys: invulnerables @@ -59,33 +60,15 @@ fn bridge_hub_rococo_genesis( ) }) .collect(), - ..Default::default() - }, - polkadot_xcm: PolkadotXcmConfig { - safe_xcm_version: Some(SAFE_XCM_VERSION), - ..Default::default() - }, - bridge_westend_grandpa: BridgeWestendGrandpaConfig { - owner: bridges_pallet_owner.clone(), - ..Default::default() }, + polkadot_xcm: PolkadotXcmConfig { safe_xcm_version: Some(SAFE_XCM_VERSION) }, + bridge_westend_grandpa: BridgeWestendGrandpaConfig { owner: bridges_pallet_owner.clone() }, bridge_westend_messages: BridgeWestendMessagesConfig { owner: bridges_pallet_owner.clone(), - ..Default::default() }, - xcm_over_bridge_hub_westend: XcmOverBridgeHubWestendConfig { - opened_bridges, - ..Default::default() - }, - ethereum_system: EthereumSystemConfig { - para_id: id, - asset_hub_para_id, - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + xcm_over_bridge_hub_westend: XcmOverBridgeHubWestendConfig { opened_bridges }, + ethereum_system: EthereumSystemConfig { para_id: id, asset_hub_para_id }, + }) } /// Provides the JSON representation of predefined genesis config for given `id`. diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_ethereum_config.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_ethereum_config.rs index 5c9a38135f9fa..e25caed95a025 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_ethereum_config.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_ethereum_config.rs @@ -295,3 +295,48 @@ pub mod benchmark_helpers { } } } + +pub(crate) mod migrations { + use alloc::vec::Vec; + use frame_support::pallet_prelude::*; + use snowbridge_core::TokenId; + + #[frame_support::storage_alias] + pub type OldNativeToForeignId = StorageMap< + snowbridge_pallet_system::Pallet, + Blake2_128Concat, + xcm::v4::Location, + TokenId, + OptionQuery, + >; + + /// One shot migration for NetworkId::Westend to NetworkId::ByGenesis(WESTEND_GENESIS_HASH) + pub struct MigrationForXcmV5(core::marker::PhantomData); + impl frame_support::traits::OnRuntimeUpgrade + for MigrationForXcmV5 + { + fn on_runtime_upgrade() -> Weight { + let mut weight = T::DbWeight::get().reads(1); + + let translate_westend = |pre: xcm::v4::Location| -> Option { + weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 1)); + Some(xcm::v5::Location::try_from(pre).expect("valid location")) + }; + snowbridge_pallet_system::ForeignToNativeId::::translate_values(translate_westend); + + let old_keys = OldNativeToForeignId::::iter_keys().collect::>(); + for old_key in old_keys { + if let Some(old_val) = OldNativeToForeignId::::get(&old_key) { + snowbridge_pallet_system::NativeToForeignId::::insert( + &xcm::v5::Location::try_from(old_key.clone()).expect("valid location"), + old_val, + ); + } + OldNativeToForeignId::::remove(old_key); + weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 2)); + } + + weight + } + } +} diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_rococo_config.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_rococo_config.rs index 2b81639c5e3c8..62c93da7c831b 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_rococo_config.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/bridge_to_rococo_config.rs @@ -334,7 +334,6 @@ mod tests { pub mod migration { use super::*; use bp_messages::LegacyLaneId; - use frame_support::traits::ConstBool; parameter_types! { pub AssetHubWestendToAssetHubRococoMessagesLane: LegacyLaneId = LegacyLaneId([0, 0, 0, 2]); @@ -342,18 +341,6 @@ pub mod migration { pub AssetHubRococoUniversalLocation: InteriorLocation = [GlobalConsensus(RococoGlobalConsensusNetwork::get()), Parachain(bp_asset_hub_rococo::ASSET_HUB_ROCOCO_PARACHAIN_ID)].into(); } - /// Ensure that the existing lanes for the AHW<>AHR bridge are correctly configured. - pub type StaticToDynamicLanes = pallet_xcm_bridge_hub::migration::OpenBridgeForLane< - Runtime, - XcmOverBridgeHubRococoInstance, - AssetHubWestendToAssetHubRococoMessagesLane, - // the lanes are already created for AHR<>AHW, but we need to link them to the bridge - // structs - ConstBool, - AssetHubWestendLocation, - AssetHubRococoUniversalLocation, - >; - mod v1_wrong { use bp_messages::{LaneState, MessageNonce, UnrewardedRelayer}; use bp_runtime::AccountIdOf; diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/genesis_config_presets.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/genesis_config_presets.rs index 421c362467745..69ba9ca9ece79 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/genesis_config_presets.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/genesis_config_presets.rs @@ -18,6 +18,7 @@ use crate::*; use alloc::{vec, vec::Vec}; use cumulus_primitives_core::ParaId; +use frame_support::build_struct_json_patch; use parachains_common::{AccountId, AuraId}; use sp_genesis_builder::PresetId; use sp_keyring::Sr25519Keyring; @@ -34,7 +35,7 @@ fn bridge_hub_westend_genesis( asset_hub_para_id: ParaId, opened_bridges: Vec<(Location, InteriorLocation, Option)>, ) -> serde_json::Value { - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -42,11 +43,10 @@ fn bridge_hub_westend_genesis( .map(|k| (k, 1u128 << 60)) .collect::>(), }, - parachain_info: ParachainInfoConfig { parachain_id: id, ..Default::default() }, + parachain_info: ParachainInfoConfig { parachain_id: id }, collator_selection: CollatorSelectionConfig { invulnerables: invulnerables.iter().cloned().map(|(acc, _)| acc).collect(), candidacy_bond: BRIDGE_HUB_WESTEND_ED * 16, - ..Default::default() }, session: SessionConfig { keys: invulnerables @@ -59,33 +59,13 @@ fn bridge_hub_westend_genesis( ) }) .collect(), - ..Default::default() }, - polkadot_xcm: PolkadotXcmConfig { - safe_xcm_version: Some(SAFE_XCM_VERSION), - ..Default::default() - }, - bridge_rococo_grandpa: BridgeRococoGrandpaConfig { - owner: bridges_pallet_owner.clone(), - ..Default::default() - }, - bridge_rococo_messages: BridgeRococoMessagesConfig { - owner: bridges_pallet_owner.clone(), - ..Default::default() - }, - xcm_over_bridge_hub_rococo: XcmOverBridgeHubRococoConfig { - opened_bridges, - ..Default::default() - }, - ethereum_system: EthereumSystemConfig { - para_id: id, - asset_hub_para_id, - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + polkadot_xcm: PolkadotXcmConfig { safe_xcm_version: Some(SAFE_XCM_VERSION) }, + bridge_rococo_grandpa: BridgeRococoGrandpaConfig { owner: bridges_pallet_owner.clone() }, + bridge_rococo_messages: BridgeRococoMessagesConfig { owner: bridges_pallet_owner.clone() }, + xcm_over_bridge_hub_rococo: XcmOverBridgeHubRococoConfig { opened_bridges }, + ethereum_system: EthereumSystemConfig { para_id: id, asset_hub_para_id }, + }) } /// Provides the JSON representation of predefined genesis config for given `id`. diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs index 39f4c4cb4f235..fa9231796b59e 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs @@ -159,20 +159,20 @@ pub type Migrations = ( Runtime, bridge_to_rococo_config::WithBridgeHubRococoMessagesInstance, >, - bridge_to_rococo_config::migration::StaticToDynamicLanes, frame_support::migrations::RemoveStorage< BridgeRococoMessagesPalletName, OutboundLanesCongestedSignalsKey, RocksDbWeight, >, pallet_bridge_relayers::migration::v1::MigrationToV1, - // permanent - pallet_xcm::migration::MigrateToLatestXcmVersion, snowbridge_pallet_system::migration::v0::InitializeOnUpgrade< Runtime, ConstU32, ConstU32, >, + bridge_to_ethereum_config::migrations::MigrationForXcmV5, + // permanent + pallet_xcm::migration::MigrateToLatestXcmVersion, ); parameter_types! { diff --git a/cumulus/parachains/runtimes/bridge-hubs/common/src/message_queue.rs b/cumulus/parachains/runtimes/bridge-hubs/common/src/message_queue.rs index 0e79883b4df71..cdc4c741d863c 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/common/src/message_queue.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/common/src/message_queue.rs @@ -24,7 +24,7 @@ use pallet_message_queue::OnQueueChanged; use scale_info::TypeInfo; use snowbridge_core::ChannelId; use sp_core::H256; -use xcm::prelude::{Junction, Location}; +use xcm::latest::prelude::{Junction, Location}; /// The aggregate origin of an inbound message. /// This is specialized for BridgeHub, as the snowbridge-outbound-queue-pallet is also using diff --git a/cumulus/parachains/runtimes/collectives/collectives-westend/src/genesis_config_presets.rs b/cumulus/parachains/runtimes/collectives/collectives-westend/src/genesis_config_presets.rs index 77e971ff8ad7f..007ff6164a741 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-westend/src/genesis_config_presets.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-westend/src/genesis_config_presets.rs @@ -18,6 +18,7 @@ use crate::*; use alloc::{vec, vec::Vec}; use cumulus_primitives_core::ParaId; +use frame_support::build_struct_json_patch; use parachains_common::{AccountId, AuraId}; use sp_genesis_builder::PresetId; use sp_keyring::Sr25519Keyring; @@ -30,7 +31,7 @@ fn collectives_westend_genesis( endowed_accounts: Vec, id: ParaId, ) -> serde_json::Value { - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -38,11 +39,10 @@ fn collectives_westend_genesis( .map(|k| (k, COLLECTIVES_WESTEND_ED * 4096)) .collect::>(), }, - parachain_info: ParachainInfoConfig { parachain_id: id, ..Default::default() }, + parachain_info: ParachainInfoConfig { parachain_id: id }, collator_selection: CollatorSelectionConfig { invulnerables: invulnerables.iter().cloned().map(|(acc, _)| acc).collect(), candidacy_bond: COLLECTIVES_WESTEND_ED * 16, - ..Default::default() }, session: SessionConfig { keys: invulnerables @@ -55,16 +55,9 @@ fn collectives_westend_genesis( ) }) .collect(), - ..Default::default() }, - polkadot_xcm: PolkadotXcmConfig { - safe_xcm_version: Some(SAFE_XCM_VERSION), - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + polkadot_xcm: PolkadotXcmConfig { safe_xcm_version: Some(SAFE_XCM_VERSION) }, + }) } /// Provides the JSON representation of predefined genesis config for given `id`. diff --git a/cumulus/polkadot-omni-node/lib/src/nodes/manual_seal.rs b/cumulus/polkadot-omni-node/lib/src/nodes/manual_seal.rs index e8043bd7b2aac..b7fc3489da254 100644 --- a/cumulus/polkadot-omni-node/lib/src/nodes/manual_seal.rs +++ b/cumulus/polkadot-omni-node/lib/src/nodes/manual_seal.rs @@ -25,7 +25,7 @@ use cumulus_primitives_core::ParaId; use sc_consensus::{DefaultImportQueue, LongestChain}; use sc_consensus_manual_seal::rpc::{ManualSeal, ManualSealApiServer}; use sc_network::NetworkBackend; -use sc_service::{build_polkadot_syncing_strategy, Configuration, PartialComponents, TaskManager}; +use sc_service::{Configuration, PartialComponents, TaskManager}; use sc_telemetry::TelemetryHandle; use sp_runtime::traits::Header; use std::{marker::PhantomData, sync::Arc}; @@ -85,7 +85,7 @@ impl ManualSealNode { // Since this is a dev node, prevent it from connecting to peers. config.network.default_peers_set.in_peers = 0; config.network.default_peers_set.out_peers = 0; - let mut net_config = sc_network::config::FullNetworkConfiguration::<_, _, Net>::new( + let net_config = sc_network::config::FullNetworkConfiguration::<_, _, Net>::new( &config.network, config.prometheus_config.as_ref().map(|cfg| cfg.registry.clone()), ); @@ -93,16 +93,6 @@ impl ManualSealNode { config.prometheus_config.as_ref().map(|cfg| &cfg.registry), ); - let syncing_strategy = build_polkadot_syncing_strategy( - config.protocol_id(), - config.chain_spec.fork_id(), - &mut net_config, - None, - client.clone(), - &task_manager.spawn_handle(), - config.prometheus_config.as_ref().map(|config| &config.registry), - )?; - let (network, system_rpc_tx, tx_handler_controller, start_network, sync_service) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, @@ -112,7 +102,7 @@ impl ManualSealNode { import_queue, net_config, block_announce_validator_builder: None, - syncing_strategy, + warp_sync_config: None, block_relay: None, metrics, })?; diff --git a/docs/sdk/packages/guides/first-runtime/src/lib.rs b/docs/sdk/packages/guides/first-runtime/src/lib.rs index 7c96f5653e527..61ca550c87507 100644 --- a/docs/sdk/packages/guides/first-runtime/src/lib.rs +++ b/docs/sdk/packages/guides/first-runtime/src/lib.rs @@ -130,23 +130,21 @@ pub mod genesis_config_presets { interface::{Balance, MinimumBalance}, BalancesConfig, RuntimeGenesisConfig, SudoConfig, }; + use frame::deps::frame_support::build_struct_json_patch; use serde_json::Value; /// Returns a development genesis config preset. #[docify::export] pub fn development_config_genesis() -> Value { let endowment = >::get().max(1) * 1000; - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: AccountKeyring::iter() .map(|a| (a.to_account_id(), endowment)) .collect::>(), }, sudo: SudoConfig { key: Some(AccountKeyring::Alice.to_account_id()) }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + }) } /// Get the set of the available genesis config presets. diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index b5362d32ad88d..30121418a2fde 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -632,6 +632,7 @@ async fn request_candidate_validation( ) -> Result { let (tx, rx) = oneshot::channel(); let is_system = candidate_receipt.descriptor.para_id().is_system(); + let relay_parent = candidate_receipt.descriptor.relay_parent(); sender .send_message(CandidateValidationMessage::ValidateFromExhaustive { @@ -641,9 +642,9 @@ async fn request_candidate_validation( pov, executor_params, exec_kind: if is_system { - PvfExecKind::BackingSystemParas + PvfExecKind::BackingSystemParas(relay_parent) } else { - PvfExecKind::Backing + PvfExecKind::Backing(relay_parent) }, response_sender: tx, }) diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index dbb974a634fe6..97e25c04282c4 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -435,7 +435,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && candidate_receipt.descriptor == assert_candidate.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -652,7 +652,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && candidate_receipt.descriptor == candidate_a.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1288,7 +1288,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && candidate_receipt.descriptor == candidate_a.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1455,7 +1455,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && candidate_receipt.descriptor == candidate_a.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1622,7 +1622,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && candidate_receipt.descriptor == candidate_a.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1662,7 +1662,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && candidate_receipt.descriptor == candidate_b.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1789,7 +1789,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && candidate_receipt.descriptor == candidate.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1933,7 +1933,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && candidate_receipt.descriptor == candidate.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2212,7 +2212,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && candidate_receipt.descriptor == candidate.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2754,7 +2754,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && candidate_receipt.descriptor == candidate.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index caddd24080578..db5409ee4bd50 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -276,7 +276,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && candidate_receipt.descriptor == candidate.descriptor && - exec_kind == PvfExecKind::BackingSystemParas && + matches!(exec_kind, PvfExecKind::BackingSystemParas(_)) && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 1e732e2f1f03e..25614349486ea 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -31,13 +31,16 @@ use polkadot_node_primitives::{InvalidCandidate, PoV, ValidationResult}; use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{ - CandidateValidationMessage, PreCheckOutcome, PvfExecKind, RuntimeApiMessage, - RuntimeApiRequest, ValidationFailed, + CandidateValidationMessage, ChainApiMessage, PreCheckOutcome, PvfExecKind, + RuntimeApiMessage, RuntimeApiRequest, ValidationFailed, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, }; -use polkadot_node_subsystem_util::{self as util, runtime::ClaimQueueSnapshot}; +use polkadot_node_subsystem_util::{ + self as util, + runtime::{prospective_parachains_mode, ClaimQueueSnapshot, ProspectiveParachainsMode}, +}; use polkadot_overseer::ActiveLeavesUpdate; use polkadot_parachain_primitives::primitives::ValidationResult as WasmValidationResult; use polkadot_primitives::{ @@ -279,6 +282,7 @@ async fn run( comm = ctx.recv().fuse() => { match comm { Ok(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update))) => { + update_active_leaves(ctx.sender(), validation_host.clone(), update.clone()).await; maybe_prepare_validation(ctx.sender(), keystore.clone(), validation_host.clone(), update, &mut prepare_state).await; }, Ok(FromOrchestra::Signal(OverseerSignal::BlockFinalized(..))) => {}, @@ -551,6 +555,66 @@ where Some(processed_code_hashes) } +async fn update_active_leaves( + sender: &mut Sender, + mut validation_backend: impl ValidationBackend, + update: ActiveLeavesUpdate, +) where + Sender: SubsystemSender + SubsystemSender, +{ + let ancestors = get_block_ancestors(sender, update.activated.as_ref().map(|x| x.hash)).await; + if let Err(err) = validation_backend.update_active_leaves(update, ancestors).await { + gum::warn!( + target: LOG_TARGET, + ?err, + "cannot update active leaves in validation backend", + ); + }; +} + +async fn get_allowed_ancestry_len(sender: &mut Sender, relay_parent: Hash) -> Option +where + Sender: SubsystemSender + SubsystemSender, +{ + match prospective_parachains_mode(sender, relay_parent).await { + Ok(ProspectiveParachainsMode::Enabled { allowed_ancestry_len, .. }) => + Some(allowed_ancestry_len), + res => { + gum::warn!(target: LOG_TARGET, ?res, "async backing is disabled"); + None + }, + } +} + +async fn get_block_ancestors( + sender: &mut Sender, + maybe_relay_parent: Option, +) -> Vec +where + Sender: SubsystemSender + SubsystemSender, +{ + let Some(relay_parent) = maybe_relay_parent else { return vec![] }; + let Some(allowed_ancestry_len) = get_allowed_ancestry_len(sender, relay_parent).await else { + return vec![] + }; + + let (tx, rx) = oneshot::channel(); + sender + .send_message(ChainApiMessage::Ancestors { + hash: relay_parent, + k: allowed_ancestry_len, + response_channel: tx, + }) + .await; + match rx.await { + Ok(Ok(x)) => x, + res => { + gum::warn!(target: LOG_TARGET, ?res, "cannot request ancestors"); + vec![] + }, + } +} + struct RuntimeRequestFailed; async fn runtime_api_request( @@ -698,7 +762,7 @@ async fn validate_candidate_exhaustive( // We only check the session index for backing. match (exec_kind, candidate_receipt.descriptor.session_index()) { - (PvfExecKind::Backing | PvfExecKind::BackingSystemParas, Some(session_index)) => { + (PvfExecKind::Backing(_) | PvfExecKind::BackingSystemParas(_), Some(session_index)) => { let Some(expected_session_index) = maybe_expected_session_index else { let error = "cannot fetch session index from the runtime"; gum::warn!( @@ -731,7 +795,7 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecKind::Backing | PvfExecKind::BackingSystemParas => { + PvfExecKind::Backing(_) | PvfExecKind::BackingSystemParas(_) => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( @@ -809,6 +873,15 @@ async fn validate_candidate_exhaustive( ); Err(ValidationFailed(e.to_string())) }, + Err(e @ ValidationError::ExecutionDeadline) => { + gum::warn!( + target: LOG_TARGET, + ?para_id, + ?e, + "Job assigned too late, execution queue probably overloaded", + ); + Err(ValidationFailed(e.to_string())) + }, Ok(res) => if res.head_data.hash() != candidate_receipt.descriptor.para_head() { gum::info!(target: LOG_TARGET, ?para_id, "Invalid candidate (para_head)"); @@ -846,7 +919,7 @@ async fn validate_candidate_exhaustive( // descriptor core index. ( Some(_core_index), - PvfExecKind::Backing | PvfExecKind::BackingSystemParas, + PvfExecKind::Backing(_) | PvfExecKind::BackingSystemParas(_), ) => { let Some(claim_queue) = maybe_claim_queue else { let error = "cannot fetch the claim queue from the runtime"; @@ -994,7 +1067,12 @@ trait ValidationBackend { retry_immediately = true; }, - Ok(_) | Err(ValidationError::Invalid(_) | ValidationError::Preparation(_)) => break, + Ok(_) | + Err( + ValidationError::Invalid(_) | + ValidationError::Preparation(_) | + ValidationError::ExecutionDeadline, + ) => break, } // If we got a possibly transient error, retry once after a brief delay, on the @@ -1035,6 +1113,12 @@ trait ValidationBackend { async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<(), PrepareError>; async fn heads_up(&mut self, active_pvfs: Vec) -> Result<(), String>; + + async fn update_active_leaves( + &mut self, + update: ActiveLeavesUpdate, + ancestors: Vec, + ) -> Result<(), String>; } #[async_trait] @@ -1085,6 +1169,14 @@ impl ValidationBackend for ValidationHost { async fn heads_up(&mut self, active_pvfs: Vec) -> Result<(), String> { self.heads_up(active_pvfs).await } + + async fn update_active_leaves( + &mut self, + update: ActiveLeavesUpdate, + ancestors: Vec, + ) -> Result<(), String> { + self.update_active_leaves(update, ancestors).await + } } /// Does basic checks of a candidate. Provide the encoded PoV-block. Returns `Ok` if basic checks diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 391247858ed60..98e34a1cb4c13 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -473,6 +473,14 @@ impl ValidationBackend for MockValidateCandidateBackend { async fn heads_up(&mut self, _active_pvfs: Vec) -> Result<(), String> { unreachable!() } + + async fn update_active_leaves( + &mut self, + _update: ActiveLeavesUpdate, + _ancestors: Vec, + ) -> Result<(), String> { + unreachable!() + } } #[test] @@ -531,7 +539,7 @@ fn session_index_checked_only_in_backing() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )) @@ -670,7 +678,7 @@ fn candidate_validation_ok_is_ok(#[case] v2_descriptor: bool) { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Some(ClaimQueueSnapshot(cq)), )) @@ -748,7 +756,7 @@ fn invalid_session_or_core_index() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )) @@ -764,7 +772,7 @@ fn invalid_session_or_core_index() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::BackingSystemParas, + PvfExecKind::BackingSystemParas(dummy_hash()), &Default::default(), Default::default(), )) @@ -782,7 +790,7 @@ fn invalid_session_or_core_index() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Some(Default::default()), )) @@ -797,7 +805,7 @@ fn invalid_session_or_core_index() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::BackingSystemParas, + PvfExecKind::BackingSystemParas(dummy_hash()), &Default::default(), Some(Default::default()), )) @@ -866,7 +874,7 @@ fn invalid_session_or_core_index() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Some(ClaimQueueSnapshot(cq.clone())), )) @@ -889,7 +897,7 @@ fn invalid_session_or_core_index() { candidate_receipt.clone(), Arc::new(pov.clone()), ExecutorParams::default(), - PvfExecKind::BackingSystemParas, + PvfExecKind::BackingSystemParas(dummy_hash()), &Default::default(), Some(ClaimQueueSnapshot(cq)), )) @@ -944,7 +952,7 @@ fn candidate_validation_bad_return_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )) @@ -1102,7 +1110,7 @@ fn candidate_validation_retry_internal_errors() { #[test] fn candidate_validation_dont_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. @@ -1136,7 +1144,7 @@ fn candidate_validation_retry_panic_errors() { #[test] fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -1233,7 +1241,7 @@ fn candidate_validation_timeout_is_internal_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )); @@ -1282,7 +1290,7 @@ fn candidate_validation_commitment_hash_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )) @@ -1337,7 +1345,7 @@ fn candidate_validation_code_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )) @@ -1397,7 +1405,7 @@ fn compressed_code_works() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecKind::Backing(dummy_hash()), &Default::default(), Default::default(), )); @@ -1436,6 +1444,14 @@ impl ValidationBackend for MockPreCheckBackend { async fn heads_up(&mut self, _active_pvfs: Vec) -> Result<(), String> { unreachable!() } + + async fn update_active_leaves( + &mut self, + _update: ActiveLeavesUpdate, + _ancestors: Vec, + ) -> Result<(), String> { + unreachable!() + } } #[test] @@ -1592,6 +1608,14 @@ impl ValidationBackend for MockHeadsUp { let _ = self.heads_up_call_count.fetch_add(1, Ordering::SeqCst); Ok(()) } + + async fn update_active_leaves( + &mut self, + _update: ActiveLeavesUpdate, + _ancestors: Vec, + ) -> Result<(), String> { + unreachable!() + } } fn alice_keystore() -> KeystorePtr { diff --git a/polkadot/node/core/prospective-parachains/src/lib.rs b/polkadot/node/core/prospective-parachains/src/lib.rs index 34c1d8823bfce..92aea8509f8c4 100644 --- a/polkadot/node/core/prospective-parachains/src/lib.rs +++ b/polkadot/node/core/prospective-parachains/src/lib.rs @@ -524,7 +524,7 @@ async fn handle_introduce_seconded_candidate( }, }; - let mut added = false; + let mut added = Vec::with_capacity(view.per_relay_parent.len()); let mut para_scheduled = false; // We don't iterate only through the active leaves. We also update the deactivated parents in // the implicit view, so that their upcoming children may see these candidates. @@ -536,18 +536,10 @@ async fn handle_introduce_seconded_candidate( match chain.try_adding_seconded_candidate(&candidate_entry) { Ok(()) => { - gum::debug!( - target: LOG_TARGET, - ?para, - ?relay_parent, - ?is_active_leaf, - "Added seconded candidate {:?}", - candidate_hash - ); - added = true; + added.push(*relay_parent); }, Err(FragmentChainError::CandidateAlreadyKnown) => { - gum::debug!( + gum::trace!( target: LOG_TARGET, ?para, ?relay_parent, @@ -555,10 +547,10 @@ async fn handle_introduce_seconded_candidate( "Attempting to introduce an already known candidate: {:?}", candidate_hash ); - added = true; + added.push(*relay_parent); }, Err(err) => { - gum::debug!( + gum::trace!( target: LOG_TARGET, ?para, ?relay_parent, @@ -580,16 +572,24 @@ async fn handle_introduce_seconded_candidate( ); } - if !added { + if added.is_empty() { gum::debug!( target: LOG_TARGET, para = ?para, candidate = ?candidate_hash, "Newly-seconded candidate cannot be kept under any relay parent", ); + } else { + gum::debug!( + target: LOG_TARGET, + ?para, + "Added/Kept seconded candidate {:?} on relay parents: {:?}", + candidate_hash, + added + ); } - let _ = tx.send(added); + let _ = tx.send(!added.is_empty()); } async fn handle_candidate_backed( @@ -779,12 +779,12 @@ fn answer_hypothetical_membership_request( membership.push(*active_leaf); }, Err(err) => { - gum::debug!( + gum::trace!( target: LOG_TARGET, para = ?para_id, leaf = ?active_leaf, candidate = ?candidate.candidate_hash(), - "Candidate is not a hypothetical member: {}", + "Candidate is not a hypothetical member on: {}", err ) }, @@ -792,6 +792,19 @@ fn answer_hypothetical_membership_request( } } + for (candidate, membership) in &response { + if membership.is_empty() { + gum::debug!( + target: LOG_TARGET, + para = ?candidate.candidate_para(), + active_leaves = ?view.active_leaves, + ?required_active_leaf, + candidate = ?candidate.candidate_hash(), + "Candidate is not a hypothetical member on any of the active leaves", + ) + } + } + let _ = tx.send(response); } diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml index 13fcdc69a99a2..a9f97c308f266 100644 --- a/polkadot/node/core/pvf/Cargo.toml +++ b/polkadot/node/core/pvf/Cargo.toml @@ -52,6 +52,7 @@ criterion = { features = [ hex-literal = { workspace = true, default-features = true } polkadot-node-core-pvf-common = { features = ["test-utils"], workspace = true, default-features = true } +polkadot-node-subsystem-test-helpers = { workspace = true } # For benches and integration tests, depend on ourselves with the test-utils # feature. polkadot-node-core-pvf = { features = ["test-utils"], workspace = true, default-features = true } diff --git a/polkadot/node/core/pvf/src/error.rs b/polkadot/node/core/pvf/src/error.rs index a0634106052d7..e68ba595ef5a1 100644 --- a/polkadot/node/core/pvf/src/error.rs +++ b/polkadot/node/core/pvf/src/error.rs @@ -39,6 +39,11 @@ pub enum ValidationError { /// Preparation or execution issue caused by an internal condition. Should not vote against. #[error("candidate validation: internal: {0}")] Internal(#[from] InternalValidationError), + /// The execution deadline of allowed_ancestry_len + 1 has been reached. Jobs like backing have + /// a limited time to execute. Once the deadline is reached, the current candidate cannot be + /// backed, regardless of its validity. + #[error("candidate validation: execution deadline has been reached.")] + ExecutionDeadline, } /// A description of an error raised during executing a PVF and can be attributed to the combination diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 2ac5116912eb9..6d27ab0261d98 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -35,8 +35,8 @@ use polkadot_node_core_pvf_common::{ SecurityStatus, }; use polkadot_node_primitives::PoV; -use polkadot_node_subsystem::messages::PvfExecKind; -use polkadot_primitives::{ExecutorParams, ExecutorParamsHash, PersistedValidationData}; +use polkadot_node_subsystem::{messages::PvfExecKind, ActiveLeavesUpdate}; +use polkadot_primitives::{ExecutorParams, ExecutorParamsHash, Hash, PersistedValidationData}; use slotmap::HopSlotMap; use std::{ collections::{HashMap, VecDeque}, @@ -45,7 +45,7 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use strum::IntoEnumIterator; +use strum::{EnumIter, IntoEnumIterator}; /// The amount of time a job for which the queue does not have a compatible worker may wait in the /// queue. After that time passes, the queue will kill the first worker which becomes idle to @@ -58,6 +58,7 @@ slotmap::new_key_type! { struct Worker; } #[derive(Debug)] pub enum ToQueue { + UpdateActiveLeaves { update: ActiveLeavesUpdate, ancestors: Vec }, Enqueue { artifact: ArtifactPathId, pending_execution_request: PendingExecutionRequest }, } @@ -82,6 +83,7 @@ pub struct PendingExecutionRequest { struct ExecuteJob { artifact: ArtifactPathId, exec_timeout: Duration, + exec_kind: PvfExecKind, pvd: Arc, pov: Arc, executor_params: ExecutorParams, @@ -172,6 +174,9 @@ struct Queue { unscheduled: Unscheduled, workers: Workers, mux: Mux, + + /// Active leaves and their ancestors to check the viability of backing jobs. + active_leaves: HashMap>, } impl Queue { @@ -202,6 +207,7 @@ impl Queue { spawn_inflight: 0, capacity: worker_capacity, }, + active_leaves: Default::default(), } } @@ -278,15 +284,74 @@ impl Queue { } let job = queue.remove(job_index).expect("Job is just checked to be in queue; qed"); + let exec_kind = job.exec_kind; if let Some(worker) = worker { assign(self, worker, job); } else { spawn_extra_worker(self, job); } - self.metrics.on_execute_kind(priority); + self.metrics.on_execute_kind(exec_kind); self.unscheduled.mark_scheduled(priority); } + + fn update_active_leaves(&mut self, update: ActiveLeavesUpdate, ancestors: Vec) { + self.prune_deactivated_leaves(&update); + self.insert_active_leaf(update, ancestors); + self.prune_old_jobs(); + } + + fn prune_deactivated_leaves(&mut self, update: &ActiveLeavesUpdate) { + for hash in &update.deactivated { + let _ = self.active_leaves.remove(&hash); + } + + gum::debug!(target: LOG_TARGET, size = ?self.active_leaves.len(), "Active leaves pruned"); + } + + fn insert_active_leaf(&mut self, update: ActiveLeavesUpdate, ancestors: Vec) { + let Some(leaf) = update.activated else { return }; + let _ = self.active_leaves.insert(leaf.hash, ancestors); + } + + fn prune_old_jobs(&mut self) { + for &priority in &[Priority::Backing, Priority::BackingSystemParas] { + let Some(queue) = self.unscheduled.get_mut(priority) else { continue }; + let to_remove: Vec = queue + .iter() + .enumerate() + .filter_map(|(index, job)| { + let relay_parent = match job.exec_kind { + PvfExecKind::Backing(x) | PvfExecKind::BackingSystemParas(x) => x, + _ => return None, + }; + let in_active_fork = self.active_leaves.iter().any(|(hash, ancestors)| { + *hash == relay_parent || ancestors.contains(&relay_parent) + }); + if in_active_fork { + None + } else { + Some(index) + } + }) + .collect(); + + for &index in to_remove.iter().rev() { + if index > queue.len() { + continue + } + + let Some(job) = queue.remove(index) else { continue }; + let _ = job.result_tx.send(Err(ValidationError::ExecutionDeadline)); + gum::warn!( + target: LOG_TARGET, + ?priority, + exec_kind = ?job.exec_kind, + "Job exceeded its deadline and was dropped without execution", + ); + } + } + } } async fn purge_dead(metrics: &Metrics, workers: &mut Workers) { @@ -305,27 +370,40 @@ async fn purge_dead(metrics: &Metrics, workers: &mut Workers) { } fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { - let ToQueue::Enqueue { artifact, pending_execution_request } = to_queue; - let PendingExecutionRequest { exec_timeout, pvd, pov, executor_params, result_tx, exec_kind } = - pending_execution_request; - gum::debug!( - target: LOG_TARGET, - validation_code_hash = ?artifact.id.code_hash, - "enqueueing an artifact for execution", - ); - queue.metrics.observe_pov_size(pov.block_data.0.len(), true); - queue.metrics.execute_enqueued(); - let job = ExecuteJob { - artifact, - exec_timeout, - pvd, - pov, - executor_params, - result_tx, - waiting_since: Instant::now(), - }; - queue.unscheduled.add(job, exec_kind); - queue.try_assign_next_job(None); + match to_queue { + ToQueue::UpdateActiveLeaves { update, ancestors } => { + queue.update_active_leaves(update, ancestors); + }, + ToQueue::Enqueue { artifact, pending_execution_request } => { + let PendingExecutionRequest { + exec_timeout, + pvd, + pov, + executor_params, + result_tx, + exec_kind, + } = pending_execution_request; + gum::debug!( + target: LOG_TARGET, + validation_code_hash = ?artifact.id.code_hash, + "enqueueing an artifact for execution", + ); + queue.metrics.observe_pov_size(pov.block_data.0.len(), true); + queue.metrics.execute_enqueued(); + let job = ExecuteJob { + artifact, + exec_timeout, + exec_kind, + pvd, + pov, + executor_params, + result_tx, + waiting_since: Instant::now(), + }; + queue.unscheduled.add(job, exec_kind.into()); + queue.try_assign_next_job(None); + }, + } } async fn handle_mux(queue: &mut Queue, event: QueueEvent) { @@ -648,9 +726,32 @@ pub fn start( (to_queue_tx, from_queue_rx, run) } +/// Priority of execution jobs based on PvfExecKind. +/// +/// The order is important, because we iterate through the values and assume it is going from higher +/// to lowest priority. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] +enum Priority { + Dispute, + Approval, + BackingSystemParas, + Backing, +} + +impl From for Priority { + fn from(kind: PvfExecKind) -> Self { + match kind { + PvfExecKind::Dispute => Priority::Dispute, + PvfExecKind::Approval => Priority::Approval, + PvfExecKind::BackingSystemParas(_) => Priority::BackingSystemParas, + PvfExecKind::Backing(_) => Priority::Backing, + } + } +} + struct Unscheduled { - unscheduled: HashMap>, - counter: HashMap, + unscheduled: HashMap>, + counter: HashMap, } impl Unscheduled { @@ -677,34 +778,34 @@ impl Unscheduled { /// approvals could not exceed 24%, even if there are no disputes. /// - We cannot fully prioritize backing system parachains over backing other parachains based /// on the distribution of the original 100%. - const PRIORITY_ALLOCATION_THRESHOLDS: &'static [(PvfExecKind, usize)] = &[ - (PvfExecKind::Dispute, 70), - (PvfExecKind::Approval, 80), - (PvfExecKind::BackingSystemParas, 100), - (PvfExecKind::Backing, 100), + const PRIORITY_ALLOCATION_THRESHOLDS: &'static [(Priority, usize)] = &[ + (Priority::Dispute, 70), + (Priority::Approval, 80), + (Priority::BackingSystemParas, 100), + (Priority::Backing, 100), ]; fn new() -> Self { Self { - unscheduled: PvfExecKind::iter().map(|priority| (priority, VecDeque::new())).collect(), - counter: PvfExecKind::iter().map(|priority| (priority, 0)).collect(), + unscheduled: Priority::iter().map(|priority| (priority, VecDeque::new())).collect(), + counter: Priority::iter().map(|priority| (priority, 0)).collect(), } } - fn select_next_priority(&self) -> PvfExecKind { + fn select_next_priority(&self) -> Priority { gum::debug!( target: LOG_TARGET, - unscheduled = ?self.unscheduled.iter().map(|(p, q)| (*p, q.len())).collect::>(), + unscheduled = ?self.unscheduled.iter().map(|(p, q)| (*p, q.len())).collect::>(), counter = ?self.counter, "Selecting next execution priority...", ); - let priority = PvfExecKind::iter() + let priority = Priority::iter() .find(|priority| self.has_pending(priority) && !self.has_reached_threshold(priority)) .unwrap_or_else(|| { - PvfExecKind::iter() + Priority::iter() .find(|priority| self.has_pending(priority)) - .unwrap_or(PvfExecKind::Backing) + .unwrap_or(Priority::Backing) }); gum::debug!( @@ -716,19 +817,19 @@ impl Unscheduled { priority } - fn get_mut(&mut self, priority: PvfExecKind) -> Option<&mut VecDeque> { + fn get_mut(&mut self, priority: Priority) -> Option<&mut VecDeque> { self.unscheduled.get_mut(&priority) } - fn add(&mut self, job: ExecuteJob, priority: PvfExecKind) { + fn add(&mut self, job: ExecuteJob, priority: Priority) { self.unscheduled.entry(priority).or_default().push_back(job); } - fn has_pending(&self, priority: &PvfExecKind) -> bool { + fn has_pending(&self, priority: &Priority) -> bool { !self.unscheduled.get(priority).unwrap_or(&VecDeque::new()).is_empty() } - fn priority_allocation_threshold(priority: &PvfExecKind) -> Option { + fn priority_allocation_threshold(priority: &Priority) -> Option { Self::PRIORITY_ALLOCATION_THRESHOLDS.iter().find_map(|&(p, value)| { if p == *priority { Some(value) @@ -740,7 +841,7 @@ impl Unscheduled { /// Checks if a given priority has reached its allocated threshold /// The thresholds are defined in `PRIORITY_ALLOCATION_THRESHOLDS`. - fn has_reached_threshold(&self, priority: &PvfExecKind) -> bool { + fn has_reached_threshold(&self, priority: &Priority) -> bool { let Some(threshold) = Self::priority_allocation_threshold(priority) else { return false }; let Some(count) = self.counter.get(&priority) else { return false }; // Every time we iterate by lower level priorities @@ -769,22 +870,28 @@ impl Unscheduled { has_reached_threshold } - fn mark_scheduled(&mut self, priority: PvfExecKind) { + fn mark_scheduled(&mut self, priority: Priority) { *self.counter.entry(priority).or_default() += 1; if self.counter.values().sum::() >= Self::SCHEDULING_WINDOW_SIZE { self.reset_counter(); } + gum::debug!( + target: LOG_TARGET, + ?priority, + "Job marked as scheduled", + ); } fn reset_counter(&mut self) { - self.counter = PvfExecKind::iter().map(|kind| (kind, 0)).collect(); + self.counter = Priority::iter().map(|kind| (kind, 0)).collect(); } } #[cfg(test)] mod tests { use polkadot_node_primitives::BlockData; + use polkadot_node_subsystem_test_helpers::mock::new_leaf; use sp_core::H256; use super::*; @@ -803,6 +910,7 @@ mod tests { ExecuteJob { artifact: ArtifactPathId { id: artifact_id(0), path: PathBuf::new() }, exec_timeout: Duration::from_secs(10), + exec_kind: PvfExecKind::Approval, pvd, pov, executor_params: ExecutorParams::default(), @@ -815,11 +923,11 @@ mod tests { fn test_unscheduled_add() { let mut unscheduled = Unscheduled::new(); - PvfExecKind::iter().for_each(|priority| { + Priority::iter().for_each(|priority| { unscheduled.add(create_execution_job(), priority); }); - PvfExecKind::iter().for_each(|priority| { + Priority::iter().for_each(|priority| { let queue = unscheduled.unscheduled.get(&priority).unwrap(); assert_eq!(queue.len(), 1); }); @@ -827,7 +935,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution() { - use PvfExecKind::*; + use Priority::*; let mut priorities = vec![]; @@ -852,7 +960,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution_without_backing_system_paras() { - use PvfExecKind::*; + use Priority::*; let mut priorities = vec![]; @@ -876,7 +984,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution_without_disputes() { - use PvfExecKind::*; + use Priority::*; let mut priorities = vec![]; @@ -900,7 +1008,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution_without_disputes_and_only_one_backing() { - use PvfExecKind::*; + use Priority::*; let mut priorities = vec![]; @@ -922,7 +1030,7 @@ mod tests { #[test] fn test_unscheduled_does_not_postpone_backing() { - use PvfExecKind::*; + use Priority::*; let mut priorities = vec![]; @@ -940,4 +1048,67 @@ mod tests { assert_eq!(&priorities[..4], &[Approval, Backing, Approval, Approval]); } + + #[tokio::test] + async fn test_prunes_old_jobs_on_active_leaves_update() { + // Set up a queue, but without a real worker, we won't execute any jobs. + let (_, to_queue_rx) = mpsc::channel(1); + let (from_queue_tx, _) = mpsc::unbounded(); + let mut queue = Queue::new( + Metrics::default(), + PathBuf::new(), + PathBuf::new(), + 1, + Duration::from_secs(1), + None, + SecurityStatus::default(), + to_queue_rx, + from_queue_tx, + ); + let old_relay_parent = Hash::random(); + let relevant_relay_parent = Hash::random(); + + assert_eq!(queue.unscheduled.unscheduled.values().map(|x| x.len()).sum::(), 0); + let mut result_rxs = vec![]; + let (result_tx, _result_rx) = oneshot::channel(); + let relevant_job = ExecuteJob { + artifact: ArtifactPathId { id: artifact_id(0), path: PathBuf::new() }, + exec_timeout: Duration::from_secs(1), + exec_kind: PvfExecKind::Backing(relevant_relay_parent), + pvd: Arc::new(PersistedValidationData::default()), + pov: Arc::new(PoV { block_data: BlockData(Vec::new()) }), + executor_params: ExecutorParams::default(), + result_tx, + waiting_since: Instant::now(), + }; + queue.unscheduled.add(relevant_job, Priority::Backing); + for _ in 0..10 { + let (result_tx, result_rx) = oneshot::channel(); + let expired_job = ExecuteJob { + artifact: ArtifactPathId { id: artifact_id(0), path: PathBuf::new() }, + exec_timeout: Duration::from_secs(1), + exec_kind: PvfExecKind::Backing(old_relay_parent), + pvd: Arc::new(PersistedValidationData::default()), + pov: Arc::new(PoV { block_data: BlockData(Vec::new()) }), + executor_params: ExecutorParams::default(), + result_tx, + waiting_since: Instant::now(), + }; + queue.unscheduled.add(expired_job, Priority::Backing); + result_rxs.push(result_rx); + } + assert_eq!(queue.unscheduled.unscheduled.values().map(|x| x.len()).sum::(), 11); + + // Add an active leaf + queue.update_active_leaves( + ActiveLeavesUpdate::start_work(new_leaf(Hash::random(), 1)), + vec![relevant_relay_parent], + ); + + // It prunes all old jobs and drops them with an `ExecutionDeadline` error. + for rx in result_rxs { + assert!(matches!(rx.await, Ok(Err(ValidationError::ExecutionDeadline)))); + } + assert_eq!(queue.unscheduled.unscheduled.values().map(|x| x.len()).sum::(), 1); + } } diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 37cd6fcbf74aa..8252904095b3f 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -37,9 +37,11 @@ use polkadot_node_core_pvf_common::{ pvf::PvfPrepData, }; use polkadot_node_primitives::PoV; -use polkadot_node_subsystem::{messages::PvfExecKind, SubsystemError, SubsystemResult}; +use polkadot_node_subsystem::{ + messages::PvfExecKind, ActiveLeavesUpdate, SubsystemError, SubsystemResult, +}; use polkadot_parachain_primitives::primitives::ValidationResult; -use polkadot_primitives::PersistedValidationData; +use polkadot_primitives::{Hash, PersistedValidationData}; use std::{ collections::HashMap, path::PathBuf, @@ -143,12 +145,27 @@ impl ValidationHost { .await .map_err(|_| "the inner loop hung up".to_string()) } + + /// Sends a signal to the validation host requesting to update best block. + /// + /// Returns an error if the request cannot be sent to the validation host, i.e. if it shut down. + pub async fn update_active_leaves( + &mut self, + update: ActiveLeavesUpdate, + ancestors: Vec, + ) -> Result<(), String> { + self.to_host_tx + .send(ToHost::UpdateActiveLeaves { update, ancestors }) + .await + .map_err(|_| "the inner loop hung up".to_string()) + } } enum ToHost { PrecheckPvf { pvf: PvfPrepData, result_tx: PrecheckResultSender }, ExecutePvf(ExecutePvfInputs), HeadsUp { active_pvfs: Vec }, + UpdateActiveLeaves { update: ActiveLeavesUpdate, ancestors: Vec }, } struct ExecutePvfInputs { @@ -488,6 +505,8 @@ async fn handle_to_host( }, ToHost::HeadsUp { active_pvfs } => handle_heads_up(artifacts, prepare_queue, active_pvfs).await?, + ToHost::UpdateActiveLeaves { update, ancestors } => + handle_update_active_leaves(execute_queue, update, ancestors).await?, } Ok(()) @@ -855,6 +874,14 @@ async fn handle_prepare_done( Ok(()) } +async fn handle_update_active_leaves( + execute_queue: &mut mpsc::Sender, + update: ActiveLeavesUpdate, + ancestors: Vec, +) -> Result<(), Fatal> { + send_execute(execute_queue, execute::ToQueue::UpdateActiveLeaves { update, ancestors }).await +} + async fn send_prepare( prepare_queue: &mut mpsc::Sender, to_queue: prepare::ToQueue, @@ -1255,7 +1282,7 @@ pub(crate) mod tests { pvd.clone(), pov1.clone(), Priority::Normal, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1268,7 +1295,7 @@ pub(crate) mod tests { pvd.clone(), pov1, Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1281,7 +1308,7 @@ pub(crate) mod tests { pvd, pov2, Priority::Normal, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1431,7 +1458,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1480,7 +1507,7 @@ pub(crate) mod tests { pvd, pov, Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1591,7 +1618,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1623,7 +1650,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx_2, ) .await @@ -1647,7 +1674,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx_3, ) .await @@ -1706,7 +1733,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await @@ -1738,7 +1765,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx_2, ) .await @@ -1762,7 +1789,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx_3, ) .await @@ -1837,7 +1864,7 @@ pub(crate) mod tests { pvd, pov, Priority::Normal, - PvfExecKind::Backing, + PvfExecKind::Backing(H256::default()), result_tx, ) .await diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 7aaeacf362209..5a58fbc8ade3a 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -43,8 +43,8 @@ impl From for Priority { match priority { PvfExecKind::Dispute => Priority::Critical, PvfExecKind::Approval => Priority::Critical, - PvfExecKind::BackingSystemParas => Priority::Normal, - PvfExecKind::Backing => Priority::Normal, + PvfExecKind::BackingSystemParas(_) => Priority::Normal, + PvfExecKind::Backing(_) => Priority::Normal, } } } diff --git a/polkadot/node/core/pvf/tests/it/adder.rs b/polkadot/node/core/pvf/tests/it/adder.rs index 1a95a28fe0773..924ea71667027 100644 --- a/polkadot/node/core/pvf/tests/it/adder.rs +++ b/polkadot/node/core/pvf/tests/it/adder.rs @@ -46,6 +46,7 @@ async fn execute_good_block_on_parent() { pvd, pov, Default::default(), + H256::default(), ) .await .unwrap(); @@ -82,6 +83,7 @@ async fn execute_good_chain_on_parent() { pvd, pov, Default::default(), + H256::default(), ) .await .unwrap(); @@ -120,6 +122,7 @@ async fn execute_bad_block_on_parent() { pvd, pov, Default::default(), + H256::default(), ) .await .unwrap_err(); @@ -145,6 +148,7 @@ async fn stress_spawn() { pvd, pov, Default::default(), + H256::default(), ) .await .unwrap(); @@ -185,6 +189,7 @@ async fn execute_can_run_serially() { pvd, pov, Default::default(), + H256::default(), ) .await .unwrap(); diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 4cbc6fb04a8ef..cfb78fd530d23 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -28,8 +28,8 @@ use polkadot_node_primitives::{PoV, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT}; use polkadot_node_subsystem::messages::PvfExecKind; use polkadot_parachain_primitives::primitives::{BlockData, ValidationResult}; use polkadot_primitives::{ - ExecutorParam, ExecutorParams, PersistedValidationData, PvfExecKind as RuntimePvfExecKind, - PvfPrepKind, + ExecutorParam, ExecutorParams, Hash, PersistedValidationData, + PvfExecKind as RuntimePvfExecKind, PvfPrepKind, }; use sp_core::H256; @@ -108,6 +108,7 @@ impl TestHost { pvd: PersistedValidationData, pov: PoV, executor_params: ExecutorParams, + relay_parent: Hash, ) -> Result { let (result_tx, result_rx) = futures::channel::oneshot::channel(); @@ -125,7 +126,7 @@ impl TestHost { Arc::new(pvd), Arc::new(pov), polkadot_node_core_pvf::Priority::Normal, - PvfExecKind::Backing, + PvfExecKind::Backing(relay_parent), result_tx, ) .await @@ -171,7 +172,13 @@ async fn execute_job_terminates_on_timeout() { let start = std::time::Instant::now(); let result = host - .validate_candidate(test_parachain_halt::wasm_binary_unwrap(), pvd, pov, Default::default()) + .validate_candidate( + test_parachain_halt::wasm_binary_unwrap(), + pvd, + pov, + Default::default(), + H256::default(), + ) .await; match result { @@ -201,12 +208,14 @@ async fn ensure_parallel_execution() { pvd.clone(), pov.clone(), Default::default(), + H256::default(), ); let execute_pvf_future_2 = host.validate_candidate( test_parachain_halt::wasm_binary_unwrap(), pvd, pov, Default::default(), + H256::default(), ); let start = std::time::Instant::now(); @@ -254,6 +263,7 @@ async fn execute_queue_doesnt_stall_if_workers_died() { pvd.clone(), pov.clone(), Default::default(), + H256::default(), ) })) .await; @@ -303,6 +313,7 @@ async fn execute_queue_doesnt_stall_with_varying_executor_params() { 0 => executor_params_1.clone(), _ => executor_params_2.clone(), }, + H256::default(), ) })) .await; @@ -359,7 +370,13 @@ async fn deleting_prepared_artifact_does_not_dispute() { // Try to validate, artifact should get recreated. let result = host - .validate_candidate(test_parachain_halt::wasm_binary_unwrap(), pvd, pov, Default::default()) + .validate_candidate( + test_parachain_halt::wasm_binary_unwrap(), + pvd, + pov, + Default::default(), + H256::default(), + ) .await; assert_matches!(result, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout))); @@ -410,7 +427,13 @@ async fn corrupted_prepared_artifact_does_not_dispute() { // Try to validate, artifact should get removed because of the corruption. let result = host - .validate_candidate(test_parachain_halt::wasm_binary_unwrap(), pvd, pov, Default::default()) + .validate_candidate( + test_parachain_halt::wasm_binary_unwrap(), + pvd, + pov, + Default::default(), + H256::default(), + ) .await; assert_matches!( @@ -684,7 +707,9 @@ async fn invalid_compressed_code_fails_validation() { let validation_code = sp_maybe_compressed_blob::compress(&raw_code, VALIDATION_CODE_BOMB_LIMIT + 1).unwrap(); - let result = host.validate_candidate(&validation_code, pvd, pov, Default::default()).await; + let result = host + .validate_candidate(&validation_code, pvd, pov, Default::default(), H256::default()) + .await; assert_matches!( result, @@ -708,7 +733,13 @@ async fn invalid_compressed_pov_fails_validation() { let pov = PoV { block_data: BlockData(block_data) }; let result = host - .validate_candidate(test_parachain_halt::wasm_binary_unwrap(), pvd, pov, Default::default()) + .validate_candidate( + test_parachain_halt::wasm_binary_unwrap(), + pvd, + pov, + Default::default(), + H256::default(), + ) .await; assert_matches!( diff --git a/polkadot/node/core/pvf/tests/it/process.rs b/polkadot/node/core/pvf/tests/it/process.rs index b3023c8a45c3b..353367b394f34 100644 --- a/polkadot/node/core/pvf/tests/it/process.rs +++ b/polkadot/node/core/pvf/tests/it/process.rs @@ -141,6 +141,7 @@ rusty_fork_test! { pvd, pov, Default::default(), + H256::default(), ) .await .unwrap(); @@ -187,6 +188,7 @@ rusty_fork_test! { pvd, pov, Default::default(), + H256::default(), ), // Send a stop signal to pause the worker. async { @@ -242,6 +244,7 @@ rusty_fork_test! { pvd, pov, Default::default(), + H256::default(), ), // Run a future that kills the job while it's running. async { @@ -301,6 +304,7 @@ rusty_fork_test! { pvd, pov, Default::default(), + H256::default(), ), // Run a future that kills the job while it's running. async { @@ -372,6 +376,7 @@ rusty_fork_test! { pvd, pov, Default::default(), + H256::default(), ), // Run a future that tests the thread count while the worker is running. async { diff --git a/polkadot/node/network/collator-protocol/src/error.rs b/polkadot/node/network/collator-protocol/src/error.rs index 0f5e0699d85c0..ae7f9a8c1fbc8 100644 --- a/polkadot/node/network/collator-protocol/src/error.rs +++ b/polkadot/node/network/collator-protocol/src/error.rs @@ -23,6 +23,7 @@ use polkadot_node_network_protocol::request_response::incoming; use polkadot_node_primitives::UncheckedSignedFullStatement; use polkadot_node_subsystem::{errors::SubsystemError, RuntimeApiError}; use polkadot_node_subsystem_util::{backing_implicit_view, runtime}; +use polkadot_primitives::vstaging::CandidateDescriptorVersion; use crate::LOG_TARGET; @@ -63,6 +64,12 @@ pub enum Error { #[error("CollationSeconded contained statement with invalid signature")] InvalidStatementSignature(UncheckedSignedFullStatement), + + #[error("Response receiver for session index request cancelled")] + CancelledSessionIndex(oneshot::Canceled), + + #[error("Response receiver for claim queue request cancelled")] + CancelledClaimQueue(oneshot::Canceled), } /// An error happened on the validator side of the protocol when attempting @@ -87,11 +94,23 @@ pub enum SecondingError { #[error("Candidate hash doesn't match the advertisement")] CandidateHashMismatch, + #[error("Relay parent hash doesn't match the advertisement")] + RelayParentMismatch, + #[error("Received duplicate collation from the peer")] Duplicate, #[error("The provided parent head data does not match the hash")] ParentHeadDataMismatch, + + #[error("Core index {0} present in descriptor is different than the assigned core {1}")] + InvalidCoreIndex(u32, u32), + + #[error("Session index {0} present in descriptor is different than the expected one {1}")] + InvalidSessionIndex(u32, u32), + + #[error("Invalid candidate receipt version {0:?}")] + InvalidReceiptVersion(CandidateDescriptorVersion), } impl SecondingError { @@ -102,7 +121,11 @@ impl SecondingError { self, PersistedValidationDataMismatch | CandidateHashMismatch | - Duplicate | ParentHeadDataMismatch + RelayParentMismatch | + Duplicate | ParentHeadDataMismatch | + InvalidCoreIndex(_, _) | + InvalidSessionIndex(_, _) | + InvalidReceiptVersion(_) ) } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 0b3e9f4b34317..cc0de1cb70f66 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -129,9 +129,6 @@ pub struct BlockedCollationId { } /// Performs a sanity check between advertised and fetched collations. -/// -/// Since the persisted validation data is constructed using the advertised -/// parent head data hash, the latter doesn't require an additional check. pub fn fetched_collation_sanity_check( advertised: &PendingCollation, fetched: &CandidateReceipt, @@ -139,17 +136,25 @@ pub fn fetched_collation_sanity_check( maybe_parent_head_and_hash: Option<(HeadData, Hash)>, ) -> Result<(), SecondingError> { if persisted_validation_data.hash() != fetched.descriptor().persisted_validation_data_hash() { - Err(SecondingError::PersistedValidationDataMismatch) - } else if advertised + return Err(SecondingError::PersistedValidationDataMismatch) + } + + if advertised .prospective_candidate .map_or(false, |pc| pc.candidate_hash() != fetched.hash()) { - Err(SecondingError::CandidateHashMismatch) - } else if maybe_parent_head_and_hash.map_or(false, |(head, hash)| head.hash() != hash) { - Err(SecondingError::ParentHeadDataMismatch) - } else { - Ok(()) + return Err(SecondingError::CandidateHashMismatch) + } + + if advertised.relay_parent != fetched.descriptor.relay_parent() { + return Err(SecondingError::RelayParentMismatch) } + + if maybe_parent_head_and_hash.map_or(false, |(head, hash)| head.hash() != hash) { + return Err(SecondingError::ParentHeadDataMismatch) + } + + Ok(()) } /// Identifier for a requested collation and the respective collator that advertised it. diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 51e987d59ce84..86358f503d04d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -49,11 +49,14 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_util::{ backing_implicit_view::View as ImplicitView, reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL}, - runtime::{fetch_claim_queue, prospective_parachains_mode, ProspectiveParachainsMode}, + request_claim_queue, request_session_index_for_child, + runtime::{prospective_parachains_mode, request_node_features, ProspectiveParachainsMode}, }; use polkadot_primitives::{ - vstaging::CoreState, CandidateHash, CollatorId, Hash, HeadData, Id as ParaId, - OccupiedCoreAssumption, PersistedValidationData, + node_features, + vstaging::{CandidateDescriptorV2, CandidateDescriptorVersion, CoreState}, + CandidateHash, CollatorId, CoreIndex, Hash, HeadData, Id as ParaId, OccupiedCoreAssumption, + PersistedValidationData, SessionIndex, }; use crate::error::{Error, FetchError, Result, SecondingError}; @@ -369,16 +372,9 @@ struct PerRelayParent { prospective_parachains_mode: ProspectiveParachainsMode, assignment: GroupAssignments, collations: Collations, -} - -impl PerRelayParent { - fn new(mode: ProspectiveParachainsMode) -> Self { - Self { - prospective_parachains_mode: mode, - assignment: GroupAssignments { current: vec![] }, - collations: Collations::default(), - } - } + v2_receipts: bool, + current_core: CoreIndex, + session_index: SessionIndex, } /// All state relevant for the validator side of the protocol lives here. @@ -460,14 +456,15 @@ fn is_relay_parent_in_implicit_view( } } -async fn assign_incoming( +async fn construct_per_relay_parent( sender: &mut Sender, - group_assignment: &mut GroupAssignments, current_assignments: &mut HashMap, keystore: &KeystorePtr, relay_parent: Hash, relay_parent_mode: ProspectiveParachainsMode, -) -> Result<()> + v2_receipts: bool, + session_index: SessionIndex, +) -> Result> where Sender: CollatorProtocolSenderTrait, { @@ -494,25 +491,25 @@ where rotation_info.core_for_group(group, cores.len()) } else { gum::trace!(target: LOG_TARGET, ?relay_parent, "Not a validator"); - return Ok(()) + return Ok(None) }; - let paras_now = match fetch_claim_queue(sender, relay_parent).await.map_err(Error::Runtime)? { - // Runtime supports claim queue - use it - // - // `relay_parent_mode` is not examined here because if the runtime supports claim queue - // then it supports async backing params too (`ASYNC_BACKING_STATE_RUNTIME_REQUIREMENT` - // < `CLAIM_QUEUE_RUNTIME_REQUIREMENT`). - Some(mut claim_queue) => claim_queue.0.remove(&core_now), - // Claim queue is not supported by the runtime - use availability cores instead. - None => cores.get(core_now.0 as usize).and_then(|c| match c { - CoreState::Occupied(core) if relay_parent_mode.is_enabled() => - core.next_up_on_available.as_ref().map(|c| [c.para_id].into_iter().collect()), - CoreState::Scheduled(core) => Some([core.para_id].into_iter().collect()), - CoreState::Occupied(_) | CoreState::Free => None, - }), - } - .unwrap_or_else(|| VecDeque::new()); + let claim_queue = request_claim_queue(relay_parent, sender) + .await + .await + .map_err(Error::CancelledClaimQueue)??; + + let paras_now = cores + .get(core_now.0 as usize) + .and_then(|c| match (c, relay_parent_mode) { + (CoreState::Occupied(_), ProspectiveParachainsMode::Disabled) => None, + ( + CoreState::Occupied(_), + ProspectiveParachainsMode::Enabled { max_candidate_depth: 0, .. }, + ) => None, + _ => claim_queue.get(&core_now).cloned(), + }) + .unwrap_or_else(|| VecDeque::new()); for para_id in paras_now.iter() { let entry = current_assignments.entry(*para_id).or_default(); @@ -527,9 +524,14 @@ where } } - *group_assignment = GroupAssignments { current: paras_now.into_iter().collect() }; - - Ok(()) + Ok(Some(PerRelayParent { + prospective_parachains_mode: relay_parent_mode, + assignment: GroupAssignments { current: paras_now.into_iter().collect() }, + collations: Collations::default(), + v2_receipts, + session_index, + current_core: core_now, + })) } fn remove_outgoing( @@ -1249,18 +1251,31 @@ where let added = view.iter().filter(|h| !current_leaves.contains_key(h)); for leaf in added { + let session_index = request_session_index_for_child(*leaf, sender) + .await + .await + .map_err(Error::CancelledSessionIndex)??; let mode = prospective_parachains_mode(sender, *leaf).await?; - - let mut per_relay_parent = PerRelayParent::new(mode); - assign_incoming( + let v2_receipts = request_node_features(*leaf, session_index, sender) + .await? + .unwrap_or_default() + .get(node_features::FeatureIndex::CandidateReceiptV2 as usize) + .map(|b| *b) + .unwrap_or(false); + + let Some(per_relay_parent) = construct_per_relay_parent( sender, - &mut per_relay_parent.assignment, &mut state.current_assignments, keystore, *leaf, mode, + v2_receipts, + session_index, ) - .await?; + .await? + else { + continue + }; state.active_leaves.insert(*leaf, mode); state.per_relay_parent.insert(*leaf, per_relay_parent); @@ -1279,18 +1294,21 @@ where .unwrap_or_default(); for block_hash in allowed_ancestry { if let Entry::Vacant(entry) = state.per_relay_parent.entry(*block_hash) { - let mut per_relay_parent = PerRelayParent::new(mode); - assign_incoming( + // Safe to use the same v2 receipts config for the allowed relay parents as well + // as the same session index since they must be in the same session. + if let Some(per_relay_parent) = construct_per_relay_parent( sender, - &mut per_relay_parent.assignment, &mut state.current_assignments, keystore, *block_hash, mode, + v2_receipts, + session_index, ) - .await?; - - entry.insert(per_relay_parent); + .await? + { + entry.insert(per_relay_parent); + } } } } @@ -1621,11 +1639,10 @@ async fn run_inner( Ok(FromOrchestra::Signal(OverseerSignal::Conclude)) | Err(_) => break, Ok(FromOrchestra::Signal(_)) => continue, } - } + }, _ = next_inactivity_stream.next() => { disconnect_inactive_peers(ctx.sender(), &eviction_policy, &state.peer_data).await; - } - + }, resp = state.collation_requests.select_next_some() => { let res = match handle_collation_fetch_response( &mut state, @@ -1684,7 +1701,7 @@ async fn run_inner( } Ok(true) => {} } - } + }, res = state.collation_fetch_timeouts.select_next_some() => { let (collator_id, maybe_candidate_hash, relay_parent) = res; gum::debug!( @@ -1814,6 +1831,10 @@ async fn kick_off_seconding( return Ok(false) }, }; + + // Sanity check of the candidate receipt version. + descriptor_version_sanity_check(candidate_receipt.descriptor(), per_relay_parent)?; + let collations = &mut per_relay_parent.collations; let fetched_collation = FetchedCollation::from(&candidate_receipt); @@ -2024,7 +2045,9 @@ async fn handle_collation_fetch_response( }, Ok( request_v1::CollationFetchingResponse::Collation(receipt, _) | - request_v1::CollationFetchingResponse::CollationWithParentHeadData { receipt, .. }, + request_v2::CollationFetchingResponse::Collation(receipt, _) | + request_v1::CollationFetchingResponse::CollationWithParentHeadData { receipt, .. } | + request_v2::CollationFetchingResponse::CollationWithParentHeadData { receipt, .. }, ) if receipt.descriptor().para_id() != pending_collation.para_id => { gum::debug!( target: LOG_TARGET, @@ -2086,3 +2109,35 @@ async fn handle_collation_fetch_response( state.metrics.on_request(metrics_result); result } + +// Sanity check the candidate descriptor version. +fn descriptor_version_sanity_check( + descriptor: &CandidateDescriptorV2, + per_relay_parent: &PerRelayParent, +) -> std::result::Result<(), SecondingError> { + match descriptor.version() { + CandidateDescriptorVersion::V1 => Ok(()), + CandidateDescriptorVersion::V2 if per_relay_parent.v2_receipts => { + if let Some(core_index) = descriptor.core_index() { + if core_index != per_relay_parent.current_core { + return Err(SecondingError::InvalidCoreIndex( + core_index.0, + per_relay_parent.current_core.0, + )) + } + } + + if let Some(session_index) = descriptor.session_index() { + if session_index != per_relay_parent.session_index { + return Err(SecondingError::InvalidSessionIndex( + session_index, + per_relay_parent.session_index, + )) + } + } + + Ok(()) + }, + descriptor_version => Err(SecondingError::InvalidReceiptVersion(descriptor_version)), + } +} diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 290c4db901d5b..7bc61dd4ebec5 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -42,9 +42,10 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt}; use polkadot_primitives::{ + node_features, vstaging::{CandidateReceiptV2 as CandidateReceipt, CoreState, OccupiedCore}, - CollatorPair, CoreIndex, GroupIndex, GroupRotationInfo, HeadData, PersistedValidationData, - ScheduledCore, ValidatorId, ValidatorIndex, + CollatorPair, CoreIndex, GroupIndex, GroupRotationInfo, HeadData, NodeFeatures, + PersistedValidationData, ScheduledCore, ValidatorId, ValidatorIndex, }; use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt_bad_sig, dummy_hash, @@ -78,6 +79,8 @@ struct TestState { group_rotation_info: GroupRotationInfo, cores: Vec, claim_queue: BTreeMap>, + node_features: NodeFeatures, + session_index: SessionIndex, } impl Default for TestState { @@ -132,6 +135,10 @@ impl Default for TestState { claim_queue.insert(CoreIndex(1), VecDeque::new()); claim_queue.insert(CoreIndex(2), [chain_ids[1]].into_iter().collect()); + let mut node_features = NodeFeatures::EMPTY; + node_features.resize(node_features::FeatureIndex::CandidateReceiptV2 as usize + 1, false); + node_features.set(node_features::FeatureIndex::CandidateReceiptV2 as u8 as usize, true); + Self { chain_ids, relay_parent, @@ -141,6 +148,8 @@ impl Default for TestState { group_rotation_info, cores, claim_queue, + node_features, + session_index: 1, } } } @@ -237,10 +246,44 @@ async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) .expect(&format!("{:?} is more than enough for sending signals.", TIMEOUT)); } -async fn respond_to_core_info_queries( +async fn respond_to_runtime_api_queries( virtual_overseer: &mut VirtualOverseer, test_state: &TestState, + hash: Hash, ) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::SessionIndexForChild(tx) + )) => { + assert_eq!(rp, hash); + tx.send(Ok(test_state.session_index)).unwrap(); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::AsyncBackingParams(tx) + )) => { + assert_eq!(rp, hash); + tx.send(Err(ASYNC_BACKING_DISABLED_ERROR)).unwrap(); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::NodeFeatures(_, tx) + )) => { + assert_eq!(rp, hash); + tx.send(Ok(test_state.node_features.clone())).unwrap(); + } + ); + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( @@ -254,9 +297,10 @@ async fn respond_to_core_info_queries( assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, + rp, RuntimeApiRequest::ValidatorGroups(tx), )) => { + assert_eq!(rp, hash); let _ = tx.send(Ok(( test_state.validator_groups.clone(), test_state.group_rotation_info.clone(), @@ -267,9 +311,10 @@ async fn respond_to_core_info_queries( assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, + rp, RuntimeApiRequest::AvailabilityCores(tx), )) => { + assert_eq!(rp, hash); let _ = tx.send(Ok(test_state.cores.clone())); } ); @@ -277,19 +322,10 @@ async fn respond_to_core_info_queries( assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Version(tx), - )) => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); - } - ); - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, + rp, RuntimeApiRequest::ClaimQueue(tx), )) => { + assert_eq!(rp, hash); let _ = tx.send(Ok(test_state.claim_queue.clone())); } ); @@ -470,19 +506,6 @@ async fn advertise_collation( .await; } -async fn assert_async_backing_params_request(virtual_overseer: &mut VirtualOverseer, hash: Hash) { - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - relay_parent, - RuntimeApiRequest::AsyncBackingParams(tx) - )) => { - assert_eq!(relay_parent, hash); - tx.send(Err(ASYNC_BACKING_DISABLED_ERROR)).unwrap(); - } - ); -} - // As we receive a relevant advertisement act on it and issue a collation request. #[test] fn act_on_advertisement() { @@ -502,8 +525,8 @@ fn act_on_advertisement() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -550,8 +573,8 @@ fn act_on_advertisement_v2() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -631,9 +654,8 @@ fn collator_reporting_works() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -748,8 +770,7 @@ fn fetch_one_collation_at_a_time() { // Iter over view since the order may change due to sorted invariant. for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, *hash).await; } let peer_b = PeerId::random(); @@ -847,8 +868,7 @@ fn fetches_next_collation() { .await; for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, *hash).await; } let peer_b = PeerId::random(); @@ -972,8 +992,8 @@ fn reject_connection_to_next_group() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -1024,8 +1044,7 @@ fn fetch_next_collation_on_invalid_collation() { .await; for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, *hash).await; } let peer_b = PeerId::random(); @@ -1135,8 +1154,8 @@ fn inactive_disconnected() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, hash_a).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -1189,8 +1208,7 @@ fn activity_extends_life() { .await; for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, *hash).await; } let peer_b = PeerId::random(); @@ -1263,8 +1281,8 @@ fn disconnect_if_no_declare() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -1302,8 +1320,8 @@ fn disconnect_if_wrong_declare() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -1364,8 +1382,8 @@ fn delay_reputation_change() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -1450,8 +1468,8 @@ fn view_change_clears_old_collators() { ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, test_state.relay_parent) + .await; let peer_b = PeerId::random(); @@ -1475,8 +1493,7 @@ fn view_change_clears_old_collators() { .await; test_state.group_rotation_info = test_state.group_rotation_info.bump_rotation(); - assert_async_backing_params_request(&mut virtual_overseer, hash_b).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + respond_to_runtime_api_queries(&mut virtual_overseer, &test_state, hash_b).await; assert_collator_disconnect(&mut virtual_overseer, peer_b).await; diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index e040163cd905b..eda26e8539a1d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -20,9 +20,10 @@ use super::*; use polkadot_node_subsystem::messages::ChainApiMessage; use polkadot_primitives::{ - vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, AsyncBackingParams, - BlockNumber, CandidateCommitments, Header, SigningContext, ValidatorId, + vstaging::{CommittedCandidateReceiptV2 as CommittedCandidateReceipt, MutateDescriptorV2}, + AsyncBackingParams, BlockNumber, CandidateCommitments, Header, SigningContext, ValidatorId, }; +use polkadot_primitives_test_helpers::dummy_committed_candidate_receipt_v2; use rstest::rstest; const ASYNC_BACKING_PARAMETERS: AsyncBackingParams = @@ -32,7 +33,7 @@ fn get_parent_hash(hash: Hash) -> Hash { Hash::from_low_u64_be(hash.to_low_u64_be() + 1) } -async fn assert_assign_incoming( +async fn assert_construct_per_relay_parent( virtual_overseer: &mut VirtualOverseer, test_state: &TestState, hash: Hash, @@ -73,16 +74,6 @@ async fn assert_assign_incoming( } ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - parent, - RuntimeApiRequest::Version(tx), - )) if parent == hash => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); - } - ); - assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( @@ -117,14 +108,34 @@ pub(super) async fn update_view( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( parent, + RuntimeApiRequest::SessionIndexForChild(tx) + )) => { + tx.send(Ok(test_state.session_index)).unwrap(); + (parent, new_view.get(&parent).copied().expect("Unknown parent requested")) + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, RuntimeApiRequest::AsyncBackingParams(tx), )) => { tx.send(Ok(ASYNC_BACKING_PARAMETERS)).unwrap(); - (parent, new_view.get(&parent).copied().expect("Unknown parent requested")) } ); - assert_assign_incoming( + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::NodeFeatures(_, tx) + )) => { + tx.send(Ok(test_state.node_features.clone())).unwrap(); + } + ); + + assert_construct_per_relay_parent( virtual_overseer, test_state, leaf_hash, @@ -193,7 +204,7 @@ pub(super) async fn update_view( // Skip the leaf. for (hash, number) in ancestry_iter.skip(1).take(requested_len.saturating_sub(1)) { - assert_assign_incoming( + assert_construct_per_relay_parent( virtual_overseer, test_state, hash, @@ -406,7 +417,7 @@ fn v1_advertisement_accepted_and_seconded() { } #[test] -fn v1_advertisement_rejected_on_non_active_leave() { +fn v1_advertisement_rejected_on_non_active_leaf() { let test_state = TestState::default(); test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { @@ -1314,3 +1325,223 @@ fn child_blocked_from_seconding_by_parent(#[case] valid_parent: bool) { virtual_overseer }); } + +#[rstest] +#[case(true)] +#[case(false)] +fn v2_descriptor(#[case] v2_feature_enabled: bool) { + let mut test_state = TestState::default(); + + if !v2_feature_enabled { + test_state.node_features = NodeFeatures::EMPTY; + } + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + let pair_a = CollatorPair::generate().0; + + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 0; + + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let peer_a = PeerId::random(); + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::V2, + ) + .await; + + let mut committed_candidate = dummy_committed_candidate_receipt_v2(head_b); + committed_candidate.descriptor.set_para_id(test_state.chain_ids[0]); + committed_candidate + .descriptor + .set_persisted_validation_data_hash(dummy_pvd().hash()); + // First para is assigned to core 0. + committed_candidate.descriptor.set_core_index(CoreIndex(0)); + committed_candidate.descriptor.set_session_index(test_state.session_index); + + let candidate: CandidateReceipt = committed_candidate.clone().to_plain(); + let pov = PoV { block_data: BlockData(vec![1]) }; + + let candidate_hash = candidate.hash(); + let parent_head_data_hash = Hash::zero(); + + advertise_collation( + &mut virtual_overseer, + peer_a, + head_b, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::CandidateBacking( + CandidateBackingMessage::CanSecond(request, tx), + ) => { + assert_eq!(request.candidate_hash, candidate_hash); + assert_eq!(request.candidate_para_id, test_state.chain_ids[0]); + assert_eq!(request.parent_head_data_hash, parent_head_data_hash); + tx.send(true).expect("receiving side should be alive"); + } + ); + + let response_channel = assert_fetch_collation_request( + &mut virtual_overseer, + head_b, + test_state.chain_ids[0], + Some(candidate_hash), + ) + .await; + + response_channel + .send(Ok(( + request_v2::CollationFetchingResponse::Collation(candidate.clone(), pov.clone()) + .encode(), + ProtocolName::from(""), + ))) + .expect("Sending response should succeed"); + + if v2_feature_enabled { + assert_candidate_backing_second( + &mut virtual_overseer, + head_b, + test_state.chain_ids[0], + &pov, + CollationVersion::V2, + ) + .await; + + send_seconded_statement(&mut virtual_overseer, keystore.clone(), &committed_candidate) + .await; + + assert_collation_seconded(&mut virtual_overseer, head_b, peer_a, CollationVersion::V2) + .await; + } else { + // Reported malicious. Used v2 descriptor without the feature being enabled + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::ReportPeer(ReportPeerMessage::Single(peer_id, rep)), + ) => { + assert_eq!(peer_a, peer_id); + assert_eq!(rep.value, COST_REPORT_BAD.cost_or_benefit()); + } + ); + } + + virtual_overseer + }); +} + +#[test] +fn invalid_v2_descriptor() { + let test_state = TestState::default(); + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, .. } = test_harness; + + let pair_a = CollatorPair::generate().0; + + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 0; + + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let peer_a = PeerId::random(); + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::V2, + ) + .await; + + let mut candidates = vec![]; + + let mut committed_candidate = dummy_committed_candidate_receipt_v2(head_b); + committed_candidate.descriptor.set_para_id(test_state.chain_ids[0]); + committed_candidate + .descriptor + .set_persisted_validation_data_hash(dummy_pvd().hash()); + // First para is assigned to core 0, set an invalid core index. + committed_candidate.descriptor.set_core_index(CoreIndex(10)); + committed_candidate.descriptor.set_session_index(test_state.session_index); + + candidates.push(committed_candidate.clone()); + + // Invalid session index. + committed_candidate.descriptor.set_core_index(CoreIndex(0)); + committed_candidate.descriptor.set_session_index(10); + + candidates.push(committed_candidate); + + for committed_candidate in candidates { + let candidate: CandidateReceipt = committed_candidate.clone().to_plain(); + let pov = PoV { block_data: BlockData(vec![1]) }; + + let candidate_hash = candidate.hash(); + let parent_head_data_hash = Hash::zero(); + + advertise_collation( + &mut virtual_overseer, + peer_a, + head_b, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::CandidateBacking( + CandidateBackingMessage::CanSecond(request, tx), + ) => { + assert_eq!(request.candidate_hash, candidate_hash); + assert_eq!(request.candidate_para_id, test_state.chain_ids[0]); + assert_eq!(request.parent_head_data_hash, parent_head_data_hash); + tx.send(true).expect("receiving side should be alive"); + } + ); + + let response_channel = assert_fetch_collation_request( + &mut virtual_overseer, + head_b, + test_state.chain_ids[0], + Some(candidate_hash), + ) + .await; + + response_channel + .send(Ok(( + request_v2::CollationFetchingResponse::Collation( + candidate.clone(), + pov.clone(), + ) + .encode(), + ProtocolName::from(""), + ))) + .expect("Sending response should succeed"); + + // Reported malicious. Invalid core index + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::ReportPeer(ReportPeerMessage::Single(peer_id, rep)), + ) => { + assert_eq!(peer_a, peer_id); + assert_eq!(rep.value, COST_REPORT_BAD.cost_or_benefit()); + } + ); + } + + virtual_overseer + }); +} diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index e1b2af733b47a..f2cf60280b723 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -83,7 +83,7 @@ impl Subsystem1 { candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecKind::Backing(dummy_hash()), response_sender: tx, }; ctx.send_message(msg).await; diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index 87ef63d8a5d70..3881ddbcc9046 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -468,6 +468,7 @@ pub async fn forward_events>(client: Arc

, mut hand )] pub struct Overseer { #[subsystem(CandidateValidationMessage, sends: [ + ChainApiMessage, RuntimeApiMessage, ])] candidate_validation: CandidateValidation, diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index c3c47335cd3e2..0b9b783ef9b15 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -111,7 +111,7 @@ where candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecKind::Backing(dummy_hash()), response_sender: tx, }) .await; @@ -811,7 +811,7 @@ fn test_candidate_validation_msg() -> CandidateValidationMessage { candidate_receipt, pov, executor_params: Default::default(), - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecKind::Backing(dummy_hash()), response_sender, } } diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 3edb3f4dadbed..6e8eade21a438 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -140,7 +140,6 @@ polkadot-node-subsystem-test-helpers = { workspace = true } polkadot-primitives-test-helpers = { workspace = true } sp-tracing = { workspace = true } assert_matches = { workspace = true } -serial_test = { workspace = true } tempfile = { workspace = true } [features] diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index abba91a38a97c..d2424474302a6 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -80,7 +80,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use prometheus_endpoint::Registry; #[cfg(feature = "full-node")] use sc_service::KeystoreContainer; -use sc_service::{build_polkadot_syncing_strategy, RpcHandlers, SpawnTaskHandle}; +use sc_service::{RpcHandlers, SpawnTaskHandle}; use sc_telemetry::TelemetryWorker; #[cfg(feature = "full-node")] use sc_telemetry::{Telemetry, TelemetryWorkerHandle}; @@ -1003,16 +1003,6 @@ pub fn new_full< }) }; - let syncing_strategy = build_polkadot_syncing_strategy( - config.protocol_id(), - config.chain_spec.fork_id(), - &mut net_config, - Some(WarpSyncConfig::WithProvider(warp_sync)), - client.clone(), - &task_manager.spawn_handle(), - config.prometheus_config.as_ref().map(|config| &config.registry), - )?; - let (network, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, @@ -1022,7 +1012,7 @@ pub fn new_full< spawn_handle: task_manager.spawn_handle(), import_queue, block_announce_validator_builder: None, - syncing_strategy, + warp_sync_config: Some(WarpSyncConfig::WithProvider(warp_sync)), block_relay: None, metrics, })?; diff --git a/polkadot/node/service/src/workers.rs b/polkadot/node/service/src/workers.rs index b35bb8302fdc4..73c3aa466608f 100644 --- a/polkadot/node/service/src/workers.rs +++ b/polkadot/node/service/src/workers.rs @@ -21,19 +21,20 @@ use is_executable::IsExecutable; use std::path::PathBuf; #[cfg(test)] -use std::sync::{Mutex, OnceLock}; +thread_local! { + static TMP_DIR: std::cell::RefCell> = std::cell::RefCell::new(None); +} /// Override the workers polkadot binary directory path, used for testing. #[cfg(test)] -fn workers_exe_path_override() -> &'static Mutex> { - static OVERRIDE: OnceLock>> = OnceLock::new(); - OVERRIDE.get_or_init(|| Mutex::new(None)) +fn workers_exe_path_override() -> Option { + TMP_DIR.with_borrow(|t| t.as_ref().map(|t| t.path().join("usr/bin"))) } + /// Override the workers lib directory path, used for testing. #[cfg(test)] -fn workers_lib_path_override() -> &'static Mutex> { - static OVERRIDE: OnceLock>> = OnceLock::new(); - OVERRIDE.get_or_init(|| Mutex::new(None)) +fn workers_lib_path_override() -> Option { + TMP_DIR.with_borrow(|t| t.as_ref().map(|t| t.path().join("usr/lib/polkadot"))) } /// Determines the final set of paths to use for the PVF workers. @@ -147,12 +148,9 @@ fn list_workers_paths( // Consider the /usr/lib/polkadot/ directory. { - #[allow(unused_mut)] - let mut lib_path = PathBuf::from("/usr/lib/polkadot"); + let lib_path = PathBuf::from("/usr/lib/polkadot"); #[cfg(test)] - if let Some(ref path_override) = *workers_lib_path_override().lock().unwrap() { - lib_path = path_override.clone(); - } + let lib_path = if let Some(o) = workers_lib_path_override() { o } else { lib_path }; let (prep_worker, exec_worker) = build_worker_paths(lib_path, workers_names); @@ -175,9 +173,10 @@ fn get_exe_path() -> Result { let mut exe_path = std::env::current_exe()?; let _ = exe_path.pop(); // executable file will always have a parent directory. #[cfg(test)] - if let Some(ref path_override) = *workers_exe_path_override().lock().unwrap() { - exe_path = path_override.clone(); + if let Some(o) = workers_exe_path_override() { + exe_path = o; } + Ok(exe_path) } @@ -205,8 +204,7 @@ mod tests { use super::*; use assert_matches::assert_matches; - use serial_test::serial; - use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path}; + use std::{fs, os::unix::fs::PermissionsExt, path::Path}; const TEST_NODE_VERSION: &'static str = "v0.1.2"; @@ -228,7 +226,7 @@ mod tests { fn get_program(version: &str) -> String { format!( - "#!/bin/bash + "#!/usr/bin/env bash if [[ $# -ne 1 ]] ; then echo \"unexpected number of arguments: $#\" @@ -253,27 +251,21 @@ echo {} ) -> Result<(), Box> { // Set up /usr/lib/polkadot and /usr/bin, both empty. - let tempdir = temp_dir(); - let lib_path = tempdir.join("usr/lib/polkadot"); - let _ = fs::remove_dir_all(&lib_path); - fs::create_dir_all(&lib_path)?; - *workers_lib_path_override().lock()? = Some(lib_path); + let tempdir = tempfile::tempdir().unwrap(); + let tmp_dir = tempdir.path().to_path_buf(); + TMP_DIR.with_borrow_mut(|t| *t = Some(tempdir)); - let exe_path = tempdir.join("usr/bin"); - let _ = fs::remove_dir_all(&exe_path); - fs::create_dir_all(&exe_path)?; - *workers_exe_path_override().lock()? = Some(exe_path.clone()); + fs::create_dir_all(workers_lib_path_override().unwrap()).unwrap(); + fs::create_dir_all(workers_exe_path_override().unwrap()).unwrap(); + let custom_path = tmp_dir.join("usr/local/bin"); // Set up custom path at /usr/local/bin. - let custom_path = tempdir.join("usr/local/bin"); - let _ = fs::remove_dir_all(&custom_path); - fs::create_dir_all(&custom_path)?; + fs::create_dir_all(&custom_path).unwrap(); - f(tempdir, exe_path) + f(tmp_dir, workers_exe_path_override().unwrap()) } #[test] - #[serial] fn test_given_worker_path() { with_temp_dir_structure(|tempdir, exe_path| { let given_workers_path = tempdir.join("usr/local/bin"); @@ -318,7 +310,6 @@ echo {} } #[test] - #[serial] fn missing_workers_paths_throws_error() { with_temp_dir_structure(|tempdir, exe_path| { // Try with both binaries missing. @@ -368,7 +359,6 @@ echo {} } #[test] - #[serial] fn should_find_workers_at_all_locations() { with_temp_dir_structure(|tempdir, _| { let prepare_worker_bin_path = tempdir.join("usr/bin/polkadot-prepare-worker"); @@ -394,7 +384,6 @@ echo {} } #[test] - #[serial] fn should_find_workers_with_custom_names_at_all_locations() { with_temp_dir_structure(|tempdir, _| { let (prep_worker_name, exec_worker_name) = ("test-prepare", "test-execute"); @@ -422,7 +411,6 @@ echo {} } #[test] - #[serial] fn workers_version_mismatch_throws_error() { let bad_version = "v9.9.9.9"; @@ -474,7 +462,6 @@ echo {} } #[test] - #[serial] fn should_find_valid_workers() { // Test bin location. with_temp_dir_structure(|tempdir, _| { diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index b8bad8f8a2953..b5686ec96be12 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -32,4 +32,3 @@ prometheus-endpoint = { workspace = true, default-features = true } thiserror = { workspace = true } async-trait = { workspace = true } bitvec = { features = ["alloc"], workspace = true } -strum = { features = ["derive"], workspace = true, default-features = true } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index ba1ba5755be0f..28a3a1ab82ab2 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -24,7 +24,6 @@ use futures::channel::oneshot; use sc_network::{Multiaddr, ReputationChange}; -use strum::EnumIter; use thiserror::Error; pub use sc_network::IfDisconnected; @@ -189,18 +188,16 @@ pub enum CandidateValidationMessage { /// Extends primitives::PvfExecKind, which is a runtime parameter we don't want to change, /// to separate and prioritize execution jobs by request type. -/// The order is important, because we iterate through the values and assume it is going from higher -/// to lowest priority. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PvfExecKind { /// For dispute requests Dispute, /// For approval requests Approval, - /// For backing requests from system parachains. - BackingSystemParas, - /// For backing requests. - Backing, + /// For backing requests from system parachains. With relay parent hash + BackingSystemParas(Hash), + /// For backing requests. With relay parent hash + Backing(Hash), } impl PvfExecKind { @@ -209,8 +206,8 @@ impl PvfExecKind { match *self { Self::Dispute => "dispute", Self::Approval => "approval", - Self::BackingSystemParas => "backing_system_paras", - Self::Backing => "backing", + Self::BackingSystemParas(_) => "backing_system_paras", + Self::Backing(_) => "backing", } } } @@ -220,8 +217,8 @@ impl From for RuntimePvfExecKind { match exec { PvfExecKind::Dispute => RuntimePvfExecKind::Approval, PvfExecKind::Approval => RuntimePvfExecKind::Approval, - PvfExecKind::BackingSystemParas => RuntimePvfExecKind::Backing, - PvfExecKind::Backing => RuntimePvfExecKind::Backing, + PvfExecKind::BackingSystemParas(_) => RuntimePvfExecKind::Backing, + PvfExecKind::Backing(_) => RuntimePvfExecKind::Backing, } } } diff --git a/polkadot/runtime/parachains/src/assigner_coretime/mod.rs b/polkadot/runtime/parachains/src/assigner_coretime/mod.rs index 33a36a1bb2ea6..866d52dc98484 100644 --- a/polkadot/runtime/parachains/src/assigner_coretime/mod.rs +++ b/polkadot/runtime/parachains/src/assigner_coretime/mod.rs @@ -236,17 +236,9 @@ pub mod pallet { #[pallet::error] pub enum Error { AssignmentsEmpty, - /// Assignments together exceeded 57600. - OverScheduled, - /// Assignments together less than 57600 - UnderScheduled, /// assign_core is only allowed to append new assignments at the end of already existing - /// ones. + /// ones or update the last entry. DisallowedInsert, - /// Tried to insert a schedule for the same core and block number as an existing schedule - DuplicateInsert, - /// Tried to add an unsorted set of assignments - AssignmentsNotSorted, } } @@ -387,67 +379,56 @@ impl Pallet { /// Append another assignment for a core. /// - /// Important only appending is allowed. Meaning, all already existing assignments must have a - /// begin smaller than the one passed here. This restriction exists, because it makes the - /// insertion O(1) and the author could not think of a reason, why this restriction should be - /// causing any problems. Inserting arbitrarily causes a `DispatchError::DisallowedInsert` - /// error. This restriction could easily be lifted if need be and in fact an implementation is - /// available - /// [here](https://github.com/paritytech/polkadot-sdk/pull/1694/commits/c0c23b01fd2830910cde92c11960dad12cdff398#diff-0c85a46e448de79a5452395829986ee8747e17a857c27ab624304987d2dde8baR386). - /// The problem is that insertion complexity then depends on the size of the existing queue, - /// which makes determining weights hard and could lead to issues like overweight blocks (at - /// least in theory). + /// Important: Only appending is allowed or insertion into the last item. Meaning, + /// all already existing assignments must have a `begin` smaller or equal than the one passed + /// here. + /// Updating the last entry is supported to allow for making a core assignment multiple calls to + /// assign_core. Thus if you have too much interlacing for e.g. a single UMP message you can + /// split that up into multiple messages, each triggering a call to `assign_core`, together + /// forming the total assignment. + /// + /// Inserting arbitrarily causes a `DispatchError::DisallowedInsert` error. + // With this restriction this function allows for O(1) complexity. It could easily be lifted, if + // need be and in fact an implementation is available + // [here](https://github.com/paritytech/polkadot-sdk/pull/1694/commits/c0c23b01fd2830910cde92c11960dad12cdff398#diff-0c85a46e448de79a5452395829986ee8747e17a857c27ab624304987d2dde8baR386). + // The problem is that insertion complexity then depends on the size of the existing queue, + // which makes determining weights hard and could lead to issues like overweight blocks (at + // least in theory). pub fn assign_core( core_idx: CoreIndex, begin: BlockNumberFor, - assignments: Vec<(CoreAssignment, PartsOf57600)>, + mut assignments: Vec<(CoreAssignment, PartsOf57600)>, end_hint: Option>, ) -> Result<(), DispatchError> { // There should be at least one assignment. ensure!(!assignments.is_empty(), Error::::AssignmentsEmpty); - // Checking for sort and unique manually, since we don't have access to iterator tools. - // This way of checking uniqueness only works since we also check sortedness. - assignments.iter().map(|x| &x.0).try_fold(None, |prev, cur| { - if prev.map_or(false, |p| p >= cur) { - Err(Error::::AssignmentsNotSorted) - } else { - Ok(Some(cur)) - } - })?; - - // Check that the total parts between all assignments are equal to 57600 - let parts_sum = assignments - .iter() - .map(|assignment| assignment.1) - .try_fold(PartsOf57600::ZERO, |sum, parts| { - sum.checked_add(parts).ok_or(Error::::OverScheduled) - })?; - ensure!(parts_sum.is_full(), Error::::UnderScheduled); - CoreDescriptors::::mutate(core_idx, |core_descriptor| { let new_queue = match core_descriptor.queue { Some(queue) => { - ensure!(begin > queue.last, Error::::DisallowedInsert); - - CoreSchedules::::try_mutate((queue.last, core_idx), |schedule| { - if let Some(schedule) = schedule.as_mut() { - debug_assert!(schedule.next_schedule.is_none(), "queue.end was supposed to be the end, so the next item must be `None`!"); - schedule.next_schedule = Some(begin); + ensure!(begin >= queue.last, Error::::DisallowedInsert); + + // Update queue if we are appending: + if begin > queue.last { + CoreSchedules::::mutate((queue.last, core_idx), |schedule| { + if let Some(schedule) = schedule.as_mut() { + debug_assert!(schedule.next_schedule.is_none(), "queue.end was supposed to be the end, so the next item must be `None`!"); + schedule.next_schedule = Some(begin); + } else { + defensive!("Queue end entry does not exist?"); + } + }); + } + + CoreSchedules::::mutate((begin, core_idx), |schedule| { + let assignments = if let Some(mut old_schedule) = schedule.take() { + old_schedule.assignments.append(&mut assignments); + old_schedule.assignments } else { - defensive!("Queue end entry does not exist?"); - } - CoreSchedules::::try_mutate((begin, core_idx), |schedule| { - // It should already be impossible to overwrite an existing schedule due - // to strictly increasing block number. But we check here for safety and - // in case the design changes. - ensure!(schedule.is_none(), Error::::DuplicateInsert); - *schedule = - Some(Schedule { assignments, end_hint, next_schedule: None }); - Ok::<(), DispatchError>(()) - })?; - Ok::<(), DispatchError>(()) - })?; + assignments + }; + *schedule = Some(Schedule { assignments, end_hint, next_schedule: None }); + }); QueueDescriptor { first: queue.first, last: begin } }, diff --git a/polkadot/runtime/parachains/src/assigner_coretime/tests.rs b/polkadot/runtime/parachains/src/assigner_coretime/tests.rs index 25007f0eed6aa..ab011bfc4ae12 100644 --- a/polkadot/runtime/parachains/src/assigner_coretime/tests.rs +++ b/polkadot/runtime/parachains/src/assigner_coretime/tests.rs @@ -235,10 +235,7 @@ fn assign_core_works_with_prior_schedule() { } #[test] -// Invariants: We assume that CoreSchedules is append only and consumed. In other words new -// schedules inserted for a core must have a higher block number than all of the already existing -// schedules. -fn assign_core_enforces_higher_block_number() { +fn assign_core_enforces_higher_or_equal_block_number() { let core_idx = CoreIndex(0); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { @@ -255,7 +252,7 @@ fn assign_core_enforces_higher_block_number() { assert_ok!(CoretimeAssigner::assign_core( core_idx, BlockNumberFor::::from(15u32), - default_test_assignments(), + vec![(CoreAssignment::Idle, PartsOf57600(28800))], None, )); @@ -281,32 +278,27 @@ fn assign_core_enforces_higher_block_number() { ), Error::::DisallowedInsert ); + // Call assign core again on last entry should work: + assert_eq!( + CoretimeAssigner::assign_core( + core_idx, + BlockNumberFor::::from(15u32), + vec![(CoreAssignment::Pool, PartsOf57600(28800))], + None, + ), + Ok(()) + ); }); } #[test] fn assign_core_enforces_well_formed_schedule() { - let para_id = ParaId::from(1u32); let core_idx = CoreIndex(0); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { run_to_block(1, |n| if n == 1 { Some(Default::default()) } else { None }); let empty_assignments: Vec<(CoreAssignment, PartsOf57600)> = vec![]; - let overscheduled = vec![ - (CoreAssignment::Pool, PartsOf57600::FULL), - (CoreAssignment::Task(para_id.into()), PartsOf57600::FULL), - ]; - let underscheduled = vec![(CoreAssignment::Pool, PartsOf57600(30000))]; - let not_unique = vec![ - (CoreAssignment::Pool, PartsOf57600::FULL / 2), - (CoreAssignment::Pool, PartsOf57600::FULL / 2), - ]; - let not_sorted = vec![ - (CoreAssignment::Task(para_id.into()), PartsOf57600(19200)), - (CoreAssignment::Pool, PartsOf57600(19200)), - (CoreAssignment::Idle, PartsOf57600(19200)), - ]; // Attempting assign_core with malformed assignments such that all error cases // are tested @@ -319,42 +311,6 @@ fn assign_core_enforces_well_formed_schedule() { ), Error::::AssignmentsEmpty ); - assert_noop!( - CoretimeAssigner::assign_core( - core_idx, - BlockNumberFor::::from(11u32), - overscheduled, - None, - ), - Error::::OverScheduled - ); - assert_noop!( - CoretimeAssigner::assign_core( - core_idx, - BlockNumberFor::::from(11u32), - underscheduled, - None, - ), - Error::::UnderScheduled - ); - assert_noop!( - CoretimeAssigner::assign_core( - core_idx, - BlockNumberFor::::from(11u32), - not_unique, - None, - ), - Error::::AssignmentsNotSorted - ); - assert_noop!( - CoretimeAssigner::assign_core( - core_idx, - BlockNumberFor::::from(11u32), - not_sorted, - None, - ), - Error::::AssignmentsNotSorted - ); }); } @@ -374,7 +330,14 @@ fn next_schedule_always_points_to_next_work_plan_item() { Schedule { next_schedule: Some(start_4), ..default_test_schedule() }; let expected_schedule_4 = Schedule { next_schedule: Some(start_5), ..default_test_schedule() }; - let expected_schedule_5 = default_test_schedule(); + let expected_schedule_5 = Schedule { + next_schedule: None, + end_hint: None, + assignments: vec![ + (CoreAssignment::Pool, PartsOf57600(28800)), + (CoreAssignment::Idle, PartsOf57600(28800)), + ], + }; // Call assign_core for each of five schedules assert_ok!(CoretimeAssigner::assign_core( @@ -408,7 +371,14 @@ fn next_schedule_always_points_to_next_work_plan_item() { assert_ok!(CoretimeAssigner::assign_core( core_idx, BlockNumberFor::::from(start_5), - default_test_assignments(), + vec![(CoreAssignment::Pool, PartsOf57600(28800))], + None, + )); + // Test updating last entry once more: + assert_ok!(CoretimeAssigner::assign_core( + core_idx, + BlockNumberFor::::from(start_5), + vec![(CoreAssignment::Idle, PartsOf57600(28800))], None, )); diff --git a/polkadot/runtime/rococo/src/genesis_config_presets.rs b/polkadot/runtime/rococo/src/genesis_config_presets.rs index 39c862660894c..bdbf6f37d92c2 100644 --- a/polkadot/runtime/rococo/src/genesis_config_presets.rs +++ b/polkadot/runtime/rococo/src/genesis_config_presets.rs @@ -23,6 +23,7 @@ use crate::{ #[cfg(not(feature = "std"))] use alloc::format; use alloc::{vec, vec::Vec}; +use frame_support::build_struct_json_patch; use polkadot_primitives::{AccountId, AssignmentId, SchedulerParams, ValidatorId}; use rococo_runtime_constants::currency::UNITS as ROC; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; @@ -163,7 +164,7 @@ fn rococo_testnet_genesis( const ENDOWMENT: u128 = 1_000_000 * ROC; - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts.iter().map(|k| (k.clone(), ENDOWMENT)).collect::>(), }, @@ -185,9 +186,8 @@ fn rococo_testnet_genesis( ) }) .collect::>(), - ..Default::default() }, - babe: BabeConfig { epoch_config: BABE_GENESIS_EPOCH_CONFIG, ..Default::default() }, + babe: BabeConfig { epoch_config: BABE_GENESIS_EPOCH_CONFIG }, sudo: SudoConfig { key: Some(root_key.clone()) }, configuration: ConfigurationConfig { config: polkadot_runtime_parachains::configuration::HostConfiguration { @@ -198,14 +198,8 @@ fn rococo_testnet_genesis( ..default_parachains_host_configuration() }, }, - registrar: RegistrarConfig { - next_free_para_id: polkadot_primitives::LOWEST_PUBLIC_ID, - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + registrar: RegistrarConfig { next_free_para_id: polkadot_primitives::LOWEST_PUBLIC_ID }, + }) } // staging_testnet @@ -427,7 +421,7 @@ fn rococo_staging_testnet_config_genesis() -> serde_json::Value { const ENDOWMENT: u128 = 1_000_000 * ROC; const STASH: u128 = 100 * ROC; - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -440,19 +434,12 @@ fn rococo_staging_testnet_config_genesis() -> serde_json::Value { .into_iter() .map(|x| (x.0.clone(), x.0, rococo_session_keys(x.2, x.3, x.4, x.5, x.6, x.7))) .collect::>(), - ..Default::default() }, - babe: BabeConfig { epoch_config: BABE_GENESIS_EPOCH_CONFIG, ..Default::default() }, + babe: BabeConfig { epoch_config: BABE_GENESIS_EPOCH_CONFIG }, sudo: SudoConfig { key: Some(endowed_accounts[0].clone()) }, configuration: ConfigurationConfig { config: default_parachains_host_configuration() }, - registrar: RegistrarConfig { - next_free_para_id: polkadot_primitives::LOWEST_PUBLIC_ID, - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + registrar: RegistrarConfig { next_free_para_id: polkadot_primitives::LOWEST_PUBLIC_ID }, + }) } //development diff --git a/polkadot/runtime/westend/src/genesis_config_presets.rs b/polkadot/runtime/westend/src/genesis_config_presets.rs index b074d54fb5827..b8f7710089e04 100644 --- a/polkadot/runtime/westend/src/genesis_config_presets.rs +++ b/polkadot/runtime/westend/src/genesis_config_presets.rs @@ -222,7 +222,7 @@ fn westend_staging_testnet_config_genesis() -> serde_json::Value { // // SECRET_SEED="slow awkward present example safe bundle science ocean cradle word tennis earn" // subkey inspect -n polkadot "$SECRET_SEED" - let endowed_accounts = vec![ + let endowed_accounts: Vec = vec![ // 15S75FkhCWEowEGfxWwVfrW3LQuy8w8PNhVmrzfsVhCMjUh1 hex!["c416837e232d9603e83162ef4bda08e61580eeefe60fe92fc044aa508559ae42"].into(), ]; @@ -338,7 +338,7 @@ fn westend_staging_testnet_config_genesis() -> serde_json::Value { const ENDOWMENT: u128 = 1_000_000 * WND; const STASH: u128 = 100 * WND; - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -364,7 +364,6 @@ fn westend_staging_testnet_config_genesis() -> serde_json::Value { ) }) .collect::>(), - ..Default::default() }, staking: StakingConfig { validator_count: 50, @@ -376,19 +375,12 @@ fn westend_staging_testnet_config_genesis() -> serde_json::Value { invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect::>(), force_era: Forcing::ForceNone, slash_reward_fraction: Perbill::from_percent(10), - ..Default::default() }, - babe: BabeConfig { epoch_config: BABE_GENESIS_EPOCH_CONFIG, ..Default::default() }, + babe: BabeConfig { epoch_config: BABE_GENESIS_EPOCH_CONFIG }, sudo: SudoConfig { key: Some(endowed_accounts[0].clone()) }, configuration: ConfigurationConfig { config: default_parachains_host_configuration() }, - registrar: RegistrarConfig { - next_free_para_id: polkadot_primitives::LOWEST_PUBLIC_ID, - ..Default::default() - }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + registrar: RegistrarConfig { next_free_para_id: polkadot_primitives::LOWEST_PUBLIC_ID }, + }) } //development diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index a9a76dbced101..4c04af111f811 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -46,6 +46,7 @@ use frame_support::{ use frame_system::{EnsureRoot, EnsureSigned}; use pallet_grandpa::{fg_primitives, AuthorityId as GrandpaId}; use pallet_identity::legacy::IdentityInfo; +use pallet_nomination_pools::PoolId; use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, FungibleAdapter, RuntimeDispatchInfo}; use polkadot_primitives::{ @@ -2492,15 +2493,15 @@ sp_api::impl_runtime_apis! { NominationPools::api_pending_rewards(member).unwrap_or_default() } - fn points_to_balance(pool_id: pallet_nomination_pools::PoolId, points: Balance) -> Balance { + fn points_to_balance(pool_id: PoolId, points: Balance) -> Balance { NominationPools::api_points_to_balance(pool_id, points) } - fn balance_to_points(pool_id: pallet_nomination_pools::PoolId, new_funds: Balance) -> Balance { + fn balance_to_points(pool_id: PoolId, new_funds: Balance) -> Balance { NominationPools::api_balance_to_points(pool_id, new_funds) } - fn pool_pending_slash(pool_id: pallet_nomination_pools::PoolId) -> Balance { + fn pool_pending_slash(pool_id: PoolId) -> Balance { NominationPools::api_pool_pending_slash(pool_id) } @@ -2508,7 +2509,7 @@ sp_api::impl_runtime_apis! { NominationPools::api_member_pending_slash(member) } - fn pool_needs_delegate_migration(pool_id: pallet_nomination_pools::PoolId) -> bool { + fn pool_needs_delegate_migration(pool_id: PoolId) -> bool { NominationPools::api_pool_needs_delegate_migration(pool_id) } @@ -2520,9 +2521,13 @@ sp_api::impl_runtime_apis! { NominationPools::api_member_total_balance(member) } - fn pool_balance(pool_id: pallet_nomination_pools::PoolId) -> Balance { + fn pool_balance(pool_id: PoolId) -> Balance { NominationPools::api_pool_balance(pool_id) } + + fn pool_accounts(pool_id: PoolId) -> (AccountId, AccountId) { + NominationPools::api_pool_accounts(pool_id) + } } impl pallet_staking_runtime_api::StakingApi for Runtime { diff --git a/polkadot/xcm/pallet-xcm-benchmarks/src/generic/benchmarking.rs b/polkadot/xcm/pallet-xcm-benchmarks/src/generic/benchmarking.rs index 6bb7fc962e346..0d80ef89a1ce1 100644 --- a/polkadot/xcm/pallet-xcm-benchmarks/src/generic/benchmarking.rs +++ b/polkadot/xcm/pallet-xcm-benchmarks/src/generic/benchmarking.rs @@ -103,8 +103,10 @@ benchmarks! { let mut executor = new_executor::(Default::default()); executor.set_holding(holding); + // Set some weight to be paid for. + executor.set_message_weight(Weight::from_parts(100_000_000, 100_000)); - let fee_asset = T::fee_asset().unwrap(); + let fee_asset: Asset = T::fee_asset().unwrap(); let instruction = Instruction::>::PayFees { asset: fee_asset }; diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr index 978faf2e868d8..e4038dc25ae64 100644 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr @@ -1,4 +1,4 @@ -error: Expected `builder(loads_holding)` +error: Expected `builder(loads_holding)` or `builder(pays_fees)` --> tests/ui/builder_pattern/badly_formatted_attribute.rs:25:5 | 25 | #[builder(funds_holding = 2)] diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs deleted file mode 100644 index dc5c679a96e72..0000000000000 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Test error when the `BuyExecution` instruction doesn't take named fields. - -use xcm_procedural::Builder; - -struct Xcm(pub Vec>); - -#[derive(Builder)] -enum Instruction { - BuyExecution(u128), - UnpaidExecution { weight_limit: (u32, u32) }, - Transact { call: Call }, -} - -fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr deleted file mode 100644 index dc8246770ba3e..0000000000000 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: BuyExecution should have named fields - --> tests/ui/builder_pattern/buy_execution_named_fields.rs:25:5 - | -25 | BuyExecution(u128), - | ^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs deleted file mode 100644 index 1ed8dd38cbad5..0000000000000 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Test error when there's no `BuyExecution` instruction. - -use xcm_procedural::Builder; - -struct Xcm(pub Vec>); - -#[derive(Builder)] -enum Instruction { - UnpaidExecution { weight_limit: (u32, u32) }, - Transact { call: Call }, -} - -fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr deleted file mode 100644 index d8798c8223f18..0000000000000 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: No BuyExecution instruction - --> tests/ui/builder_pattern/no_buy_execution.rs:25:5 - | -25 | / UnpaidExecution { weight_limit: (u32, u32) }, -26 | | Transact { call: Call }, - | |____________________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs deleted file mode 100644 index 5808ec571ce75..0000000000000 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Test error when using wrong attribute. - -use xcm_procedural::Builder; - -struct Xcm(pub Vec>); - -#[derive(Builder)] -enum Instruction { - #[builder(funds_holding)] - WithdrawAsset(u128), - BuyExecution { fees: u128 }, - UnpaidExecution { weight_limit: (u32, u32) }, - Transact { call: Call }, -} - -fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr deleted file mode 100644 index 1ff9d18513686..0000000000000 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Expected `builder(loads_holding)` - --> tests/ui/builder_pattern/unexpected_attribute.rs:25:5 - | -25 | #[builder(funds_holding)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/xcm-builder/src/weight.rs b/polkadot/xcm/xcm-builder/src/weight.rs index cbb92afac54bd..f8c0275d0f545 100644 --- a/polkadot/xcm/xcm-builder/src/weight.rs +++ b/polkadot/xcm/xcm-builder/src/weight.rs @@ -231,7 +231,7 @@ impl< log::trace!(target: "xcm::weight", "UsingComponents::buy_weight weight: {:?}, payment: {:?}, context: {:?}", weight, payment, context); let amount = WeightToFee::weight_to_fee(&weight); let u128_amount: u128 = amount.try_into().map_err(|_| XcmError::Overflow)?; - let required = (AssetId(AssetIdValue::get()), u128_amount).into(); + let required = Asset { id: AssetId(AssetIdValue::get()), fun: Fungible(u128_amount) }; let unused = payment.checked_sub(required).map_err(|_| XcmError::TooExpensive)?; self.0 = self.0.saturating_add(weight); self.1 = self.1.saturating_add(amount); diff --git a/polkadot/xcm/xcm-executor/src/lib.rs b/polkadot/xcm/xcm-executor/src/lib.rs index 8e55912979d4c..a823dc6fec789 100644 --- a/polkadot/xcm/xcm-executor/src/lib.rs +++ b/polkadot/xcm/xcm-executor/src/lib.rs @@ -197,6 +197,9 @@ impl XcmExecutor { pub fn asset_claimer(&self) -> Option { self.asset_claimer.clone() } + pub fn set_message_weight(&mut self, weight: Weight) { + self.message_weight = weight; + } } pub struct WeighedMessage(Weight, Xcm); @@ -1298,9 +1301,22 @@ impl XcmExecutor { result }, PayFees { asset } => { + // Message was not weighed, there is nothing to pay. + if self.message_weight == Weight::zero() { + tracing::warn!( + target: "xcm::executor::PayFees", + "Message was not weighed or weight was 0. Nothing will be charged.", + ); + return Ok(()); + } // Record old holding in case we need to rollback. let old_holding = self.holding.clone(); // The max we're willing to pay for fees is decided by the `asset` operand. + tracing::trace!( + target: "xcm::executor::PayFees", + asset_for_fees = ?asset, + message_weight = ?self.message_weight, + ); let max_fee = self.holding.try_take(asset.into()).map_err(|_| XcmError::NotHoldingFees)?; // Pay for execution fees. diff --git a/polkadot/zombienet_tests/functional/0018-shared-core-idle-parachain.zndsl b/polkadot/zombienet_tests/functional/0018-shared-core-idle-parachain.zndsl index 80ecf6ae1b9be..dce52505444e9 100644 --- a/polkadot/zombienet_tests/functional/0018-shared-core-idle-parachain.zndsl +++ b/polkadot/zombienet_tests/functional/0018-shared-core-idle-parachain.zndsl @@ -8,4 +8,4 @@ validator-0: js-script ./force-register-paras.js with "2000" return is 0 within # assign core 0 to be shared by two paras, but only one exists validator-0: js-script ./assign-core.js with "0,2000,28800,2001,28800" return is 0 within 600 seconds -collator-2000: reports block height is at least 10 within 180 seconds +collator-2000: reports block height is at least 10 within 210 seconds diff --git a/prdoc/pr_4012.prdoc b/prdoc/pr_4012.prdoc new file mode 100644 index 0000000000000..3a53e31a7fc6e --- /dev/null +++ b/prdoc/pr_4012.prdoc @@ -0,0 +1,37 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "`impl_runtime_apis!`: replace the use of `Self` with `Runtime`" + +doc: + - audience: Runtime Dev + description: | + Currently, if there is a type alias similar to `type HeaderFor` in the scope, it makes sense to expect that + `HeaderFor` and `HeaderFor` are equivalent. However, this is not the case. It currently leads to + a compilation error that `Self is not in scope`, which is confusing. This PR introduces a visitor, similar to + `CheckTraitDecl` in `decl_runtime_apis!`, `ReplaceSelfImpl`. It identifies usage of `Self` as a type argument in + `impl_runtime_apis!` and replaces `Self` with an explicit `Runtime` type. + + For example, the following example code will be transformed before expansion: + ```rust + impl apis::Core for Runtime { + fn initialize_block(header: &HeaderFor) -> ExtrinsicInclusionMode { + let _: HeaderFor = header.clone(); + RuntimeExecutive::initialize_block(header) + } + } + ``` + Instead, it will be passed to macro as: + ```rust + impl apis::Core for Runtime { + fn initialize_block(header: &HeaderFor) -> ExtrinsicInclusionMode { + let _: HeaderFor = header.clone(); + RuntimeExecutive::initialize_block(header) + } + } + ``` +crates: + - name: sp-api + bump: none + - name: sp-api-proc-macro + bump: none \ No newline at end of file diff --git a/prdoc/pr_5616.prdoc b/prdoc/pr_5616.prdoc new file mode 100644 index 0000000000000..16d81c291c30f --- /dev/null +++ b/prdoc/pr_5616.prdoc @@ -0,0 +1,25 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "PVF: drop backing jobs if it is too late" + +doc: + - audience: [ Node Dev, Node Operator ] + description: | + Introduces the removal of backing jobs that have been back pressured for longer than `allowedAncestryLen`, as these candidates are no longer viable. + +crates: + - name: polkadot-overseer + bump: major + - name: polkadot-node-core-pvf + bump: major + - name: polkadot-node-subsystem-types + bump: major + - name: polkadot-node-core-approval-voting + bump: patch + - name: polkadot-node-core-backing + bump: patch + - name: polkadot-node-core-candidate-validation + bump: patch + - name: polkadot-node-core-dispute-coordinator + bump: patch diff --git a/prdoc/pr_5737.prdoc b/prdoc/pr_5737.prdoc new file mode 100644 index 0000000000000..a122e4574a9ce --- /dev/null +++ b/prdoc/pr_5737.prdoc @@ -0,0 +1,25 @@ +title: Make syncing service an argument of `build_network` + +doc: + - audience: Node Dev + description: | + `build_network` is accompanied with lower-level `build_network_advanced` with simpler API that does not create + syncing engine internally, but instead takes a handle to syncing service as an argument. In most cases typical + syncing engine with polkadot syncing strategy and default block downloader can be created with newly introduced + `sc_service::build_default_syncing_engine()` function, but lower-level `build_default_block_downloader` also + exists for those needing more customization. + + These changes allow developers higher than ever control over syncing implementation, but `build_network` is still + available for easier high-level usage. + +crates: + - name: cumulus-client-service + bump: patch + - name: polkadot-service + bump: patch + - name: sc-consensus + bump: major + - name: sc-service + bump: major + - name: sc-network-sync + bump: major diff --git a/prdoc/pr_5984.prdoc b/prdoc/pr_5984.prdoc new file mode 100644 index 0000000000000..3b6651bac6b93 --- /dev/null +++ b/prdoc/pr_5984.prdoc @@ -0,0 +1,14 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Add page information to staking::PayoutStarted event + +doc: + - audience: Runtime User + description: | + Adds page index that is claimed, and optional next page that can be claimed. If next is none, then the page is the + last one. + +crates: + - name: pallet-staking + bump: major diff --git a/prdoc/pr_6011.prdoc b/prdoc/pr_6011.prdoc new file mode 100644 index 0000000000000..e053b607085f7 --- /dev/null +++ b/prdoc/pr_6011.prdoc @@ -0,0 +1,12 @@ +title: "collator protocol: validate descriptor version on the validator side" + +doc: + - audience: Node Dev + description: | + Implement checks needed for RFC 103 https://github.com/polkadot-fellows/rfcs/pull/103 in the validator + side of the collator protocol. + + +crates: + - name: polkadot-collator-protocol + bump: major diff --git a/prdoc/pr_6077.prdoc b/prdoc/pr_6077.prdoc new file mode 100644 index 0000000000000..f222fb27ce07b --- /dev/null +++ b/prdoc/pr_6077.prdoc @@ -0,0 +1,9 @@ +title: Add networking benchmarks for libp2p +doc: +- audience: node_dev + description: |- + Adds benchmarks for Notifications and RequestResponse protocols with libp2p implementation + +crates: +- name: sc-network + validate: false diff --git a/prdoc/pr_6163.prdoc b/prdoc/pr_6163.prdoc new file mode 100644 index 0000000000000..c8571f80ed52f --- /dev/null +++ b/prdoc/pr_6163.prdoc @@ -0,0 +1,13 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Expose more syncing types to enable custom syncing strategy + +doc: + - audience: Node Dev + description: | + Exposes additional syncing types to facilitate the development of a custom syncing strategy. + +crates: + - name: sc-network-sync + bump: patch diff --git a/prdoc/pr_6261.prdoc b/prdoc/pr_6261.prdoc new file mode 100644 index 0000000000000..20ee5563bcfd5 --- /dev/null +++ b/prdoc/pr_6261.prdoc @@ -0,0 +1,13 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Add missing events to identity pallet + +doc: + - audience: Runtime Dev + description: | + Extrinsics from `pallet_identity` that were missing an event emission on success now emit one. + +crates: + - name: pallet-identity + bump: major diff --git a/prdoc/pr_6290.prdoc b/prdoc/pr_6290.prdoc new file mode 100644 index 0000000000000..a05d0cd15acf7 --- /dev/null +++ b/prdoc/pr_6290.prdoc @@ -0,0 +1,11 @@ +title: Migrate pallet-transaction-storage and pallet-indices to benchmark v2 +doc: +- audience: Runtime Dev + description: |- + Part of: + #6202 +crates: +- name: pallet-indices + bump: patch +- name: pallet-transaction-storage + bump: patch diff --git a/prdoc/pr_6314.prdoc b/prdoc/pr_6314.prdoc new file mode 100644 index 0000000000000..2ebbc68158d59 --- /dev/null +++ b/prdoc/pr_6314.prdoc @@ -0,0 +1,10 @@ +title: Migrate pallet-elections-phragmen benchmark to v2 and improve doc +doc: +- audience: Runtime Dev + description: |- + Part of: + + - #6202. +crates: +- name: pallet-elections-phragmen + bump: patch diff --git a/prdoc/pr_6315.prdoc b/prdoc/pr_6315.prdoc new file mode 100644 index 0000000000000..6fc070b43b353 --- /dev/null +++ b/prdoc/pr_6315.prdoc @@ -0,0 +1,8 @@ +title: Migrate pallet-election-provider-support-benchmarking benchmark to v2 +doc: +- audience: Runtime Dev + description: |- + Migrate pallet-election-provider-support-benchmarking benchmark to v2 +crates: +- name: pallet-election-provider-support-benchmarking + bump: patch diff --git a/prdoc/pr_6337.prdoc b/prdoc/pr_6337.prdoc new file mode 100644 index 0000000000000..aeab61cdf933e --- /dev/null +++ b/prdoc/pr_6337.prdoc @@ -0,0 +1,17 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Don't expose metadata for Runtime APIs that haven't been implemented + +doc: + - audience: Runtime User + description: | + Prior to this PR, the metadata for runtime APIs would contain all methods for the + latest version of each API, regardless of which version a runtime implements. This + PR fixes that, so that the runtime API metadata reflects what is actually implemented. + +crates: + - name: sp-api-proc-macro + bump: major + - name: sp-consensus-babe + bump: patch \ No newline at end of file diff --git a/prdoc/pr_6349.prdoc b/prdoc/pr_6349.prdoc new file mode 100644 index 0000000000000..40f02712c99ae --- /dev/null +++ b/prdoc/pr_6349.prdoc @@ -0,0 +1,44 @@ +title: "runtimes: presets are provided as config patches" + +doc: + - audience: Runtime Dev + description: | + This PR introduces usage of build_struct_json_patch macro in all + runtimes (also guides) within the code base. It also fixes macro to support + field init shorthand, and Struct Update syntax which were missing in original + implementation. + +crates: + - name: frame-support + bump: major + + - name: westend-runtime + bump: patch + + - name: rococo-runtime + bump: patch + + - name: asset-hub-westend-runtime + bump: patch + + - name: bridge-hub-rococo-runtime + bump: patch + + - name: bridge-hub-westend-runtime + bump: patch + + - name: collectives-westend-runtime + bump: patch + + - name: minimal-template-runtime + bump: patch + + - name: solochain-template-runtime + bump: patch + + - name: parachain-template-runtime + bump: patch + + - name: polkadot-sdk-docs-first-runtime + bump: patch + diff --git a/prdoc/pr_6357.prdoc b/prdoc/pr_6357.prdoc new file mode 100644 index 0000000000000..b3155b1a60505 --- /dev/null +++ b/prdoc/pr_6357.prdoc @@ -0,0 +1,20 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: New runtime api that returns the associated pool accounts with a nomination pool. + +doc: + - audience: Runtime User + description: | + Each nomination pool has two associated pot accounts: the bonded account, where funds are pooled for staking, and + the reward account. This update introduces a runtime api that clients can query to retrieve these accounts. + +crates: + - name: westend-runtime + bump: minor + - name: kitchensink-runtime + bump: minor + - name: pallet-nomination-pools + bump: minor + - name: pallet-nomination-pools-runtime-api + bump: minor diff --git a/prdoc/pr_6380.prdoc b/prdoc/pr_6380.prdoc new file mode 100644 index 0000000000000..72853bcf230c8 --- /dev/null +++ b/prdoc/pr_6380.prdoc @@ -0,0 +1,11 @@ +title: Do not propagate external addr with different peerIDs + +doc: + - audience: [ Node Dev, Node Operator ] + description: | + External addresses that belong to a different peerID are no longer + propagated to the higher layers of the networking backends. + +crates: + - name: sc-network + bump: patch diff --git a/prdoc/pr_6382.prdoc b/prdoc/pr_6382.prdoc new file mode 100644 index 0000000000000..ac6821c1100a1 --- /dev/null +++ b/prdoc/pr_6382.prdoc @@ -0,0 +1,12 @@ +title: 'gensis-config: patching default `RuntimeGenesisConfig` fixed' +doc: +- audience: Node Dev + description: |- + This PR fixes issue reported in #6306. + It changes the behavior of `sc_chain_spec::json_patch::merge` function which no longer removes any keys from the base JSON object. + +crates: +- name: staging-chain-spec-builder + bump: major +- name: sc-chain-spec + bump: major diff --git a/prdoc/pr_6384.prdoc b/prdoc/pr_6384.prdoc new file mode 100644 index 0000000000000..2ea0bc1043c33 --- /dev/null +++ b/prdoc/pr_6384.prdoc @@ -0,0 +1,12 @@ +title: Relax requirements on `assign_core`. +doc: +- audience: Runtime Dev + description: |- + Relax requirements for `assign_core` so that it accepts updates for the last scheduled entry. + This will allow the coretime chain to split up assignments into multiple + messages, which allows for interlacing down to single block granularity. + + Fixes: https://github.com/paritytech/polkadot-sdk/issues/6102 +crates: +- name: polkadot-runtime-parachains + bump: major diff --git a/prdoc/pr_6406.prdoc b/prdoc/pr_6406.prdoc new file mode 100644 index 0000000000000..9da4462263b9e --- /dev/null +++ b/prdoc/pr_6406.prdoc @@ -0,0 +1,9 @@ +title: 'make prospective-parachains debug logs less spammy' +doc: +- audience: [Node Dev, Node Operator] + description: | + Demote some of the frequent prospective-parachains debug logs to trace level and prefer printing aggregate debug logs. + +crates: +- name: polkadot-node-core-prospective-parachains + bump: patch diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index 057e0bbdcefc7..008cac4ef8a88 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -32,7 +32,6 @@ use frame_system_rpc_runtime_api::AccountNonceApi; use futures::prelude::*; use kitchensink_runtime::RuntimeApi; use node_primitives::Block; -use polkadot_sdk::sc_service::build_polkadot_syncing_strategy; use sc_client_api::{Backend, BlockBackend}; use sc_consensus_babe::{self, SlotProportion}; use sc_network::{ @@ -514,16 +513,6 @@ pub fn new_full_base::Hash>>( Vec::default(), )); - let syncing_strategy = build_polkadot_syncing_strategy( - config.protocol_id(), - config.chain_spec.fork_id(), - &mut net_config, - Some(WarpSyncConfig::WithProvider(warp_sync)), - client.clone(), - &task_manager.spawn_handle(), - config.prometheus_config.as_ref().map(|config| &config.registry), - )?; - let (network, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, @@ -533,7 +522,7 @@ pub fn new_full_base::Hash>>( spawn_handle: task_manager.spawn_handle(), import_queue, block_announce_validator_builder: None, - syncing_strategy, + warp_sync_config: Some(WarpSyncConfig::WithProvider(warp_sync)), block_relay: None, metrics, })?; diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 18db43b1c1201..5a2ff3ceb7f6a 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -83,6 +83,7 @@ use pallet_identity::legacy::IdentityInfo; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use pallet_nfts::PalletFeatures; use pallet_nis::WithMaximumOf; +use pallet_nomination_pools::PoolId; use pallet_revive::{evm::runtime::EthExtra, AddressMapper}; use pallet_session::historical as pallet_session_historical; // Can't use `FungibleAdapter` here until Treasury pallet migrates to fungibles @@ -3004,15 +3005,15 @@ impl_runtime_apis! { NominationPools::api_pending_rewards(who).unwrap_or_default() } - fn points_to_balance(pool_id: pallet_nomination_pools::PoolId, points: Balance) -> Balance { + fn points_to_balance(pool_id: PoolId, points: Balance) -> Balance { NominationPools::api_points_to_balance(pool_id, points) } - fn balance_to_points(pool_id: pallet_nomination_pools::PoolId, new_funds: Balance) -> Balance { + fn balance_to_points(pool_id: PoolId, new_funds: Balance) -> Balance { NominationPools::api_balance_to_points(pool_id, new_funds) } - fn pool_pending_slash(pool_id: pallet_nomination_pools::PoolId) -> Balance { + fn pool_pending_slash(pool_id: PoolId) -> Balance { NominationPools::api_pool_pending_slash(pool_id) } @@ -3020,7 +3021,7 @@ impl_runtime_apis! { NominationPools::api_member_pending_slash(member) } - fn pool_needs_delegate_migration(pool_id: pallet_nomination_pools::PoolId) -> bool { + fn pool_needs_delegate_migration(pool_id: PoolId) -> bool { NominationPools::api_pool_needs_delegate_migration(pool_id) } @@ -3032,9 +3033,13 @@ impl_runtime_apis! { NominationPools::api_member_total_balance(member) } - fn pool_balance(pool_id: pallet_nomination_pools::PoolId) -> Balance { + fn pool_balance(pool_id: PoolId) -> Balance { NominationPools::api_pool_balance(pool_id) } + + fn pool_accounts(pool_id: PoolId) -> (AccountId, AccountId) { + NominationPools::api_pool_accounts(pool_id) + } } impl pallet_staking_runtime_api::StakingApi for Runtime { diff --git a/substrate/bin/utils/chain-spec-builder/src/lib.rs b/substrate/bin/utils/chain-spec-builder/src/lib.rs index 6f3128ed7eb05..73c2868b3312e 100644 --- a/substrate/bin/utils/chain-spec-builder/src/lib.rs +++ b/substrate/bin/utils/chain-spec-builder/src/lib.rs @@ -261,19 +261,19 @@ impl ChainSpecBuilder { .map_err(|e| format!("Conversion to json failed: {e}"))?; // We want to extract only raw genesis ("genesis::raw" key), and apply it as a patch - // for the original json file. However, the file also contains original plain - // genesis ("genesis::runtimeGenesis") so set it to null so the patch will erase it. + // for the original json file. genesis_json.as_object_mut().map(|map| { map.retain(|key, _| key == "genesis"); - map.get_mut("genesis").map(|genesis| { - genesis.as_object_mut().map(|genesis_map| { - genesis_map - .insert("runtimeGenesis".to_string(), serde_json::Value::Null); - }); - }); }); let mut org_chain_spec_json = extract_chain_spec_json(input_chain_spec.as_path())?; + + // The original plain genesis ("genesis::runtimeGenesis") is no longer needed, so + // just remove it: + org_chain_spec_json + .get_mut("genesis") + .and_then(|genesis| genesis.as_object_mut()) + .and_then(|genesis| genesis.remove("runtimeGenesis")); json_patch::merge(&mut org_chain_spec_json, genesis_json); let chain_spec_json = serde_json::to_string_pretty(&org_chain_spec_json) diff --git a/substrate/client/chain-spec/src/genesis_config_builder.rs b/substrate/client/chain-spec/src/genesis_config_builder.rs index 66989495d4231..5fe8f9dc053c1 100644 --- a/substrate/client/chain-spec/src/genesis_config_builder.rs +++ b/substrate/client/chain-spec/src/genesis_config_builder.rs @@ -142,11 +142,9 @@ where /// The patching process modifies the default `RuntimeGenesisConfig` according to the following /// rules: /// 1. Existing keys in the default configuration will be overridden by the corresponding values - /// in the patch. + /// in the patch (also applies to `null` values). /// 2. If a key exists in the patch but not in the default configuration, it will be added to /// the resulting `RuntimeGenesisConfig`. - /// 3. Keys in the default configuration that have null values in the patch will be removed from - /// the resulting `RuntimeGenesisConfig`. This is helpful for changing enum variant value. /// /// Please note that the patch may contain full `RuntimeGenesisConfig`. pub fn get_storage_for_patch(&self, patch: Value) -> core::result::Result { diff --git a/substrate/client/chain-spec/src/json_patch.rs b/substrate/client/chain-spec/src/json_patch.rs index c3930069a60d0..a223792374e0c 100644 --- a/substrate/client/chain-spec/src/json_patch.rs +++ b/substrate/client/chain-spec/src/json_patch.rs @@ -22,9 +22,10 @@ use serde_json::Value; /// Recursively merges two JSON objects, `a` and `b`, into a single object. /// -/// If a key exists in both objects, the value from `b` will override the value from `a`. -/// If a key exists in `b` with a `null` value, it will be removed from `a`. +/// If a key exists in both objects, the value from `b` will override the value from `a` (also if +/// value in `b` is `null`). /// If a key exists only in `b` and not in `a`, it will be added to `a`. +/// No keys will be removed from `a`. /// /// # Arguments /// @@ -34,11 +35,7 @@ pub fn merge(a: &mut Value, b: Value) { match (a, b) { (Value::Object(a), Value::Object(b)) => for (k, v) in b { - if v.is_null() { - a.remove(&k); - } else { - merge(a.entry(k).or_insert(Value::Null), v); - } + merge(a.entry(k).or_insert(Value::Null), v); }, (a, b) => *a = b, }; @@ -166,7 +163,7 @@ mod tests { } #[test] - fn test6_patch_removes_keys_if_null() { + fn test6_patch_does_not_remove_keys_if_null() { let mut j1 = json!({ "a": { "name": "xxx", @@ -186,6 +183,16 @@ mod tests { }); merge(&mut j1, j2); - assert_eq!(j1, json!({ "a": {"name":"xxx", "value":456, "enum_variant_2": 32 }})); + assert_eq!( + j1, + json!({ + "a": { + "name":"xxx", + "value":456, + "enum_variant_1": null, + "enum_variant_2": 32 + } + }) + ); } } diff --git a/substrate/client/consensus/common/src/import_queue.rs b/substrate/client/consensus/common/src/import_queue.rs index 1baa67398a49c..602683907d482 100644 --- a/substrate/client/consensus/common/src/import_queue.rs +++ b/substrate/client/consensus/common/src/import_queue.rs @@ -107,7 +107,7 @@ pub trait Verifier: Send + Sync { /// /// The `import_*` methods can be called in order to send elements for the import queue to verify. pub trait ImportQueueService: Send { - /// Import bunch of blocks, every next block must be an ancestor of the previous block in the + /// Import a bunch of blocks, every next block must be an ancestor of the previous block in the /// list. fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec>); @@ -132,21 +132,21 @@ pub trait ImportQueue: Send { /// This method should behave in a way similar to `Future::poll`. It can register the current /// task and notify later when more actions are ready to be polled. To continue the comparison, /// it is as if this method always returned `Poll::Pending`. - fn poll_actions(&mut self, cx: &mut futures::task::Context, link: &mut dyn Link); + fn poll_actions(&mut self, cx: &mut futures::task::Context, link: &dyn Link); /// Start asynchronous runner for import queue. /// /// Takes an object implementing [`Link`] which allows the import queue to /// influence the synchronization process. - async fn run(self, link: Box>); + async fn run(self, link: &dyn Link); } /// Hooks that the verification queue can use to influence the synchronization /// algorithm. -pub trait Link: Send { +pub trait Link: Send + Sync { /// Batch of blocks imported, with or without error. fn blocks_processed( - &mut self, + &self, _imported: usize, _count: usize, _results: Vec<(BlockImportResult, B::Hash)>, @@ -155,7 +155,7 @@ pub trait Link: Send { /// Justification import result. fn justification_imported( - &mut self, + &self, _who: RuntimeOrigin, _hash: &B::Hash, _number: NumberFor, @@ -164,7 +164,7 @@ pub trait Link: Send { } /// Request a justification for the given block. - fn request_justification(&mut self, _hash: &B::Hash, _number: NumberFor) {} + fn request_justification(&self, _hash: &B::Hash, _number: NumberFor) {} } /// Block import successful result. diff --git a/substrate/client/consensus/common/src/import_queue/basic_queue.rs b/substrate/client/consensus/common/src/import_queue/basic_queue.rs index 7b371145e2e7d..21270859dd75c 100644 --- a/substrate/client/consensus/common/src/import_queue/basic_queue.rs +++ b/substrate/client/consensus/common/src/import_queue/basic_queue.rs @@ -177,7 +177,7 @@ impl ImportQueue for BasicQueue { } /// Poll actions from network. - fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link) { + fn poll_actions(&mut self, cx: &mut Context, link: &dyn Link) { if self.result_port.poll_actions(cx, link).is_err() { log::error!( target: LOG_TARGET, @@ -190,9 +190,9 @@ impl ImportQueue for BasicQueue { /// /// Takes an object implementing [`Link`] which allows the import queue to /// influence the synchronization process. - async fn run(mut self, mut link: Box>) { + async fn run(mut self, link: &dyn Link) { loop { - if let Err(_) = self.result_port.next_action(&mut *link).await { + if let Err(_) = self.result_port.next_action(link).await { log::error!(target: "sync", "poll_actions: Background import task is no longer alive"); return } @@ -223,7 +223,7 @@ mod worker_messages { async fn block_import_process( mut block_import: BoxBlockImport, verifier: impl Verifier, - mut result_sender: BufferedLinkSender, + result_sender: BufferedLinkSender, mut block_import_receiver: TracingUnboundedReceiver>, metrics: Option, ) { @@ -501,6 +501,7 @@ mod tests { import_queue::Verifier, }; use futures::{executor::block_on, Future}; + use parking_lot::Mutex; use sp_test_primitives::{Block, BlockNumber, Hash, Header}; #[async_trait::async_trait] @@ -558,29 +559,29 @@ mod tests { #[derive(Default)] struct TestLink { - events: Vec, + events: Mutex>, } impl Link for TestLink { fn blocks_processed( - &mut self, + &self, _imported: usize, _count: usize, results: Vec<(Result, BlockImportError>, Hash)>, ) { if let Some(hash) = results.into_iter().find_map(|(r, h)| r.ok().map(|_| h)) { - self.events.push(Event::BlockImported(hash)); + self.events.lock().push(Event::BlockImported(hash)); } } fn justification_imported( - &mut self, + &self, _who: RuntimeOrigin, hash: &Hash, _number: BlockNumber, _success: bool, ) { - self.events.push(Event::JustificationImported(*hash)) + self.events.lock().push(Event::JustificationImported(*hash)) } } @@ -638,7 +639,7 @@ mod tests { hash }; - let mut link = TestLink::default(); + let link = TestLink::default(); // we send a bunch of tasks to the worker let block1 = import_block(1); @@ -653,13 +654,13 @@ mod tests { // we poll the worker until we have processed 9 events block_on(futures::future::poll_fn(|cx| { - while link.events.len() < 9 { + while link.events.lock().len() < 9 { match Future::poll(Pin::new(&mut worker), cx) { Poll::Pending => {}, Poll::Ready(()) => panic!("import queue worker should not conclude."), } - result_port.poll_actions(cx, &mut link).unwrap(); + result_port.poll_actions(cx, &link).unwrap(); } Poll::Ready(()) @@ -667,8 +668,8 @@ mod tests { // all justification tasks must be done before any block import work assert_eq!( - link.events, - vec![ + &*link.events.lock(), + &[ Event::JustificationImported(justification1), Event::JustificationImported(justification2), Event::JustificationImported(justification3), diff --git a/substrate/client/consensus/common/src/import_queue/buffered_link.rs b/substrate/client/consensus/common/src/import_queue/buffered_link.rs index c23a4b0d5d0ab..67131b06a32e5 100644 --- a/substrate/client/consensus/common/src/import_queue/buffered_link.rs +++ b/substrate/client/consensus/common/src/import_queue/buffered_link.rs @@ -27,13 +27,13 @@ //! # use sc_consensus::import_queue::buffered_link::buffered_link; //! # use sp_test_primitives::Block; //! # struct DummyLink; impl Link for DummyLink {} -//! # let mut my_link = DummyLink; +//! # let my_link = DummyLink; //! let (mut tx, mut rx) = buffered_link::(100_000); //! tx.blocks_processed(0, 0, vec![]); //! //! // Calls `my_link.blocks_processed(0, 0, vec![])` when polled. //! let _fut = futures::future::poll_fn(move |cx| { -//! rx.poll_actions(cx, &mut my_link); +//! rx.poll_actions(cx, &my_link).unwrap(); //! std::task::Poll::Pending::<()> //! }); //! ``` @@ -90,7 +90,7 @@ pub enum BlockImportWorkerMsg { impl Link for BufferedLinkSender { fn blocks_processed( - &mut self, + &self, imported: usize, count: usize, results: Vec<(BlockImportResult, B::Hash)>, @@ -101,7 +101,7 @@ impl Link for BufferedLinkSender { } fn justification_imported( - &mut self, + &self, who: RuntimeOrigin, hash: &B::Hash, number: NumberFor, @@ -111,7 +111,7 @@ impl Link for BufferedLinkSender { let _ = self.tx.unbounded_send(msg); } - fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { + fn request_justification(&self, hash: &B::Hash, number: NumberFor) { let _ = self .tx .unbounded_send(BlockImportWorkerMsg::RequestJustification(*hash, number)); @@ -125,7 +125,7 @@ pub struct BufferedLinkReceiver { impl BufferedLinkReceiver { /// Send action for the synchronization to perform. - pub fn send_actions(&mut self, msg: BlockImportWorkerMsg, link: &mut dyn Link) { + pub fn send_actions(&mut self, msg: BlockImportWorkerMsg, link: &dyn Link) { match msg { BlockImportWorkerMsg::BlocksProcessed(imported, count, results) => link.blocks_processed(imported, count, results), @@ -144,7 +144,7 @@ impl BufferedLinkReceiver { /// it is as if this method always returned `Poll::Pending`. /// /// Returns an error if the corresponding [`BufferedLinkSender`] has been closed. - pub fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link) -> Result<(), ()> { + pub fn poll_actions(&mut self, cx: &mut Context, link: &dyn Link) -> Result<(), ()> { loop { let msg = match Stream::poll_next(Pin::new(&mut self.rx), cx) { Poll::Ready(Some(msg)) => msg, @@ -152,12 +152,12 @@ impl BufferedLinkReceiver { Poll::Pending => break Ok(()), }; - self.send_actions(msg, &mut *link); + self.send_actions(msg, link); } } /// Poll next element from import queue and send the corresponding action command over the link. - pub async fn next_action(&mut self, link: &mut dyn Link) -> Result<(), ()> { + pub async fn next_action(&mut self, link: &dyn Link) -> Result<(), ()> { if let Some(msg) = self.rx.next().await { self.send_actions(msg, link); return Ok(()) diff --git a/substrate/client/consensus/common/src/import_queue/mock.rs b/substrate/client/consensus/common/src/import_queue/mock.rs index 64ac532ded854..a238f72568ca6 100644 --- a/substrate/client/consensus/common/src/import_queue/mock.rs +++ b/substrate/client/consensus/common/src/import_queue/mock.rs @@ -40,7 +40,7 @@ mockall::mock! { impl ImportQueue for ImportQueue { fn service(&self) -> Box>; fn service_ref(&mut self) -> &mut dyn ImportQueueService; - fn poll_actions<'a>(&mut self, cx: &mut futures::task::Context<'a>, link: &mut dyn Link); - async fn run(self, link: Box>); + fn poll_actions<'a>(&mut self, cx: &mut futures::task::Context<'a>, link: &dyn Link); + async fn run(self, link: &'__mockall_link dyn Link); } } diff --git a/substrate/client/network/Cargo.toml b/substrate/client/network/Cargo.toml index 8ae3de72f7964..c8fd28e081094 100644 --- a/substrate/client/network/Cargo.toml +++ b/substrate/client/network/Cargo.toml @@ -70,7 +70,7 @@ mockall = { workspace = true } multistream-select = { workspace = true } rand = { workspace = true, default-features = true } tempfile = { workspace = true } -tokio = { features = ["macros"], workspace = true, default-features = true } +tokio = { features = ["macros", "rt-multi-thread"], workspace = true, default-features = true } tokio-util = { features = ["compat"], workspace = true } tokio-test = { workspace = true } sc-block-builder = { workspace = true, default-features = true } @@ -83,5 +83,17 @@ sp-tracing = { workspace = true, default-features = true } substrate-test-runtime = { workspace = true } substrate-test-runtime-client = { workspace = true } +criterion = { workspace = true, default-features = true, features = ["async_tokio"] } +sc-consensus = { workspace = true, default-features = true } + [features] default = [] + + +[[bench]] +name = "notifications_protocol" +harness = false + +[[bench]] +name = "request_response_protocol" +harness = false diff --git a/substrate/client/network/benches/notifications_protocol.rs b/substrate/client/network/benches/notifications_protocol.rs new file mode 100644 index 0000000000000..7d32c9faeba10 --- /dev/null +++ b/substrate/client/network/benches/notifications_protocol.rs @@ -0,0 +1,290 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use criterion::{ + criterion_group, criterion_main, AxisScale, BenchmarkId, Criterion, PlotConfiguration, + Throughput, +}; +use sc_network::{ + config::{ + FullNetworkConfiguration, MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, + NonReservedPeerMode, NotificationHandshake, Params, ProtocolId, Role, SetConfig, + }, + service::traits::NotificationEvent, + NetworkWorker, NotificationMetrics, NotificationService, Roles, +}; +use sc_network_common::sync::message::BlockAnnouncesHandshake; +use sc_network_types::build_multiaddr; +use sp_runtime::traits::Zero; +use std::{ + net::{IpAddr, Ipv4Addr, TcpListener}, + str::FromStr, +}; +use substrate_test_runtime_client::runtime; + +const MAX_SIZE: u64 = 2u64.pow(30); +const SAMPLE_SIZE: usize = 50; +const NOTIFICATIONS: usize = 50; +const EXPONENTS: &[(u32, &'static str)] = &[ + (6, "64B"), + (9, "512B"), + (12, "4KB"), + (15, "64KB"), + (18, "256KB"), + (21, "2MB"), + (24, "16MB"), + (27, "128MB"), +]; + +// TODO: It's be better to bind system-provided port when initializing the worker +fn get_listen_address() -> sc_network::Multiaddr { + let ip = Ipv4Addr::from_str("127.0.0.1").unwrap(); + let listener = TcpListener::bind((IpAddr::V4(ip), 0)).unwrap(); // Bind to a random port + let local_addr = listener.local_addr().unwrap(); + let port = local_addr.port(); + + build_multiaddr!(Ip4(ip), Tcp(port)) +} + +pub fn create_network_worker( + listen_addr: sc_network::Multiaddr, +) -> (NetworkWorker, Box) { + let role = Role::Full; + let genesis_hash = runtime::Hash::zero(); + let (block_announce_config, notification_service) = NonDefaultSetConfig::new( + "/block-announces/1".into(), + vec!["/bench-notifications-protocol/block-announces/1".into()], + MAX_SIZE, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + Roles::from(&role), + Zero::zero(), + genesis_hash, + genesis_hash, + ))), + SetConfig { + in_peers: 1, + out_peers: 1, + reserved_nodes: vec![], + non_reserved_mode: NonReservedPeerMode::Accept, + }, + ); + let mut net_conf = NetworkConfiguration::new_local(); + net_conf.listen_addresses = vec![listen_addr]; + let worker = NetworkWorker::::new(Params::< + runtime::Block, + runtime::Hash, + NetworkWorker<_, _>, + > { + block_announce_config, + role, + executor: Box::new(|f| { + tokio::spawn(f); + }), + genesis_hash, + network_config: FullNetworkConfiguration::new(&net_conf, None), + protocol_id: ProtocolId::from("bench-protocol-name"), + fork_id: None, + metrics_registry: None, + bitswap_config: None, + notification_metrics: NotificationMetrics::new(None), + }) + .unwrap(); + + (worker, notification_service) +} + +async fn run_serially(size: usize, limit: usize) { + let listen_address1 = get_listen_address(); + let listen_address2 = get_listen_address(); + let (worker1, mut notification_service1) = create_network_worker(listen_address1); + let (worker2, mut notification_service2) = create_network_worker(listen_address2.clone()); + let peer_id2: sc_network::PeerId = (*worker2.local_peer_id()).into(); + + worker1 + .add_reserved_peer(MultiaddrWithPeerId { multiaddr: listen_address2, peer_id: peer_id2 }) + .unwrap(); + + let network1_run = worker1.run(); + let network2_run = worker2.run(); + let (tx, rx) = async_channel::bounded(10); + + let network1 = tokio::spawn(async move { + tokio::pin!(network1_run); + loop { + tokio::select! { + _ = &mut network1_run => {}, + event = notification_service1.next_event() => { + match event { + Some(NotificationEvent::NotificationStreamOpened { .. }) => { + notification_service1 + .send_async_notification(&peer_id2, vec![0; size]) + .await + .unwrap(); + }, + event => panic!("Unexpected event {:?}", event), + }; + }, + message = rx.recv() => { + match message { + Ok(Some(_)) => { + notification_service1 + .send_async_notification(&peer_id2, vec![0; size]) + .await + .unwrap(); + }, + Ok(None) => break, + Err(err) => panic!("Unexpected error {:?}", err), + + } + } + } + } + }); + let network2 = tokio::spawn(async move { + let mut received_counter = 0; + tokio::pin!(network2_run); + loop { + tokio::select! { + _ = &mut network2_run => {}, + event = notification_service2.next_event() => { + match event { + Some(NotificationEvent::ValidateInboundSubstream { result_tx, .. }) => { + result_tx.send(sc_network::service::traits::ValidationResult::Accept).unwrap(); + }, + Some(NotificationEvent::NotificationStreamOpened { .. }) => {}, + Some(NotificationEvent::NotificationReceived { .. }) => { + received_counter += 1; + if received_counter >= limit { + let _ = tx.send(None).await; + break + } + let _ = tx.send(Some(())).await; + }, + event => panic!("Unexpected event {:?}", event), + }; + }, + } + } + }); + + let _ = tokio::join!(network1, network2); +} + +async fn run_with_backpressure(size: usize, limit: usize) { + let listen_address1 = get_listen_address(); + let listen_address2 = get_listen_address(); + let (worker1, mut notification_service1) = create_network_worker(listen_address1); + let (worker2, mut notification_service2) = create_network_worker(listen_address2.clone()); + let peer_id2: sc_network::PeerId = (*worker2.local_peer_id()).into(); + + worker1 + .add_reserved_peer(MultiaddrWithPeerId { multiaddr: listen_address2, peer_id: peer_id2 }) + .unwrap(); + + let network1_run = worker1.run(); + let network2_run = worker2.run(); + + let network1 = tokio::spawn(async move { + let mut sent_counter = 0; + tokio::pin!(network1_run); + loop { + tokio::select! { + _ = &mut network1_run => {}, + event = notification_service1.next_event() => { + match event { + Some(NotificationEvent::NotificationStreamOpened { .. }) => { + while sent_counter < limit { + sent_counter += 1; + notification_service1 + .send_async_notification(&peer_id2, vec![0; size]) + .await + .unwrap(); + } + }, + Some(NotificationEvent::NotificationStreamClosed { .. }) => { + if sent_counter != limit { panic!("Stream closed unexpectedly") } + break + }, + event => panic!("Unexpected event {:?}", event), + }; + }, + } + } + }); + let network2 = tokio::spawn(async move { + let mut received_counter = 0; + tokio::pin!(network2_run); + loop { + tokio::select! { + _ = &mut network2_run => {}, + event = notification_service2.next_event() => { + match event { + Some(NotificationEvent::ValidateInboundSubstream { result_tx, .. }) => { + result_tx.send(sc_network::service::traits::ValidationResult::Accept).unwrap(); + }, + Some(NotificationEvent::NotificationStreamOpened { .. }) => {}, + Some(NotificationEvent::NotificationStreamClosed { .. }) => { + if received_counter != limit { panic!("Stream closed unexpectedly") } + break + }, + Some(NotificationEvent::NotificationReceived { .. }) => { + received_counter += 1; + if received_counter >= limit { break } + }, + event => panic!("Unexpected event {:?}", event), + }; + }, + } + } + }); + + let _ = tokio::join!(network1, network2); +} + +fn run_benchmark(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic); + let mut group = c.benchmark_group("notifications_benchmark"); + group.plot_config(plot_config); + + for &(exponent, label) in EXPONENTS.iter() { + let size = 2usize.pow(exponent); + group.throughput(Throughput::Bytes(NOTIFICATIONS as u64 * size as u64)); + group.bench_with_input( + BenchmarkId::new("consistently", label), + &(size, NOTIFICATIONS), + |b, &(size, limit)| { + b.to_async(&rt).iter(|| run_serially(size, limit)); + }, + ); + group.bench_with_input( + BenchmarkId::new("with_backpressure", label), + &(size, NOTIFICATIONS), + |b, &(size, limit)| { + b.to_async(&rt).iter(|| run_with_backpressure(size, limit)); + }, + ); + } +} + +criterion_group! { + name = benches; + config = Criterion::default().sample_size(SAMPLE_SIZE); + targets = run_benchmark +} +criterion_main!(benches); diff --git a/substrate/client/network/benches/request_response_protocol.rs b/substrate/client/network/benches/request_response_protocol.rs new file mode 100644 index 0000000000000..09bf829f5a7e9 --- /dev/null +++ b/substrate/client/network/benches/request_response_protocol.rs @@ -0,0 +1,278 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use criterion::{ + criterion_group, criterion_main, AxisScale, BenchmarkId, Criterion, PlotConfiguration, + Throughput, +}; +use sc_network::{ + config::{ + FullNetworkConfiguration, IncomingRequest, NetworkConfiguration, NonDefaultSetConfig, + NonReservedPeerMode, NotificationHandshake, OutgoingResponse, Params, ProtocolId, Role, + SetConfig, + }, + IfDisconnected, NetworkBackend, NetworkRequest, NetworkWorker, NotificationMetrics, + NotificationService, Roles, +}; +use sc_network_common::sync::message::BlockAnnouncesHandshake; +use sc_network_types::build_multiaddr; +use sp_runtime::traits::Zero; +use std::{ + net::{IpAddr, Ipv4Addr, TcpListener}, + str::FromStr, + time::Duration, +}; +use substrate_test_runtime_client::runtime; + +const MAX_SIZE: u64 = 2u64.pow(30); +const SAMPLE_SIZE: usize = 50; +const REQUESTS: usize = 50; +const EXPONENTS: &[(u32, &'static str)] = &[ + (6, "64B"), + (9, "512B"), + (12, "4KB"), + (15, "64KB"), + (18, "256KB"), + (21, "2MB"), + (24, "16MB"), + (27, "128MB"), +]; + +fn get_listen_address() -> sc_network::Multiaddr { + let ip = Ipv4Addr::from_str("127.0.0.1").unwrap(); + let listener = TcpListener::bind((IpAddr::V4(ip), 0)).unwrap(); // Bind to a random port + let local_addr = listener.local_addr().unwrap(); + let port = local_addr.port(); + + build_multiaddr!(Ip4(ip), Tcp(port)) +} + +pub fn create_network_worker( + listen_addr: sc_network::Multiaddr, +) -> ( + NetworkWorker, + async_channel::Receiver, + Box, +) { + let (tx, rx) = async_channel::bounded(10); + let request_response_config = + NetworkWorker::::request_response_config( + "/request-response/1".into(), + vec![], + MAX_SIZE, + MAX_SIZE, + Duration::from_secs(2), + Some(tx), + ); + let mut net_conf = NetworkConfiguration::new_local(); + net_conf.listen_addresses = vec![listen_addr]; + let mut network_config = FullNetworkConfiguration::new(&net_conf, None); + network_config.add_request_response_protocol(request_response_config); + let (block_announce_config, notification_service) = NonDefaultSetConfig::new( + "/block-announces/1".into(), + vec![], + 1024, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + Roles::from(&Role::Full), + Zero::zero(), + runtime::Hash::zero(), + runtime::Hash::zero(), + ))), + SetConfig { + in_peers: 1, + out_peers: 1, + reserved_nodes: vec![], + non_reserved_mode: NonReservedPeerMode::Accept, + }, + ); + let worker = NetworkWorker::::new(Params::< + runtime::Block, + runtime::Hash, + NetworkWorker<_, _>, + > { + block_announce_config, + role: Role::Full, + executor: Box::new(|f| { + tokio::spawn(f); + }), + genesis_hash: runtime::Hash::zero(), + network_config, + protocol_id: ProtocolId::from("bench-request-response-protocol"), + fork_id: None, + metrics_registry: None, + bitswap_config: None, + notification_metrics: NotificationMetrics::new(None), + }) + .unwrap(); + + (worker, rx, notification_service) +} + +async fn run_serially(size: usize, limit: usize) { + let listen_address1 = get_listen_address(); + let listen_address2 = get_listen_address(); + let (mut worker1, _rx1, _notification_service1) = create_network_worker(listen_address1); + let service1 = worker1.service().clone(); + let (worker2, rx2, _notification_service2) = create_network_worker(listen_address2.clone()); + let peer_id2 = *worker2.local_peer_id(); + + worker1.add_known_address(peer_id2, listen_address2.into()); + + let network1_run = worker1.run(); + let network2_run = worker2.run(); + let (break_tx, break_rx) = async_channel::bounded(10); + let requests = async move { + let mut sent_counter = 0; + while sent_counter < limit { + let _ = service1 + .request( + peer_id2.into(), + "/request-response/1".into(), + vec![0; 2], + None, + IfDisconnected::TryConnect, + ) + .await + .unwrap(); + sent_counter += 1; + } + let _ = break_tx.send(()).await; + }; + + let network1 = tokio::spawn(async move { + tokio::pin!(requests); + tokio::pin!(network1_run); + loop { + tokio::select! { + _ = &mut network1_run => {}, + _ = &mut requests => break, + } + } + }); + let network2 = tokio::spawn(async move { + tokio::pin!(network2_run); + loop { + tokio::select! { + _ = &mut network2_run => {}, + res = rx2.recv() => { + let IncomingRequest { pending_response, .. } = res.unwrap(); + pending_response.send(OutgoingResponse { + result: Ok(vec![0; size]), + reputation_changes: vec![], + sent_feedback: None, + }).unwrap(); + }, + _ = break_rx.recv() => break, + } + } + }); + + let _ = tokio::join!(network1, network2); +} + +// The libp2p request-response implementation does not provide any backpressure feedback. +// So this benchmark is useless until we implement it for litep2p. +#[allow(dead_code)] +async fn run_with_backpressure(size: usize, limit: usize) { + let listen_address1 = get_listen_address(); + let listen_address2 = get_listen_address(); + let (mut worker1, _rx1, _notification_service1) = create_network_worker(listen_address1); + let service1 = worker1.service().clone(); + let (worker2, rx2, _notification_service2) = create_network_worker(listen_address2.clone()); + let peer_id2 = *worker2.local_peer_id(); + + worker1.add_known_address(peer_id2, listen_address2.into()); + + let network1_run = worker1.run(); + let network2_run = worker2.run(); + let (break_tx, break_rx) = async_channel::bounded(10); + let requests = futures::future::join_all((0..limit).into_iter().map(|_| { + let (tx, rx) = futures::channel::oneshot::channel(); + service1.start_request( + peer_id2.into(), + "/request-response/1".into(), + vec![0; 8], + None, + tx, + IfDisconnected::TryConnect, + ); + rx + })); + + let network1 = tokio::spawn(async move { + tokio::pin!(requests); + tokio::pin!(network1_run); + loop { + tokio::select! { + _ = &mut network1_run => {}, + responses = &mut requests => { + for res in responses { + res.unwrap().unwrap(); + } + let _ = break_tx.send(()).await; + break; + }, + } + } + }); + let network2 = tokio::spawn(async move { + tokio::pin!(network2_run); + loop { + tokio::select! { + _ = &mut network2_run => {}, + res = rx2.recv() => { + let IncomingRequest { pending_response, .. } = res.unwrap(); + pending_response.send(OutgoingResponse { + result: Ok(vec![0; size]), + reputation_changes: vec![], + sent_feedback: None, + }).unwrap(); + }, + _ = break_rx.recv() => break, + } + } + }); + + let _ = tokio::join!(network1, network2); +} + +fn run_benchmark(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic); + let mut group = c.benchmark_group("request_response_benchmark"); + group.plot_config(plot_config); + + for &(exponent, label) in EXPONENTS.iter() { + let size = 2usize.pow(exponent); + group.throughput(Throughput::Bytes(REQUESTS as u64 * size as u64)); + group.bench_with_input( + BenchmarkId::new("consistently", label), + &(size, REQUESTS), + |b, &(size, limit)| { + b.to_async(&rt).iter(|| run_serially(size, limit)); + }, + ); + } +} + +criterion_group! { + name = benches; + config = Criterion::default().sample_size(SAMPLE_SIZE); + targets = run_benchmark +} +criterion_main!(benches); diff --git a/substrate/client/network/src/discovery.rs b/substrate/client/network/src/discovery.rs index 49e0797c126c5..8080bda9a5749 100644 --- a/substrate/client/network/src/discovery.rs +++ b/substrate/client/network/src/discovery.rs @@ -749,16 +749,28 @@ impl NetworkBehaviour for DiscoveryBehaviour { self.mdns.on_swarm_event(FromSwarm::NewListenAddr(e)); }, FromSwarm::ExternalAddrConfirmed(e @ ExternalAddrConfirmed { addr }) => { - let new_addr = addr.clone().with(Protocol::P2p(self.local_peer_id)); + let mut address = addr.clone(); - if Self::can_add_to_dht(addr) { + if let Some(Protocol::P2p(peer_id)) = addr.iter().last() { + if peer_id != self.local_peer_id { + warn!( + target: "sub-libp2p", + "🔍 Discovered external address for a peer that is not us: {addr}", + ); + // Ensure this address is not propagated to kademlia. + return + } + } else { + address.push(Protocol::P2p(self.local_peer_id)); + } + + if Self::can_add_to_dht(&address) { // NOTE: we might re-discover the same address multiple times // in which case we just want to refrain from logging. - if self.known_external_addresses.insert(new_addr.clone()) { + if self.known_external_addresses.insert(address.clone()) { info!( target: "sub-libp2p", - "🔍 Discovered new external address for our node: {}", - new_addr, + "🔍 Discovered new external address for our node: {address}", ); } } diff --git a/substrate/client/network/src/litep2p/discovery.rs b/substrate/client/network/src/litep2p/discovery.rs index 7b2e713dffd23..9043f9420e8dc 100644 --- a/substrate/client/network/src/litep2p/discovery.rs +++ b/substrate/client/network/src/litep2p/discovery.rs @@ -162,6 +162,9 @@ pub enum DiscoveryEvent { /// Discovery. pub struct Discovery { + /// Local peer ID. + local_peer_id: litep2p::PeerId, + /// Ping event stream. ping_event_stream: Box + Send + Unpin>, @@ -233,6 +236,7 @@ impl Discovery { /// Enables `/ipfs/ping/1.0.0` and `/ipfs/identify/1.0.0` by default and starts /// the mDNS peer discovery if it was enabled. pub fn new + Clone>( + local_peer_id: litep2p::PeerId, config: &NetworkConfiguration, genesis_hash: Hash, fork_id: Option<&str>, @@ -273,6 +277,7 @@ impl Discovery { ( Self { + local_peer_id, ping_event_stream, identify_event_stream, mdns_event_stream, @@ -591,24 +596,43 @@ impl Stream for Discovery { observed_address, .. })) => { - let (is_new, expired_address) = - this.is_new_external_address(&observed_address, peer); - - if let Some(expired_address) = expired_address { - log::trace!( - target: LOG_TARGET, - "Removing expired external address expired={expired_address} is_new={is_new} observed={observed_address}", - ); - - this.pending_events.push_back(DiscoveryEvent::ExternalAddressExpired { - address: expired_address, - }); - } + let observed_address = + if let Some(Protocol::P2p(peer_id)) = observed_address.iter().last() { + if peer_id != *this.local_peer_id.as_ref() { + log::warn!( + target: LOG_TARGET, + "Discovered external address for a peer that is not us: {observed_address}", + ); + None + } else { + Some(observed_address) + } + } else { + Some(observed_address.with(Protocol::P2p(this.local_peer_id.into()))) + }; + + // Ensure that an external address with a different peer ID does not have + // side effects of evicting other external addresses via `ExternalAddressExpired`. + if let Some(observed_address) = observed_address { + let (is_new, expired_address) = + this.is_new_external_address(&observed_address, peer); + + if let Some(expired_address) = expired_address { + log::trace!( + target: LOG_TARGET, + "Removing expired external address expired={expired_address} is_new={is_new} observed={observed_address}", + ); + + this.pending_events.push_back(DiscoveryEvent::ExternalAddressExpired { + address: expired_address, + }); + } - if is_new { - this.pending_events.push_back(DiscoveryEvent::ExternalAddressDiscovered { - address: observed_address.clone(), - }); + if is_new { + this.pending_events.push_back(DiscoveryEvent::ExternalAddressDiscovered { + address: observed_address.clone(), + }); + } } return Poll::Ready(Some(DiscoveryEvent::Identified { diff --git a/substrate/client/network/src/litep2p/mod.rs b/substrate/client/network/src/litep2p/mod.rs index df4244890f967..87b9924236744 100644 --- a/substrate/client/network/src/litep2p/mod.rs +++ b/substrate/client/network/src/litep2p/mod.rs @@ -540,6 +540,7 @@ impl NetworkBackend for Litep2pNetworkBac let listen_addresses = Arc::new(Default::default()); let (discovery, ping_config, identify_config, kademlia_config, maybe_mdns_config) = Discovery::new( + local_peer_id, &network_config, params.genesis_hash, params.fork_id.as_deref(), diff --git a/substrate/client/network/sync/src/block_relay_protocol.rs b/substrate/client/network/sync/src/block_relay_protocol.rs index 3c5b3739e8222..13639d851b272 100644 --- a/substrate/client/network/sync/src/block_relay_protocol.rs +++ b/substrate/client/network/sync/src/block_relay_protocol.rs @@ -21,7 +21,7 @@ use sc_network::{request_responses::RequestFailure, NetworkBackend, ProtocolName use sc_network_common::sync::message::{BlockData, BlockRequest}; use sc_network_types::PeerId; use sp_runtime::traits::Block as BlockT; -use std::sync::Arc; +use std::{fmt, sync::Arc}; /// The serving side of the block relay protocol. It runs a single instance /// of the server task that processes the incoming protocol messages. @@ -34,7 +34,10 @@ pub trait BlockServer: Send { /// The client side stub to download blocks from peers. This is a handle /// that can be used to initiate concurrent downloads. #[async_trait::async_trait] -pub trait BlockDownloader: Send + Sync { +pub trait BlockDownloader: fmt::Debug + Send + Sync { + /// Protocol name used by block downloader. + fn protocol_name(&self) -> &ProtocolName; + /// Performs the protocol specific sequence to fetch the blocks from the peer. /// Output: if the download succeeds, the response is a `Vec` which is /// in a format specific to the protocol implementation. The block data diff --git a/substrate/client/network/sync/src/block_request_handler.rs b/substrate/client/network/sync/src/block_request_handler.rs index 6e970b3993106..80234170bc203 100644 --- a/substrate/client/network/sync/src/block_request_handler.rs +++ b/substrate/client/network/sync/src/block_request_handler.rs @@ -502,6 +502,7 @@ enum HandleRequestError { } /// The full block downloader implementation of [`BlockDownloader]. +#[derive(Debug)] pub struct FullBlockDownloader { protocol_name: ProtocolName, network: NetworkServiceHandle, @@ -576,6 +577,10 @@ impl FullBlockDownloader { #[async_trait::async_trait] impl BlockDownloader for FullBlockDownloader { + fn protocol_name(&self) -> &ProtocolName { + &self.protocol_name + } + async fn download_blocks( &self, who: PeerId, diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index dceea9954c6e9..cc2089d1974c3 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -23,30 +23,22 @@ use crate::{ block_announce_validator::{ BlockAnnounceValidationResult, BlockAnnounceValidator as BlockAnnounceValidatorStream, }, - block_relay_protocol::{BlockDownloader, BlockResponseError}, pending_responses::{PendingResponses, ResponseEvent}, - schema::v1::{StateRequest, StateResponse}, service::{ self, syncing_service::{SyncingService, ToServiceCommand}, }, - strategy::{ - warp::{EncodedProof, WarpProofRequest}, - StrategyKey, SyncingAction, SyncingStrategy, - }, - types::{ - BadPeer, ExtendedPeerInfo, OpaqueStateRequest, OpaqueStateResponse, PeerRequest, SyncEvent, - }, + strategy::{SyncingAction, SyncingStrategy}, + types::{BadPeer, ExtendedPeerInfo, SyncEvent}, LOG_TARGET, }; use codec::{Decode, DecodeAll, Encode}; -use futures::{channel::oneshot, FutureExt, StreamExt}; +use futures::{channel::oneshot, StreamExt}; use log::{debug, error, trace, warn}; use prometheus_endpoint::{ register, Counter, Gauge, MetricSource, Opts, PrometheusError, Registry, SourcedGauge, U64, }; -use prost::Message; use schnellru::{ByLength, LruMap}; use tokio::time::{Interval, MissedTickBehavior}; @@ -55,7 +47,7 @@ use sc_consensus::{import_queue::ImportQueueService, IncomingBlock}; use sc_network::{ config::{FullNetworkConfiguration, NotificationHandshake, ProtocolId, SetConfig}, peer_store::PeerStoreProvider, - request_responses::{IfDisconnected, OutboundFailure, RequestFailure}, + request_responses::{OutboundFailure, RequestFailure}, service::{ traits::{Direction, NotificationConfig, NotificationEvent, ValidationResult}, NotificationMetrics, @@ -66,7 +58,7 @@ use sc_network::{ }; use sc_network_common::{ role::Roles, - sync::message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, + sync::message::{BlockAnnounce, BlockAnnouncesHandshake, BlockState}, }; use sc_network_types::PeerId; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; @@ -102,8 +94,6 @@ mod rep { pub const GENESIS_MISMATCH: Rep = Rep::new_fatal("Genesis mismatch"); /// Peer send us a block announcement that failed at validation. pub const BAD_BLOCK_ANNOUNCEMENT: Rep = Rep::new(-(1 << 12), "Bad block announcement"); - /// We received a message that failed to decode. - pub const BAD_MESSAGE: Rep = Rep::new(-(1 << 12), "Bad message"); /// Peer is on unsupported protocol version. pub const BAD_PROTOCOL: Rep = Rep::new_fatal("Unsupported protocol"); /// Reputation change when a peer refuses a request. @@ -264,10 +254,7 @@ pub struct SyncingEngine { peer_store_handle: Arc, /// Pending responses - pending_responses: PendingResponses, - - /// Block downloader - block_downloader: Arc>, + pending_responses: PendingResponses, /// Handle to import queue. import_queue: Box>, @@ -291,12 +278,11 @@ where network_metrics: NotificationMetrics, net_config: &FullNetworkConfiguration::Hash, N>, protocol_id: ProtocolId, - fork_id: &Option, + fork_id: Option<&str>, block_announce_validator: Box + Send>, syncing_strategy: Box>, network_service: service::network::NetworkServiceHandle, import_queue: Box>, - block_downloader: Arc>, peer_store_handle: Arc, ) -> Result<(Self, SyncingService, N::NotificationProtocolConfig), ClientError> where @@ -417,7 +403,6 @@ where None }, pending_responses: PendingResponses::new(), - block_downloader, import_queue, }, SyncingService::new(tx, num_connected, is_major_syncing), @@ -583,57 +568,42 @@ where } fn process_strategy_actions(&mut self) -> Result<(), ClientError> { - for action in self.strategy.actions()? { + for action in self.strategy.actions(&self.network_service)? { match action { - SyncingAction::SendBlockRequest { peer_id, key, request } => { - // Sending block request implies dropping obsolete pending response as we are - // not interested in it anymore (see [`SyncingAction::SendBlockRequest`]). - let removed = self.pending_responses.remove(peer_id, key); - self.send_block_request(peer_id, key, request.clone()); - - if removed { - warn!( - target: LOG_TARGET, - "Processed `ChainSyncAction::SendBlockRequest` to {} from {:?} with {:?}. \ - Stale response removed!", - peer_id, - key, - request, - ) - } else { + SyncingAction::StartRequest { peer_id, key, request, remove_obsolete } => { + if !self.peers.contains_key(&peer_id) { trace!( target: LOG_TARGET, - "Processed `ChainSyncAction::SendBlockRequest` to {} from {:?} with {:?}.", - peer_id, - key, - request, - ) + "Cannot start request with strategy key {key:?} to unknown peer \ + {peer_id}", + ); + debug_assert!(false); + continue; } + if remove_obsolete { + if self.pending_responses.remove(peer_id, key) { + warn!( + target: LOG_TARGET, + "Processed `SyncingAction::StartRequest` to {peer_id} with \ + strategy key {key:?}. Stale response removed!", + ) + } else { + trace!( + target: LOG_TARGET, + "Processed `SyncingAction::StartRequest` to {peer_id} with \ + strategy key {key:?}.", + ) + } + } + + self.pending_responses.insert(peer_id, key, request); }, SyncingAction::CancelRequest { peer_id, key } => { let removed = self.pending_responses.remove(peer_id, key); trace!( target: LOG_TARGET, - "Processed {action:?}, response removed: {removed}.", - ); - }, - SyncingAction::SendStateRequest { peer_id, key, protocol_name, request } => { - self.send_state_request(peer_id, key, protocol_name, request); - - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::SendStateRequest` to {peer_id}.", - ); - }, - SyncingAction::SendWarpProofRequest { peer_id, key, protocol_name, request } => { - self.send_warp_proof_request(peer_id, key, protocol_name, request.clone()); - - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::SendWarpProofRequest` to {}, request: {:?}.", - peer_id, - request, + "Processed `SyncingAction::CancelRequest`, response removed: {removed}.", ); }, SyncingAction::DropPeer(BadPeer(peer_id, rep)) => { @@ -1000,160 +970,12 @@ where Ok(()) } - fn send_block_request(&mut self, peer_id: PeerId, key: StrategyKey, request: BlockRequest) { - if !self.peers.contains_key(&peer_id) { - trace!(target: LOG_TARGET, "Cannot send block request to unknown peer {peer_id}"); - debug_assert!(false); - return; - } - - let downloader = self.block_downloader.clone(); - - self.pending_responses.insert( - peer_id, - key, - PeerRequest::Block(request.clone()), - async move { downloader.download_blocks(peer_id, request).await }.boxed(), - ); - } - - fn send_state_request( - &mut self, - peer_id: PeerId, - key: StrategyKey, - protocol_name: ProtocolName, - request: OpaqueStateRequest, - ) { - if !self.peers.contains_key(&peer_id) { - trace!(target: LOG_TARGET, "Cannot send state request to unknown peer {peer_id}"); - debug_assert!(false); - return; - } - - let (tx, rx) = oneshot::channel(); - - self.pending_responses.insert(peer_id, key, PeerRequest::State, rx.boxed()); - - match Self::encode_state_request(&request) { - Ok(data) => { - self.network_service.start_request( - peer_id, - protocol_name, - data, - tx, - IfDisconnected::ImmediateError, - ); - }, - Err(err) => { - log::warn!( - target: LOG_TARGET, - "Failed to encode state request {request:?}: {err:?}", - ); - }, - } - } - - fn send_warp_proof_request( - &mut self, - peer_id: PeerId, - key: StrategyKey, - protocol_name: ProtocolName, - request: WarpProofRequest, - ) { - if !self.peers.contains_key(&peer_id) { - trace!(target: LOG_TARGET, "Cannot send warp proof request to unknown peer {peer_id}"); - debug_assert!(false); - return; - } - - let (tx, rx) = oneshot::channel(); - - self.pending_responses.insert(peer_id, key, PeerRequest::WarpProof, rx.boxed()); - - self.network_service.start_request( - peer_id, - protocol_name, - request.encode(), - tx, - IfDisconnected::ImmediateError, - ); - } - - fn encode_state_request(request: &OpaqueStateRequest) -> Result, String> { - let request: &StateRequest = request.0.downcast_ref().ok_or_else(|| { - "Failed to downcast opaque state response during encoding, this is an \ - implementation bug." - .to_string() - })?; - - Ok(request.encode_to_vec()) - } - - fn decode_state_response(response: &[u8]) -> Result { - let response = StateResponse::decode(response) - .map_err(|error| format!("Failed to decode state response: {error}"))?; - - Ok(OpaqueStateResponse(Box::new(response))) - } - - fn process_response_event(&mut self, response_event: ResponseEvent) { - let ResponseEvent { peer_id, key, request, response } = response_event; + fn process_response_event(&mut self, response_event: ResponseEvent) { + let ResponseEvent { peer_id, key, response: response_result } = response_event; - match response { - Ok(Ok((resp, _))) => match request { - PeerRequest::Block(req) => { - match self.block_downloader.block_response_into_blocks(&req, resp) { - Ok(blocks) => { - self.strategy.on_block_response(peer_id, key, req, blocks); - }, - Err(BlockResponseError::DecodeFailed(e)) => { - debug!( - target: LOG_TARGET, - "Failed to decode block response from peer {:?}: {:?}.", - peer_id, - e - ); - self.network_service.report_peer(peer_id, rep::BAD_MESSAGE); - self.network_service.disconnect_peer( - peer_id, - self.block_announce_protocol_name.clone(), - ); - return; - }, - Err(BlockResponseError::ExtractionFailed(e)) => { - debug!( - target: LOG_TARGET, - "Failed to extract blocks from peer response {:?}: {:?}.", - peer_id, - e - ); - self.network_service.report_peer(peer_id, rep::BAD_MESSAGE); - return; - }, - } - }, - PeerRequest::State => { - let response = match Self::decode_state_response(&resp[..]) { - Ok(proto) => proto, - Err(e) => { - debug!( - target: LOG_TARGET, - "Failed to decode state response from peer {peer_id:?}: {e:?}.", - ); - self.network_service.report_peer(peer_id, rep::BAD_MESSAGE); - self.network_service.disconnect_peer( - peer_id, - self.block_announce_protocol_name.clone(), - ); - return; - }, - }; - - self.strategy.on_state_response(peer_id, key, response); - }, - PeerRequest::WarpProof => { - self.strategy.on_warp_proof_response(&peer_id, key, EncodedProof(resp)); - }, + match response_result { + Ok(Ok((response, protocol_name))) => { + self.strategy.on_generic_response(&peer_id, key, protocol_name, response); }, Ok(Err(e)) => { debug!(target: LOG_TARGET, "Request to peer {peer_id:?} failed: {e:?}."); @@ -1214,7 +1036,7 @@ where /// Get config for the block announcement protocol fn get_block_announce_proto_config::Hash>>( protocol_id: ProtocolId, - fork_id: &Option, + fork_id: Option<&str>, roles: Roles, best_number: NumberFor, best_hash: B::Hash, @@ -1225,7 +1047,7 @@ where ) -> (N::NotificationProtocolConfig, Box) { let block_announces_protocol = { let genesis_hash = genesis_hash.as_ref(); - if let Some(ref fork_id) = fork_id { + if let Some(fork_id) = fork_id { format!( "/{}/{}/block-announces/1", array_bytes::bytes2hex("", genesis_hash), diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index c458c7a5da498..e503a1cbdb185 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -18,6 +18,7 @@ //! Blockchain syncing implementation in Substrate. +pub use schema::v1::*; pub use service::syncing_service::SyncingService; pub use strategy::warp::{WarpSyncConfig, WarpSyncPhase, WarpSyncProgress}; pub use types::{SyncEvent, SyncEventStream, SyncState, SyncStatus, SyncStatusProvider}; diff --git a/substrate/client/network/sync/src/mock.rs b/substrate/client/network/sync/src/mock.rs index 741fa7139583f..bf25156f9703d 100644 --- a/substrate/client/network/sync/src/mock.rs +++ b/substrate/client/network/sync/src/mock.rs @@ -27,10 +27,13 @@ use sc_network_types::PeerId; use sp_runtime::traits::Block as BlockT; mockall::mock! { + #[derive(Debug)] pub BlockDownloader {} #[async_trait::async_trait] impl BlockDownloaderT for BlockDownloader { + fn protocol_name(&self) -> &ProtocolName; + async fn download_blocks( &self, who: PeerId, diff --git a/substrate/client/network/sync/src/pending_responses.rs b/substrate/client/network/sync/src/pending_responses.rs index 7d2d598a2e061..46e6ae6263281 100644 --- a/substrate/client/network/sync/src/pending_responses.rs +++ b/substrate/client/network/sync/src/pending_responses.rs @@ -19,7 +19,7 @@ //! [`PendingResponses`] is responsible for keeping track of pending responses and //! polling them. [`Stream`] implemented by [`PendingResponses`] never terminates. -use crate::{strategy::StrategyKey, types::PeerRequest, LOG_TARGET}; +use crate::{strategy::StrategyKey, LOG_TARGET}; use futures::{ channel::oneshot, future::BoxFuture, @@ -27,61 +27,49 @@ use futures::{ FutureExt, StreamExt, }; use log::error; +use std::any::Any; use sc_network::{request_responses::RequestFailure, types::ProtocolName}; use sc_network_types::PeerId; -use sp_runtime::traits::Block as BlockT; use std::task::{Context, Poll, Waker}; use tokio_stream::StreamMap; /// Response result. -type ResponseResult = Result, ProtocolName), RequestFailure>, oneshot::Canceled>; +type ResponseResult = + Result, ProtocolName), RequestFailure>, oneshot::Canceled>; /// A future yielding [`ResponseResult`]. -type ResponseFuture = BoxFuture<'static, ResponseResult>; +pub(crate) type ResponseFuture = BoxFuture<'static, ResponseResult>; /// An event we receive once a pending response future resolves. -pub(crate) struct ResponseEvent { +pub(crate) struct ResponseEvent { pub peer_id: PeerId, pub key: StrategyKey, - pub request: PeerRequest, pub response: ResponseResult, } /// Stream taking care of polling pending responses. -pub(crate) struct PendingResponses { +pub(crate) struct PendingResponses { /// Pending responses - pending_responses: - StreamMap<(PeerId, StrategyKey), BoxStream<'static, (PeerRequest, ResponseResult)>>, + pending_responses: StreamMap<(PeerId, StrategyKey), BoxStream<'static, ResponseResult>>, /// Waker to implement never terminating stream waker: Option, } -impl PendingResponses { +impl PendingResponses { pub fn new() -> Self { Self { pending_responses: StreamMap::new(), waker: None } } - pub fn insert( - &mut self, - peer_id: PeerId, - key: StrategyKey, - request: PeerRequest, - response_future: ResponseFuture, - ) { - let request_type = request.get_type(); - + pub fn insert(&mut self, peer_id: PeerId, key: StrategyKey, response_future: ResponseFuture) { if self .pending_responses - .insert( - (peer_id, key), - Box::pin(async move { (request, response_future.await) }.into_stream()), - ) + .insert((peer_id, key), Box::pin(response_future.into_stream())) .is_some() { error!( target: LOG_TARGET, - "Discarded pending response from peer {peer_id}, request type: {request_type:?}.", + "Discarded pending response from peer {peer_id}, strategy key: {key:?}.", ); debug_assert!(false); } @@ -112,21 +100,21 @@ impl PendingResponses { } } -impl Stream for PendingResponses { - type Item = ResponseEvent; +impl Stream for PendingResponses { + type Item = ResponseEvent; fn poll_next( mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll> { match self.pending_responses.poll_next_unpin(cx) { - Poll::Ready(Some(((peer_id, key), (request, response)))) => { + Poll::Ready(Some(((peer_id, key), response))) => { // We need to manually remove the stream, because `StreamMap` doesn't know yet that // it's going to yield `None`, so may not remove it before the next request is made // to the same peer. self.pending_responses.remove(&(peer_id, key)); - Poll::Ready(Some(ResponseEvent { peer_id, key, request, response })) + Poll::Ready(Some(ResponseEvent { peer_id, key, response })) }, Poll::Ready(None) | Poll::Pending => { self.waker = Some(cx.waker().clone()); @@ -138,7 +126,7 @@ impl Stream for PendingResponses { } // As [`PendingResponses`] never terminates, we can easily implement [`FusedStream`] for it. -impl FusedStream for PendingResponses { +impl FusedStream for PendingResponses { fn is_terminated(&self) -> bool { false } diff --git a/substrate/client/network/sync/src/service/mock.rs b/substrate/client/network/sync/src/service/mock.rs index 141edc7c88414..300aa076515f8 100644 --- a/substrate/client/network/sync/src/service/mock.rs +++ b/substrate/client/network/sync/src/service/mock.rs @@ -45,19 +45,19 @@ mockall::mock! { impl sc_consensus::Link for ChainSyncInterface { fn blocks_processed( - &mut self, + &self, imported: usize, count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)>, ); fn justification_imported( - &mut self, + &self, who: PeerId, hash: &B::Hash, number: NumberFor, success: bool, ); - fn request_justification(&mut self, hash: &B::Hash, number: NumberFor); + fn request_justification(&self, hash: &B::Hash, number: NumberFor); } } diff --git a/substrate/client/network/sync/src/service/network.rs b/substrate/client/network/sync/src/service/network.rs index e848b5f62c1b8..139e1a986a927 100644 --- a/substrate/client/network/sync/src/service/network.rs +++ b/substrate/client/network/sync/src/service/network.rs @@ -39,9 +39,11 @@ impl Network for T where T: NetworkPeers + NetworkRequest {} /// calls the `NetworkService` on its behalf. pub struct NetworkServiceProvider { rx: TracingUnboundedReceiver, + handle: NetworkServiceHandle, } /// Commands that `ChainSync` wishes to send to `NetworkService` +#[derive(Debug)] pub enum ToServiceCommand { /// Call `NetworkPeers::disconnect_peer()` DisconnectPeer(PeerId, ProtocolName), @@ -61,7 +63,7 @@ pub enum ToServiceCommand { /// Handle that is (temporarily) passed to `ChainSync` so it can /// communicate with `NetworkService` through `SyncingEngine` -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct NetworkServiceHandle { tx: TracingUnboundedSender, } @@ -99,15 +101,23 @@ impl NetworkServiceHandle { impl NetworkServiceProvider { /// Create new `NetworkServiceProvider` - pub fn new() -> (Self, NetworkServiceHandle) { + pub fn new() -> Self { let (tx, rx) = tracing_unbounded("mpsc_network_service_provider", 100_000); - (Self { rx }, NetworkServiceHandle::new(tx)) + Self { rx, handle: NetworkServiceHandle::new(tx) } + } + + /// Get handle to talk to the provider + pub fn handle(&self) -> NetworkServiceHandle { + self.handle.clone() } /// Run the `NetworkServiceProvider` - pub async fn run(mut self, service: Arc) { - while let Some(inner) = self.rx.next().await { + pub async fn run(self, service: Arc) { + let Self { mut rx, handle } = self; + drop(handle); + + while let Some(inner) = rx.next().await { match inner { ToServiceCommand::DisconnectPeer(peer, protocol_name) => service.disconnect_peer(peer, protocol_name), @@ -129,7 +139,8 @@ mod tests { // and then reported #[tokio::test] async fn disconnect_and_report_peer() { - let (provider, handle) = NetworkServiceProvider::new(); + let provider = NetworkServiceProvider::new(); + let handle = provider.handle(); let peer = PeerId::random(); let proto = ProtocolName::from("test-protocol"); diff --git a/substrate/client/network/sync/src/service/syncing_service.rs b/substrate/client/network/sync/src/service/syncing_service.rs index 08a2b36118a9d..b56af2b9976a1 100644 --- a/substrate/client/network/sync/src/service/syncing_service.rs +++ b/substrate/client/network/sync/src/service/syncing_service.rs @@ -177,7 +177,7 @@ impl SyncStatusProvider for SyncingService { impl Link for SyncingService { fn blocks_processed( - &mut self, + &self, imported: usize, count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)>, @@ -188,7 +188,7 @@ impl Link for SyncingService { } fn justification_imported( - &mut self, + &self, who: PeerId, hash: &B::Hash, number: NumberFor, @@ -199,7 +199,7 @@ impl Link for SyncingService { .unbounded_send(ToServiceCommand::JustificationImported(who, *hash, number, success)); } - fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { + fn request_justification(&self, hash: &B::Hash, number: NumberFor) { let _ = self.tx.unbounded_send(ToServiceCommand::RequestJustification(*hash, number)); } } diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 81998b7576bbf..2ac6674231e54 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -16,50 +16,35 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! [`PolkadotSyncingStrategy`] is a proxy between [`crate::engine::SyncingEngine`] -//! and specific syncing algorithms. +//! [`SyncingStrategy`] defines an interface [`crate::engine::SyncingEngine`] uses as a specific +//! syncing algorithm. +//! +//! A few different strategies are provided by Substrate out of the box with custom strategies +//! possible too. pub mod chain_sync; mod disconnected_peers; -mod state; +pub mod polkadot; +pub mod state; pub mod state_sync; pub mod warp; use crate::{ - block_request_handler::MAX_BLOCKS_IN_RESPONSE, - types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncStatus}, - LOG_TARGET, + pending_responses::ResponseFuture, + service::network::NetworkServiceHandle, + types::{BadPeer, SyncStatus}, }; -use chain_sync::{ChainSync, ChainSyncMode}; -use log::{debug, error, info}; -use prometheus_endpoint::Registry; -use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network::ProtocolName; -use sc_network_common::sync::{ - message::{BlockAnnounce, BlockData, BlockRequest}, - SyncMode, -}; +use sc_network_common::sync::message::BlockAnnounce; use sc_network_types::PeerId; -use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; +use sp_blockchain::Error as ClientError; use sp_consensus::BlockOrigin; use sp_runtime::{ - traits::{Block as BlockT, Header, NumberFor}, + traits::{Block as BlockT, NumberFor}, Justifications, }; -use state::{StateStrategy, StateStrategyAction}; -use std::{collections::HashMap, sync::Arc}; -use warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction, WarpSyncConfig}; - -/// Corresponding `ChainSync` mode. -fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { - match sync_mode { - SyncMode::Full => ChainSyncMode::Full, - SyncMode::LightState { skip_proofs, storage_chain_mode } => - ChainSyncMode::LightState { skip_proofs, storage_chain_mode }, - SyncMode::Warp => ChainSyncMode::Full, - } -} +use std::any::Any; /// Syncing strategy for syncing engine to use pub trait SyncingStrategy: Send @@ -101,29 +86,16 @@ where /// Report a justification import (successful or not). fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool); - /// Process block response. - fn on_block_response( - &mut self, - peer_id: PeerId, - key: StrategyKey, - request: BlockRequest, - blocks: Vec>, - ); - - /// Process state response. - fn on_state_response( - &mut self, - peer_id: PeerId, - key: StrategyKey, - response: OpaqueStateResponse, - ); - - /// Process warp proof response. - fn on_warp_proof_response( + /// Process generic response. + /// + /// Strategy has to create opaque response and should be to downcast it back into concrete type + /// internally. Failure to downcast is an implementation bug. + fn on_generic_response( &mut self, peer_id: &PeerId, key: StrategyKey, - response: EncodedProof, + protocol_name: ProtocolName, + response: Box, ); /// A batch of blocks that have been processed, with or without errors. @@ -160,52 +132,32 @@ where /// Get actions that should be performed by the owner on the strategy's behalf #[must_use] - fn actions(&mut self) -> Result>, ClientError>; -} - -/// Syncing configuration containing data for all strategies. -#[derive(Clone, Debug)] -pub struct SyncingConfig { - /// Syncing mode. - pub mode: SyncMode, - /// The number of parallel downloads to guard against slow peers. - pub max_parallel_downloads: u32, - /// Maximum number of blocks to request. - pub max_blocks_per_request: u32, - /// Prometheus metrics registry. - pub metrics_registry: Option, - /// Protocol name used to send out state requests - pub state_request_protocol_name: ProtocolName, + fn actions( + &mut self, + // TODO: Consider making this internal property of the strategy + network_service: &NetworkServiceHandle, + ) -> Result>, ClientError>; } /// The key identifying a specific strategy for responses routing. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum StrategyKey { - /// Warp sync initiated this request. - Warp, - /// State sync initiated this request. - State, - /// `ChainSync` initiated this request. - ChainSync, +pub struct StrategyKey(&'static str); + +impl StrategyKey { + /// Instantiate opaque strategy key. + pub const fn new(key: &'static str) -> Self { + Self(key) + } } -#[derive(Debug)] pub enum SyncingAction { - /// Send block request to peer. Always implies dropping a stale block request to the same peer. - SendBlockRequest { peer_id: PeerId, key: StrategyKey, request: BlockRequest }, - /// Send state request to peer. - SendStateRequest { - peer_id: PeerId, - key: StrategyKey, - protocol_name: ProtocolName, - request: OpaqueStateRequest, - }, - /// Send warp proof request to peer. - SendWarpProofRequest { + /// Start request to peer. + StartRequest { peer_id: PeerId, key: StrategyKey, - protocol_name: ProtocolName, - request: WarpProofRequest, + request: ResponseFuture, + // Whether to remove obsolete pending responses. + remove_obsolete: bool, }, /// Drop stale request. CancelRequest { peer_id: PeerId, key: StrategyKey }, @@ -225,444 +177,20 @@ pub enum SyncingAction { } impl SyncingAction { - fn is_finished(&self) -> bool { + /// Returns `true` if the syncing action has completed. + pub fn is_finished(&self) -> bool { matches!(self, SyncingAction::Finished) } -} - -impl From> for SyncingAction { - fn from(action: WarpSyncAction) -> Self { - match action { - WarpSyncAction::SendWarpProofRequest { peer_id, protocol_name, request } => - SyncingAction::SendWarpProofRequest { - peer_id, - key: StrategyKey::Warp, - protocol_name, - request, - }, - WarpSyncAction::SendBlockRequest { peer_id, request } => - SyncingAction::SendBlockRequest { peer_id, key: StrategyKey::Warp, request }, - WarpSyncAction::DropPeer(bad_peer) => SyncingAction::DropPeer(bad_peer), - WarpSyncAction::Finished => SyncingAction::Finished, - } - } -} - -impl From> for SyncingAction { - fn from(action: StateStrategyAction) -> Self { - match action { - StateStrategyAction::SendStateRequest { peer_id, protocol_name, request } => - SyncingAction::SendStateRequest { - peer_id, - key: StrategyKey::State, - protocol_name, - request, - }, - StateStrategyAction::DropPeer(bad_peer) => SyncingAction::DropPeer(bad_peer), - StateStrategyAction::ImportBlocks { origin, blocks } => - SyncingAction::ImportBlocks { origin, blocks }, - StateStrategyAction::Finished => SyncingAction::Finished, - } - } -} - -/// Proxy to specific syncing strategies used in Polkadot. -pub struct PolkadotSyncingStrategy { - /// Initial syncing configuration. - config: SyncingConfig, - /// Client used by syncing strategies. - client: Arc, - /// Warp strategy. - warp: Option>, - /// State strategy. - state: Option>, - /// `ChainSync` strategy.` - chain_sync: Option>, - /// Connected peers and their best blocks used to seed a new strategy when switching to it in - /// `PolkadotSyncingStrategy::proceed_to_next`. - peer_best_blocks: HashMap)>, -} - -impl SyncingStrategy for PolkadotSyncingStrategy -where - B: BlockT, - Client: HeaderBackend - + BlockBackend - + HeaderMetadata - + ProofProvider - + Send - + Sync - + 'static, -{ - fn add_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { - self.peer_best_blocks.insert(peer_id, (best_hash, best_number)); - - self.warp.as_mut().map(|s| s.add_peer(peer_id, best_hash, best_number)); - self.state.as_mut().map(|s| s.add_peer(peer_id, best_hash, best_number)); - self.chain_sync.as_mut().map(|s| s.add_peer(peer_id, best_hash, best_number)); - } - - fn remove_peer(&mut self, peer_id: &PeerId) { - self.warp.as_mut().map(|s| s.remove_peer(peer_id)); - self.state.as_mut().map(|s| s.remove_peer(peer_id)); - self.chain_sync.as_mut().map(|s| s.remove_peer(peer_id)); - - self.peer_best_blocks.remove(peer_id); - } - - fn on_validated_block_announce( - &mut self, - is_best: bool, - peer_id: PeerId, - announce: &BlockAnnounce, - ) -> Option<(B::Hash, NumberFor)> { - let new_best = if let Some(ref mut warp) = self.warp { - warp.on_validated_block_announce(is_best, peer_id, announce) - } else if let Some(ref mut state) = self.state { - state.on_validated_block_announce(is_best, peer_id, announce) - } else if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.on_validated_block_announce(is_best, peer_id, announce) - } else { - error!(target: LOG_TARGET, "No syncing strategy is active."); - debug_assert!(false); - Some((announce.header.hash(), *announce.header.number())) - }; - - if let Some(new_best) = new_best { - if let Some(best) = self.peer_best_blocks.get_mut(&peer_id) { - *best = new_best; - } else { - debug!( - target: LOG_TARGET, - "Cannot update `peer_best_blocks` as peer {peer_id} is not known to `Strategy` \ - (already disconnected?)", - ); - } - } - - new_best - } - - fn set_sync_fork_request(&mut self, peers: Vec, hash: &B::Hash, number: NumberFor) { - // Fork requests are only handled by `ChainSync`. - if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.set_sync_fork_request(peers.clone(), hash, number); - } - } - - fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { - // Justifications can only be requested via `ChainSync`. - if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.request_justification(hash, number); - } - } - - fn clear_justification_requests(&mut self) { - // Justification requests can only be cleared by `ChainSync`. - if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.clear_justification_requests(); - } - } - - fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { - // Only `ChainSync` is interested in justification import. - if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.on_justification_import(hash, number, success); - } - } - - fn on_block_response( - &mut self, - peer_id: PeerId, - key: StrategyKey, - request: BlockRequest, - blocks: Vec>, - ) { - if let (StrategyKey::Warp, Some(ref mut warp)) = (key, &mut self.warp) { - warp.on_block_response(peer_id, request, blocks); - } else if let (StrategyKey::ChainSync, Some(ref mut chain_sync)) = - (key, &mut self.chain_sync) - { - chain_sync.on_block_response(peer_id, key, request, blocks); - } else { - error!( - target: LOG_TARGET, - "`on_block_response()` called with unexpected key {key:?} \ - or corresponding strategy is not active.", - ); - debug_assert!(false); - } - } - - fn on_state_response( - &mut self, - peer_id: PeerId, - key: StrategyKey, - response: OpaqueStateResponse, - ) { - if let (StrategyKey::State, Some(ref mut state)) = (key, &mut self.state) { - state.on_state_response(peer_id, response); - } else if let (StrategyKey::ChainSync, Some(ref mut chain_sync)) = - (key, &mut self.chain_sync) - { - chain_sync.on_state_response(peer_id, key, response); - } else { - error!( - target: LOG_TARGET, - "`on_state_response()` called with unexpected key {key:?} \ - or corresponding strategy is not active.", - ); - debug_assert!(false); - } - } - - fn on_warp_proof_response( - &mut self, - peer_id: &PeerId, - key: StrategyKey, - response: EncodedProof, - ) { - if let (StrategyKey::Warp, Some(ref mut warp)) = (key, &mut self.warp) { - warp.on_warp_proof_response(peer_id, response); - } else { - error!( - target: LOG_TARGET, - "`on_warp_proof_response()` called with unexpected key {key:?} \ - or warp strategy is not active", - ); - debug_assert!(false); - } - } - - fn on_blocks_processed( - &mut self, - imported: usize, - count: usize, - results: Vec<(Result>, BlockImportError>, B::Hash)>, - ) { - // Only `StateStrategy` and `ChainSync` are interested in block processing notifications. - if let Some(ref mut state) = self.state { - state.on_blocks_processed(imported, count, results); - } else if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.on_blocks_processed(imported, count, results); - } - } - - fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { - // Only `ChainSync` is interested in block finalization notifications. - if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.on_block_finalized(hash, number); - } - } - - fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { - // This is relevant to `ChainSync` only. - if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.update_chain_info(best_hash, best_number); - } - } - - fn is_major_syncing(&self) -> bool { - self.warp.is_some() || - self.state.is_some() || - match self.chain_sync { - Some(ref s) => s.status().state.is_major_syncing(), - None => unreachable!("At least one syncing strategy is active; qed"), - } - } - - fn num_peers(&self) -> usize { - self.peer_best_blocks.len() - } - - fn status(&self) -> SyncStatus { - // This function presumes that strategies are executed serially and must be refactored - // once we have parallel strategies. - if let Some(ref warp) = self.warp { - warp.status() - } else if let Some(ref state) = self.state { - state.status() - } else if let Some(ref chain_sync) = self.chain_sync { - chain_sync.status() - } else { - unreachable!("At least one syncing strategy is always active; qed") - } - } - - fn num_downloaded_blocks(&self) -> usize { - self.chain_sync - .as_ref() - .map_or(0, |chain_sync| chain_sync.num_downloaded_blocks()) - } - - fn num_sync_requests(&self) -> usize { - self.chain_sync.as_ref().map_or(0, |chain_sync| chain_sync.num_sync_requests()) - } - - fn actions(&mut self) -> Result>, ClientError> { - // This function presumes that strategies are executed serially and must be refactored once - // we have parallel strategies. - let actions: Vec<_> = if let Some(ref mut warp) = self.warp { - warp.actions().map(Into::into).collect() - } else if let Some(ref mut state) = self.state { - state.actions().map(Into::into).collect() - } else if let Some(ref mut chain_sync) = self.chain_sync { - chain_sync.actions()? - } else { - unreachable!("At least one syncing strategy is always active; qed") - }; - - if actions.iter().any(SyncingAction::is_finished) { - self.proceed_to_next()?; - } - - Ok(actions) - } -} - -impl PolkadotSyncingStrategy -where - B: BlockT, - Client: HeaderBackend - + BlockBackend - + HeaderMetadata - + ProofProvider - + Send - + Sync - + 'static, -{ - /// Initialize a new syncing strategy. - pub fn new( - mut config: SyncingConfig, - client: Arc, - warp_sync_config: Option>, - warp_sync_protocol_name: Option, - ) -> Result { - if config.max_blocks_per_request > MAX_BLOCKS_IN_RESPONSE as u32 { - info!( - target: LOG_TARGET, - "clamping maximum blocks per request to {MAX_BLOCKS_IN_RESPONSE}", - ); - config.max_blocks_per_request = MAX_BLOCKS_IN_RESPONSE as u32; - } - - if let SyncMode::Warp = config.mode { - let warp_sync_config = warp_sync_config - .expect("Warp sync configuration must be supplied in warp sync mode."); - let warp_sync = - WarpSync::new(client.clone(), warp_sync_config, warp_sync_protocol_name); - Ok(Self { - config, - client, - warp: Some(warp_sync), - state: None, - chain_sync: None, - peer_best_blocks: Default::default(), - }) - } else { - let chain_sync = ChainSync::new( - chain_sync_mode(config.mode), - client.clone(), - config.max_parallel_downloads, - config.max_blocks_per_request, - config.state_request_protocol_name.clone(), - config.metrics_registry.as_ref(), - std::iter::empty(), - )?; - Ok(Self { - config, - client, - warp: None, - state: None, - chain_sync: Some(chain_sync), - peer_best_blocks: Default::default(), - }) - } - } - - /// Proceed with the next strategy if the active one finished. - pub fn proceed_to_next(&mut self) -> Result<(), ClientError> { - // The strategies are switched as `WarpSync` -> `StateStrategy` -> `ChainSync`. - if let Some(ref mut warp) = self.warp { - match warp.take_result() { - Some(res) => { - info!( - target: LOG_TARGET, - "Warp sync is complete, continuing with state sync." - ); - let state_sync = StateStrategy::new( - self.client.clone(), - res.target_header, - res.target_body, - res.target_justifications, - false, - self.peer_best_blocks - .iter() - .map(|(peer_id, (_, best_number))| (*peer_id, *best_number)), - self.config.state_request_protocol_name.clone(), - ); - - self.warp = None; - self.state = Some(state_sync); - Ok(()) - }, - None => { - error!( - target: LOG_TARGET, - "Warp sync failed. Continuing with full sync." - ); - let chain_sync = match ChainSync::new( - chain_sync_mode(self.config.mode), - self.client.clone(), - self.config.max_parallel_downloads, - self.config.max_blocks_per_request, - self.config.state_request_protocol_name.clone(), - self.config.metrics_registry.as_ref(), - self.peer_best_blocks.iter().map(|(peer_id, (best_hash, best_number))| { - (*peer_id, *best_hash, *best_number) - }), - ) { - Ok(chain_sync) => chain_sync, - Err(e) => { - error!(target: LOG_TARGET, "Failed to start `ChainSync`."); - return Err(e) - }, - }; - - self.warp = None; - self.chain_sync = Some(chain_sync); - Ok(()) - }, - } - } else if let Some(state) = &self.state { - if state.is_succeeded() { - info!(target: LOG_TARGET, "State sync is complete, continuing with block sync."); - } else { - error!(target: LOG_TARGET, "State sync failed. Falling back to full sync."); - } - let chain_sync = match ChainSync::new( - chain_sync_mode(self.config.mode), - self.client.clone(), - self.config.max_parallel_downloads, - self.config.max_blocks_per_request, - self.config.state_request_protocol_name.clone(), - self.config.metrics_registry.as_ref(), - self.peer_best_blocks.iter().map(|(peer_id, (best_hash, best_number))| { - (*peer_id, *best_hash, *best_number) - }), - ) { - Ok(chain_sync) => chain_sync, - Err(e) => { - error!(target: LOG_TARGET, "Failed to start `ChainSync`."); - return Err(e); - }, - }; - self.state = None; - self.chain_sync = Some(chain_sync); - Ok(()) - } else { - unreachable!("Only warp & state strategies can finish; qed") + #[cfg(test)] + pub(crate) fn name(&self) -> &'static str { + match self { + Self::StartRequest { .. } => "StartRequest", + Self::CancelRequest { .. } => "CancelRequest", + Self::DropPeer(_) => "DropPeer", + Self::ImportBlocks { .. } => "ImportBlocks", + Self::ImportJustifications { .. } => "ImportJustifications", + Self::Finished => "Finished", } } } diff --git a/substrate/client/network/sync/src/strategy/chain_sync.rs b/substrate/client/network/sync/src/strategy/chain_sync.rs index 202033e8e00af..18170b77881ea 100644 --- a/substrate/client/network/sync/src/strategy/chain_sync.rs +++ b/substrate/client/network/sync/src/strategy/chain_sync.rs @@ -29,24 +29,28 @@ //! order to update it. use crate::{ + block_relay_protocol::{BlockDownloader, BlockResponseError}, blocks::BlockCollection, justification_requests::ExtraRequests, - schema::v1::StateResponse, + schema::v1::{StateRequest, StateResponse}, + service::network::NetworkServiceHandle, strategy::{ disconnected_peers::DisconnectedPeers, state_sync::{ImportResult, StateSync, StateSyncProvider}, - warp::{EncodedProof, WarpSyncPhase, WarpSyncProgress}, + warp::{WarpSyncPhase, WarpSyncProgress}, StrategyKey, SyncingAction, SyncingStrategy, }, - types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, + types::{BadPeer, SyncState, SyncStatus}, LOG_TARGET, }; +use futures::{channel::oneshot, FutureExt}; use log::{debug, error, info, trace, warn}; use prometheus_endpoint::{register, Gauge, PrometheusError, Registry, U64}; +use prost::Message; use sc_client_api::{blockchain::BlockGap, BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; -use sc_network::ProtocolName; +use sc_network::{IfDisconnected, ProtocolName}; use sc_network_common::sync::message::{ BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, Direction, FromBlock, }; @@ -62,6 +66,7 @@ use sp_runtime::{ }; use std::{ + any::Any, collections::{HashMap, HashSet}, ops::Range, sync::Arc, @@ -123,6 +128,9 @@ mod rep { /// Peer response data does not have requested bits. pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); + + /// We received a message that failed to decode. + pub const BAD_MESSAGE: Rep = Rep::new(-(1 << 12), "Bad message"); } struct Metrics { @@ -324,9 +332,11 @@ pub struct ChainSync { downloaded_blocks: usize, /// State sync in progress, if any. state_sync: Option>, - /// Enable importing existing blocks. This is used used after the state download to + /// Enable importing existing blocks. This is used after the state download to /// catch up to the latest state while re-importing blocks. import_existing: bool, + /// Block downloader + block_downloader: Arc>, /// Gap download process. gap_sync: Option>, /// Pending actions. @@ -348,11 +358,10 @@ where { fn add_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { match self.add_peer_inner(peer_id, best_hash, best_number) { - Ok(Some(request)) => self.actions.push(SyncingAction::SendBlockRequest { - peer_id, - key: StrategyKey::ChainSync, - request, - }), + Ok(Some(request)) => { + let action = self.create_block_request_action(peer_id, request); + self.actions.push(action); + }, Ok(None) => {}, Err(bad_peer) => self.actions.push(SyncingAction::DropPeer(bad_peer)), } @@ -564,82 +573,77 @@ where self.allowed_requests.set_all(); } - fn on_block_response( + fn on_generic_response( &mut self, - peer_id: PeerId, + peer_id: &PeerId, key: StrategyKey, - request: BlockRequest, - blocks: Vec>, + protocol_name: ProtocolName, + response: Box, ) { - if key != StrategyKey::ChainSync { - error!( + if Self::STRATEGY_KEY != key { + warn!( target: LOG_TARGET, - "`on_block_response()` called with unexpected key {key:?} for chain sync", + "Unexpected generic response strategy key {key:?}, protocol {protocol_name}", ); debug_assert!(false); + return; } - let block_response = BlockResponse:: { id: request.id, blocks }; - let blocks_range = || match ( - block_response - .blocks - .first() - .and_then(|b| b.header.as_ref().map(|h| h.number())), - block_response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), - ) { - (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), - (Some(first), Some(_)) => format!(" ({})", first), - _ => Default::default(), - }; - trace!( - target: LOG_TARGET, - "BlockResponse {} from {} with {} blocks {}", - block_response.id, - peer_id, - block_response.blocks.len(), - blocks_range(), - ); + if protocol_name == self.state_request_protocol_name { + let Ok(response) = response.downcast::>() else { + warn!(target: LOG_TARGET, "Failed to downcast state response"); + debug_assert!(false); + return; + }; - let res = if request.fields == BlockAttributes::JUSTIFICATION { - self.on_block_justification(peer_id, block_response) - } else { - self.on_block_data(&peer_id, Some(request), block_response) - }; + if let Err(bad_peer) = self.on_state_data(&peer_id, &response) { + self.actions.push(SyncingAction::DropPeer(bad_peer)); + } + } else if &protocol_name == self.block_downloader.protocol_name() { + let Ok(response) = response + .downcast::<(BlockRequest, Result>, BlockResponseError>)>() + else { + warn!(target: LOG_TARGET, "Failed to downcast block response"); + debug_assert!(false); + return; + }; - if let Err(bad_peer) = res { - self.actions.push(SyncingAction::DropPeer(bad_peer)); - } - } + let (request, response) = *response; + let blocks = match response { + Ok(blocks) => blocks, + Err(BlockResponseError::DecodeFailed(e)) => { + debug!( + target: LOG_TARGET, + "Failed to decode block response from peer {:?}: {:?}.", + peer_id, + e + ); + self.actions.push(SyncingAction::DropPeer(BadPeer(*peer_id, rep::BAD_MESSAGE))); + return; + }, + Err(BlockResponseError::ExtractionFailed(e)) => { + debug!( + target: LOG_TARGET, + "Failed to extract blocks from peer response {:?}: {:?}.", + peer_id, + e + ); + self.actions.push(SyncingAction::DropPeer(BadPeer(*peer_id, rep::BAD_MESSAGE))); + return; + }, + }; - fn on_state_response( - &mut self, - peer_id: PeerId, - key: StrategyKey, - response: OpaqueStateResponse, - ) { - if key != StrategyKey::ChainSync { - error!( + if let Err(bad_peer) = self.on_block_response(peer_id, key, request, blocks) { + self.actions.push(SyncingAction::DropPeer(bad_peer)); + } + } else { + warn!( target: LOG_TARGET, - "`on_state_response()` called with unexpected key {key:?} for chain sync", + "Unexpected generic response protocol {protocol_name}, strategy key \ + {key:?}", ); debug_assert!(false); } - if let Err(bad_peer) = self.on_state_data(&peer_id, response) { - self.actions.push(SyncingAction::DropPeer(bad_peer)); - } - } - - fn on_warp_proof_response( - &mut self, - _peer_id: &PeerId, - _key: StrategyKey, - _response: EncodedProof, - ) { - error!( - target: LOG_TARGET, - "`on_warp_proof_response()` called for chain sync strategy", - ); - debug_assert!(false); } fn on_blocks_processed( @@ -863,30 +867,56 @@ where .count() } - fn actions(&mut self) -> Result>, ClientError> { + fn actions( + &mut self, + network_service: &NetworkServiceHandle, + ) -> Result>, ClientError> { if !self.peers.is_empty() && self.queue_blocks.is_empty() { if let Some((hash, number, skip_proofs)) = self.pending_state_sync_attempt.take() { self.attempt_state_sync(hash, number, skip_proofs); } } - let block_requests = self.block_requests().into_iter().map(|(peer_id, request)| { - SyncingAction::SendBlockRequest { peer_id, key: StrategyKey::ChainSync, request } - }); + let block_requests = self + .block_requests() + .into_iter() + .map(|(peer_id, request)| self.create_block_request_action(peer_id, request)) + .collect::>(); self.actions.extend(block_requests); - let justification_requests = - self.justification_requests().into_iter().map(|(peer_id, request)| { - SyncingAction::SendBlockRequest { peer_id, key: StrategyKey::ChainSync, request } - }); + let justification_requests = self + .justification_requests() + .into_iter() + .map(|(peer_id, request)| self.create_block_request_action(peer_id, request)) + .collect::>(); self.actions.extend(justification_requests); let state_request = self.state_request().into_iter().map(|(peer_id, request)| { - SyncingAction::SendStateRequest { + trace!( + target: LOG_TARGET, + "Created `StrategyRequest` to {peer_id}.", + ); + + let (tx, rx) = oneshot::channel(); + + network_service.start_request( peer_id, - key: StrategyKey::ChainSync, - protocol_name: self.state_request_protocol_name.clone(), - request, + self.state_request_protocol_name.clone(), + request.encode_to_vec(), + tx, + IfDisconnected::ImmediateError, + ); + + SyncingAction::StartRequest { + peer_id, + key: Self::STRATEGY_KEY, + request: async move { + Ok(rx.await?.and_then(|(response, protocol_name)| { + Ok((Box::new(response) as Box, protocol_name)) + })) + } + .boxed(), + remove_obsolete: false, } }); self.actions.extend(state_request); @@ -906,6 +936,9 @@ where + Sync + 'static, { + /// Strategy key used by chain sync. + pub const STRATEGY_KEY: StrategyKey = StrategyKey::new("ChainSync"); + /// Create a new instance. pub fn new( mode: ChainSyncMode, @@ -913,6 +946,7 @@ where max_parallel_downloads: u32, max_blocks_per_request: u32, state_request_protocol_name: ProtocolName, + block_downloader: Arc>, metrics_registry: Option<&Registry>, initial_peers: impl Iterator)>, ) -> Result { @@ -935,6 +969,7 @@ where downloaded_blocks: 0, state_sync: None, import_existing: false, + block_downloader, gap_sync: None, actions: Vec::new(), metrics: metrics_registry.and_then(|r| match Metrics::register(r) { @@ -1075,6 +1110,33 @@ where } } + fn create_block_request_action( + &mut self, + peer_id: PeerId, + request: BlockRequest, + ) -> SyncingAction { + let downloader = self.block_downloader.clone(); + + SyncingAction::StartRequest { + peer_id, + key: Self::STRATEGY_KEY, + request: async move { + Ok(downloader.download_blocks(peer_id, request.clone()).await?.and_then( + |(response, protocol_name)| { + let decoded_response = + downloader.block_response_into_blocks(&request, response); + let result = Box::new((request, decoded_response)) as Box; + Ok((result, protocol_name)) + }, + )) + } + .boxed(), + // Sending block request implies dropping obsolete pending response as we are not + // interested in it anymore. + remove_obsolete: true, + } + } + /// Submit a block response for processing. #[must_use] fn on_block_data( @@ -1248,11 +1310,8 @@ where state: next_state, }; let request = ancestry_request::(next_num); - self.actions.push(SyncingAction::SendBlockRequest { - peer_id: *peer_id, - key: StrategyKey::ChainSync, - request, - }); + let action = self.create_block_request_action(*peer_id, request); + self.actions.push(action); return Ok(()); } else { // Ancestry search is complete. Check if peer is on a stale fork unknown @@ -1334,6 +1393,49 @@ where Ok(()) } + fn on_block_response( + &mut self, + peer_id: &PeerId, + key: StrategyKey, + request: BlockRequest, + blocks: Vec>, + ) -> Result<(), BadPeer> { + if key != Self::STRATEGY_KEY { + error!( + target: LOG_TARGET, + "`on_block_response()` called with unexpected key {key:?} for chain sync", + ); + debug_assert!(false); + } + let block_response = BlockResponse:: { id: request.id, blocks }; + + let blocks_range = || match ( + block_response + .blocks + .first() + .and_then(|b| b.header.as_ref().map(|h| h.number())), + block_response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), + ) { + (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), + (Some(first), Some(_)) => format!(" ({})", first), + _ => Default::default(), + }; + trace!( + target: LOG_TARGET, + "BlockResponse {} from {} with {} blocks {}", + block_response.id, + peer_id, + block_response.blocks.len(), + blocks_range(), + ); + + if request.fields == BlockAttributes::JUSTIFICATION { + self.on_block_justification(*peer_id, block_response) + } else { + self.on_block_data(peer_id, Some(request), block_response) + } + } + /// Submit a justification response for processing. #[must_use] fn on_block_justification( @@ -1548,10 +1650,8 @@ where PeerSyncState::DownloadingGap(_) | PeerSyncState::DownloadingState => { // Cancel a request first, as `add_peer` may generate a new request. - self.actions.push(SyncingAction::CancelRequest { - peer_id, - key: StrategyKey::ChainSync, - }); + self.actions + .push(SyncingAction::CancelRequest { peer_id, key: Self::STRATEGY_KEY }); self.add_peer(peer_id, peer_sync.best_hash, peer_sync.best_number); }, PeerSyncState::DownloadingJustification(_) => { @@ -1831,7 +1931,7 @@ where } /// Get a state request scheduled by sync to be sent out (if any). - fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { + fn state_request(&mut self) -> Option<(PeerId, StateRequest)> { if self.allowed_requests.is_empty() { return None; } @@ -1855,7 +1955,7 @@ where let request = sync.next_request(); trace!(target: LOG_TARGET, "New StateRequest for {}: {:?}", id, request); self.allowed_requests.clear(); - return Some((*id, OpaqueStateRequest(Box::new(request)))); + return Some((*id, request)); } } } @@ -1863,19 +1963,18 @@ where } #[must_use] - fn on_state_data( - &mut self, - peer_id: &PeerId, - response: OpaqueStateResponse, - ) -> Result<(), BadPeer> { - let response: Box = response.0.downcast().map_err(|_error| { - error!( - target: LOG_TARGET, - "Failed to downcast opaque state response, this is an implementation bug." - ); + fn on_state_data(&mut self, peer_id: &PeerId, response: &[u8]) -> Result<(), BadPeer> { + let response = match StateResponse::decode(response) { + Ok(response) => response, + Err(error) => { + debug!( + target: LOG_TARGET, + "Failed to decode state response from peer {peer_id:?}: {error:?}.", + ); - BadPeer(*peer_id, rep::BAD_RESPONSE) - })?; + return Err(BadPeer(*peer_id, rep::BAD_RESPONSE)); + }, + }; if let Some(peer) = self.peers.get_mut(peer_id) { if let PeerSyncState::DownloadingState = peer.state { @@ -1891,7 +1990,7 @@ where response.entries.len(), response.proof.len(), ); - sync.import(*response) + sync.import(response) } else { debug!(target: LOG_TARGET, "Ignored obsolete state response from {peer_id}"); return Err(BadPeer(*peer_id, rep::NOT_REQUESTED)); diff --git a/substrate/client/network/sync/src/strategy/chain_sync/test.rs b/substrate/client/network/sync/src/strategy/chain_sync/test.rs index d13f034e2e8da..4a5682722389a 100644 --- a/substrate/client/network/sync/src/strategy/chain_sync/test.rs +++ b/substrate/client/network/sync/src/strategy/chain_sync/test.rs @@ -19,16 +19,64 @@ //! Tests of [`ChainSync`]. use super::*; -use futures::executor::block_on; +use crate::{ + block_relay_protocol::BlockResponseError, mock::MockBlockDownloader, + service::network::NetworkServiceProvider, +}; +use futures::{channel::oneshot::Canceled, executor::block_on}; use sc_block_builder::BlockBuilderBuilder; +use sc_network::RequestFailure; use sc_network_common::sync::message::{BlockAnnounce, BlockData, BlockState, FromBlock}; use sp_blockchain::HeaderBackend; +use std::sync::Mutex; use substrate_test_runtime_client::{ runtime::{Block, Hash, Header}, BlockBuilderExt, ClientBlockImportExt, ClientExt, DefaultTestClientBuilderExt, TestClient, TestClientBuilder, TestClientBuilderExt, }; +#[derive(Debug)] +struct ProxyBlockDownloader { + protocol_name: ProtocolName, + sender: std::sync::mpsc::Sender>, + request: Mutex>>, +} + +#[async_trait::async_trait] +impl BlockDownloader for ProxyBlockDownloader { + fn protocol_name(&self) -> &ProtocolName { + &self.protocol_name + } + + async fn download_blocks( + &self, + _who: PeerId, + request: BlockRequest, + ) -> Result, ProtocolName), RequestFailure>, Canceled> { + self.sender.send(request).unwrap(); + Ok(Ok((Vec::new(), self.protocol_name.clone()))) + } + + fn block_response_into_blocks( + &self, + _request: &BlockRequest, + _response: Vec, + ) -> Result>, BlockResponseError> { + Ok(Vec::new()) + } +} + +impl ProxyBlockDownloader { + fn new(protocol_name: ProtocolName) -> Self { + let (sender, receiver) = std::sync::mpsc::channel(); + Self { protocol_name, sender, request: Mutex::new(receiver) } + } + + fn next_request(&self) -> BlockRequest { + self.request.lock().unwrap().recv().unwrap() + } +} + #[test] fn processes_empty_response_on_justification_request_for_unknown_block() { // if we ask for a justification for a given block to a peer that doesn't know that block @@ -44,6 +92,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { 1, 64, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) @@ -108,6 +157,7 @@ fn restart_doesnt_affect_peers_downloading_finality_data() { 1, 8, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) @@ -140,13 +190,15 @@ fn restart_doesnt_affect_peers_downloading_finality_data() { sync.add_peer(peer_id1, Hash::random(), 42); sync.add_peer(peer_id2, Hash::random(), 10); + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); + // we wil send block requests to these peers // for these blocks we don't know about - let actions = sync.actions().unwrap(); + let actions = sync.actions(&network_handle).unwrap(); assert_eq!(actions.len(), 2); assert!(actions.iter().all(|action| match action { - SyncingAction::SendBlockRequest { peer_id, .. } => - peer_id == &peer_id1 || peer_id == &peer_id2, + SyncingAction::StartRequest { peer_id, .. } => peer_id == &peer_id1 || peer_id == &peer_id2, _ => false, })); @@ -176,7 +228,7 @@ fn restart_doesnt_affect_peers_downloading_finality_data() { sync.restart(); // which should make us cancel and send out again block requests to the first two peers - let actions = sync.actions().unwrap(); + let actions = sync.actions(&network_handle).unwrap(); assert_eq!(actions.len(), 4); let mut cancelled_first = HashSet::new(); assert!(actions.iter().all(|action| match action { @@ -184,7 +236,7 @@ fn restart_doesnt_affect_peers_downloading_finality_data() { cancelled_first.insert(peer_id); peer_id == &peer_id1 || peer_id == &peer_id2 }, - SyncingAction::SendBlockRequest { peer_id, .. } => { + SyncingAction::StartRequest { peer_id, .. } => { assert!(cancelled_first.remove(peer_id)); peer_id == &peer_id1 || peer_id == &peer_id2 }, @@ -311,6 +363,7 @@ fn do_ancestor_search_when_common_block_to_best_queued_gap_is_to_big() { 5, 64, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) @@ -459,12 +512,16 @@ fn can_sync_huge_fork() { let info = client.info(); + let protocol_name = ProtocolName::Static(""); + let proxy_block_downloader = Arc::new(ProxyBlockDownloader::new(protocol_name.clone())); + let mut sync = ChainSync::new( ChainSyncMode::Full, client.clone(), 5, 64, - ProtocolName::Static(""), + protocol_name, + proxy_block_downloader.clone(), None, std::iter::empty(), ) @@ -494,18 +551,21 @@ fn can_sync_huge_fork() { let block = &fork_blocks[unwrap_from_block_number(request.from.clone()) as usize - 1]; let response = create_block_response(vec![block.clone()]); - sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + sync.on_block_data(&peer_id1, Some(request.clone()), response).unwrap(); - let actions = sync.take_actions().collect::>(); + let mut actions = sync.take_actions().collect::>(); request = if actions.is_empty() { // We found the ancestor break } else { assert_eq!(actions.len(), 1); - match &actions[0] { - SyncingAction::SendBlockRequest { peer_id: _, request, key: _ } => request.clone(), - action @ _ => panic!("Unexpected action: {action:?}"), + match actions.pop().unwrap() { + SyncingAction::StartRequest { request, .. } => { + block_on(request).unwrap().unwrap(); + proxy_block_downloader.next_request() + }, + action => panic!("Unexpected action: {}", action.name()), } }; @@ -600,12 +660,16 @@ fn syncs_fork_without_duplicate_requests() { let info = client.info(); + let protocol_name = ProtocolName::Static(""); + let proxy_block_downloader = Arc::new(ProxyBlockDownloader::new(protocol_name.clone())); + let mut sync = ChainSync::new( ChainSyncMode::Full, client.clone(), 5, 64, - ProtocolName::Static(""), + protocol_name, + proxy_block_downloader.clone(), None, std::iter::empty(), ) @@ -637,16 +701,19 @@ fn syncs_fork_without_duplicate_requests() { sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - let actions = sync.take_actions().collect::>(); + let mut actions = sync.take_actions().collect::>(); request = if actions.is_empty() { // We found the ancestor break } else { assert_eq!(actions.len(), 1); - match &actions[0] { - SyncingAction::SendBlockRequest { peer_id: _, request, key: _ } => request.clone(), - action @ _ => panic!("Unexpected action: {action:?}"), + match actions.pop().unwrap() { + SyncingAction::StartRequest { request, .. } => { + block_on(request).unwrap().unwrap(); + proxy_block_downloader.next_request() + }, + action => panic!("Unexpected action: {}", action.name()), } }; @@ -750,6 +817,7 @@ fn removes_target_fork_on_disconnect() { 1, 64, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) @@ -784,6 +852,7 @@ fn can_import_response_with_missing_blocks() { 1, 64, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) @@ -824,6 +893,7 @@ fn sync_restart_removes_block_but_not_justification_requests() { 1, 64, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) @@ -898,17 +968,17 @@ fn sync_restart_removes_block_but_not_justification_requests() { SyncingAction::CancelRequest { peer_id, key: _ } => { pending_responses.remove(&peer_id); }, - SyncingAction::SendBlockRequest { peer_id, .. } => { + SyncingAction::StartRequest { peer_id, .. } => { // we drop obsolete response, but don't register a new request, it's checked in // the `assert!` below pending_responses.remove(&peer_id); }, - action @ _ => panic!("Unexpected action: {action:?}"), + action @ _ => panic!("Unexpected action: {}", action.name()), } } assert!(actions.iter().any(|action| { match action { - SyncingAction::SendBlockRequest { peer_id, .. } => peer_id == &peers[0], + SyncingAction::StartRequest { peer_id, .. } => peer_id == &peers[0], _ => false, } })); @@ -975,6 +1045,7 @@ fn request_across_forks() { 5, 64, ProtocolName::Static(""), + Arc::new(MockBlockDownloader::new()), None, std::iter::empty(), ) diff --git a/substrate/client/network/sync/src/strategy/polkadot.rs b/substrate/client/network/sync/src/strategy/polkadot.rs new file mode 100644 index 0000000000000..44b05966af064 --- /dev/null +++ b/substrate/client/network/sync/src/strategy/polkadot.rs @@ -0,0 +1,481 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! [`PolkadotSyncingStrategy`] is a proxy between [`crate::engine::SyncingEngine`] +//! and specific syncing algorithms. + +use crate::{ + block_relay_protocol::BlockDownloader, + block_request_handler::MAX_BLOCKS_IN_RESPONSE, + service::network::NetworkServiceHandle, + strategy::{ + chain_sync::{ChainSync, ChainSyncMode}, + state::StateStrategy, + warp::{WarpSync, WarpSyncConfig}, + StrategyKey, SyncingAction, SyncingStrategy, + }, + types::SyncStatus, + LOG_TARGET, +}; +use log::{debug, error, info, warn}; +use prometheus_endpoint::Registry; +use sc_client_api::{BlockBackend, ProofProvider}; +use sc_consensus::{BlockImportError, BlockImportStatus}; +use sc_network::ProtocolName; +use sc_network_common::sync::{message::BlockAnnounce, SyncMode}; +use sc_network_types::PeerId; +use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; +use sp_runtime::traits::{Block as BlockT, Header, NumberFor}; +use std::{any::Any, collections::HashMap, sync::Arc}; + +/// Corresponding `ChainSync` mode. +fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { + match sync_mode { + SyncMode::Full => ChainSyncMode::Full, + SyncMode::LightState { skip_proofs, storage_chain_mode } => + ChainSyncMode::LightState { skip_proofs, storage_chain_mode }, + SyncMode::Warp => ChainSyncMode::Full, + } +} + +/// Syncing configuration containing data for [`PolkadotSyncingStrategy`]. +#[derive(Clone, Debug)] +pub struct PolkadotSyncingStrategyConfig +where + Block: BlockT, +{ + /// Syncing mode. + pub mode: SyncMode, + /// The number of parallel downloads to guard against slow peers. + pub max_parallel_downloads: u32, + /// Maximum number of blocks to request. + pub max_blocks_per_request: u32, + /// Prometheus metrics registry. + pub metrics_registry: Option, + /// Protocol name used to send out state requests + pub state_request_protocol_name: ProtocolName, + /// Block downloader + pub block_downloader: Arc>, +} + +/// Proxy to specific syncing strategies used in Polkadot. +pub struct PolkadotSyncingStrategy { + /// Initial syncing configuration. + config: PolkadotSyncingStrategyConfig, + /// Client used by syncing strategies. + client: Arc, + /// Warp strategy. + warp: Option>, + /// State strategy. + state: Option>, + /// `ChainSync` strategy.` + chain_sync: Option>, + /// Connected peers and their best blocks used to seed a new strategy when switching to it in + /// `PolkadotSyncingStrategy::proceed_to_next`. + peer_best_blocks: HashMap)>, +} + +impl SyncingStrategy for PolkadotSyncingStrategy +where + B: BlockT, + Client: HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, +{ + fn add_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { + self.peer_best_blocks.insert(peer_id, (best_hash, best_number)); + + self.warp.as_mut().map(|s| s.add_peer(peer_id, best_hash, best_number)); + self.state.as_mut().map(|s| s.add_peer(peer_id, best_hash, best_number)); + self.chain_sync.as_mut().map(|s| s.add_peer(peer_id, best_hash, best_number)); + } + + fn remove_peer(&mut self, peer_id: &PeerId) { + self.warp.as_mut().map(|s| s.remove_peer(peer_id)); + self.state.as_mut().map(|s| s.remove_peer(peer_id)); + self.chain_sync.as_mut().map(|s| s.remove_peer(peer_id)); + + self.peer_best_blocks.remove(peer_id); + } + + fn on_validated_block_announce( + &mut self, + is_best: bool, + peer_id: PeerId, + announce: &BlockAnnounce, + ) -> Option<(B::Hash, NumberFor)> { + let new_best = if let Some(ref mut warp) = self.warp { + warp.on_validated_block_announce(is_best, peer_id, announce) + } else if let Some(ref mut state) = self.state { + state.on_validated_block_announce(is_best, peer_id, announce) + } else if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.on_validated_block_announce(is_best, peer_id, announce) + } else { + error!(target: LOG_TARGET, "No syncing strategy is active."); + debug_assert!(false); + Some((announce.header.hash(), *announce.header.number())) + }; + + if let Some(new_best) = new_best { + if let Some(best) = self.peer_best_blocks.get_mut(&peer_id) { + *best = new_best; + } else { + debug!( + target: LOG_TARGET, + "Cannot update `peer_best_blocks` as peer {peer_id} is not known to `Strategy` \ + (already disconnected?)", + ); + } + } + + new_best + } + + fn set_sync_fork_request(&mut self, peers: Vec, hash: &B::Hash, number: NumberFor) { + // Fork requests are only handled by `ChainSync`. + if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.set_sync_fork_request(peers.clone(), hash, number); + } + } + + fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { + // Justifications can only be requested via `ChainSync`. + if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.request_justification(hash, number); + } + } + + fn clear_justification_requests(&mut self) { + // Justification requests can only be cleared by `ChainSync`. + if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.clear_justification_requests(); + } + } + + fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { + // Only `ChainSync` is interested in justification import. + if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.on_justification_import(hash, number, success); + } + } + + fn on_generic_response( + &mut self, + peer_id: &PeerId, + key: StrategyKey, + protocol_name: ProtocolName, + response: Box, + ) { + match key { + StateStrategy::::STRATEGY_KEY => + if let Some(state) = &mut self.state { + let Ok(response) = response.downcast::>() else { + warn!(target: LOG_TARGET, "Failed to downcast state response"); + debug_assert!(false); + return; + }; + + state.on_state_response(peer_id, *response); + } else if let Some(chain_sync) = &mut self.chain_sync { + chain_sync.on_generic_response(peer_id, key, protocol_name, response); + } else { + error!( + target: LOG_TARGET, + "`on_generic_response()` called with unexpected key {key:?} \ + or corresponding strategy is not active.", + ); + debug_assert!(false); + }, + WarpSync::::STRATEGY_KEY => + if let Some(warp) = &mut self.warp { + warp.on_generic_response(peer_id, protocol_name, response); + } else { + error!( + target: LOG_TARGET, + "`on_generic_response()` called with unexpected key {key:?} \ + or warp strategy is not active", + ); + debug_assert!(false); + }, + ChainSync::::STRATEGY_KEY => + if let Some(chain_sync) = &mut self.chain_sync { + chain_sync.on_generic_response(peer_id, key, protocol_name, response); + } else { + error!( + target: LOG_TARGET, + "`on_generic_response()` called with unexpected key {key:?} \ + or corresponding strategy is not active.", + ); + debug_assert!(false); + }, + key => { + warn!( + target: LOG_TARGET, + "Unexpected generic response strategy key {key:?}, protocol {protocol_name}", + ); + debug_assert!(false); + }, + } + } + + fn on_blocks_processed( + &mut self, + imported: usize, + count: usize, + results: Vec<(Result>, BlockImportError>, B::Hash)>, + ) { + // Only `StateStrategy` and `ChainSync` are interested in block processing notifications. + if let Some(ref mut state) = self.state { + state.on_blocks_processed(imported, count, results); + } else if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.on_blocks_processed(imported, count, results); + } + } + + fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { + // Only `ChainSync` is interested in block finalization notifications. + if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.on_block_finalized(hash, number); + } + } + + fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { + // This is relevant to `ChainSync` only. + if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.update_chain_info(best_hash, best_number); + } + } + + fn is_major_syncing(&self) -> bool { + self.warp.is_some() || + self.state.is_some() || + match self.chain_sync { + Some(ref s) => s.status().state.is_major_syncing(), + None => unreachable!("At least one syncing strategy is active; qed"), + } + } + + fn num_peers(&self) -> usize { + self.peer_best_blocks.len() + } + + fn status(&self) -> SyncStatus { + // This function presumes that strategies are executed serially and must be refactored + // once we have parallel strategies. + if let Some(ref warp) = self.warp { + warp.status() + } else if let Some(ref state) = self.state { + state.status() + } else if let Some(ref chain_sync) = self.chain_sync { + chain_sync.status() + } else { + unreachable!("At least one syncing strategy is always active; qed") + } + } + + fn num_downloaded_blocks(&self) -> usize { + self.chain_sync + .as_ref() + .map_or(0, |chain_sync| chain_sync.num_downloaded_blocks()) + } + + fn num_sync_requests(&self) -> usize { + self.chain_sync.as_ref().map_or(0, |chain_sync| chain_sync.num_sync_requests()) + } + + fn actions( + &mut self, + network_service: &NetworkServiceHandle, + ) -> Result>, ClientError> { + // This function presumes that strategies are executed serially and must be refactored once + // we have parallel strategies. + let actions: Vec<_> = if let Some(ref mut warp) = self.warp { + warp.actions(network_service).map(Into::into).collect() + } else if let Some(ref mut state) = self.state { + state.actions(network_service).map(Into::into).collect() + } else if let Some(ref mut chain_sync) = self.chain_sync { + chain_sync.actions(network_service)? + } else { + unreachable!("At least one syncing strategy is always active; qed") + }; + + if actions.iter().any(SyncingAction::is_finished) { + self.proceed_to_next()?; + } + + Ok(actions) + } +} + +impl PolkadotSyncingStrategy +where + B: BlockT, + Client: HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, +{ + /// Initialize a new syncing strategy. + pub fn new( + mut config: PolkadotSyncingStrategyConfig, + client: Arc, + warp_sync_config: Option>, + warp_sync_protocol_name: Option, + ) -> Result { + if config.max_blocks_per_request > MAX_BLOCKS_IN_RESPONSE as u32 { + info!( + target: LOG_TARGET, + "clamping maximum blocks per request to {MAX_BLOCKS_IN_RESPONSE}", + ); + config.max_blocks_per_request = MAX_BLOCKS_IN_RESPONSE as u32; + } + + if let SyncMode::Warp = config.mode { + let warp_sync_config = warp_sync_config + .expect("Warp sync configuration must be supplied in warp sync mode."); + let warp_sync = WarpSync::new( + client.clone(), + warp_sync_config, + warp_sync_protocol_name, + config.block_downloader.clone(), + ); + Ok(Self { + config, + client, + warp: Some(warp_sync), + state: None, + chain_sync: None, + peer_best_blocks: Default::default(), + }) + } else { + let chain_sync = ChainSync::new( + chain_sync_mode(config.mode), + client.clone(), + config.max_parallel_downloads, + config.max_blocks_per_request, + config.state_request_protocol_name.clone(), + config.block_downloader.clone(), + config.metrics_registry.as_ref(), + std::iter::empty(), + )?; + Ok(Self { + config, + client, + warp: None, + state: None, + chain_sync: Some(chain_sync), + peer_best_blocks: Default::default(), + }) + } + } + + /// Proceed with the next strategy if the active one finished. + pub fn proceed_to_next(&mut self) -> Result<(), ClientError> { + // The strategies are switched as `WarpSync` -> `StateStrategy` -> `ChainSync`. + if let Some(ref mut warp) = self.warp { + match warp.take_result() { + Some(res) => { + info!( + target: LOG_TARGET, + "Warp sync is complete, continuing with state sync." + ); + let state_sync = StateStrategy::new( + self.client.clone(), + res.target_header, + res.target_body, + res.target_justifications, + false, + self.peer_best_blocks + .iter() + .map(|(peer_id, (_, best_number))| (*peer_id, *best_number)), + self.config.state_request_protocol_name.clone(), + ); + + self.warp = None; + self.state = Some(state_sync); + Ok(()) + }, + None => { + error!( + target: LOG_TARGET, + "Warp sync failed. Continuing with full sync." + ); + let chain_sync = match ChainSync::new( + chain_sync_mode(self.config.mode), + self.client.clone(), + self.config.max_parallel_downloads, + self.config.max_blocks_per_request, + self.config.state_request_protocol_name.clone(), + self.config.block_downloader.clone(), + self.config.metrics_registry.as_ref(), + self.peer_best_blocks.iter().map(|(peer_id, (best_hash, best_number))| { + (*peer_id, *best_hash, *best_number) + }), + ) { + Ok(chain_sync) => chain_sync, + Err(e) => { + error!(target: LOG_TARGET, "Failed to start `ChainSync`."); + return Err(e) + }, + }; + + self.warp = None; + self.chain_sync = Some(chain_sync); + Ok(()) + }, + } + } else if let Some(state) = &self.state { + if state.is_succeeded() { + info!(target: LOG_TARGET, "State sync is complete, continuing with block sync."); + } else { + error!(target: LOG_TARGET, "State sync failed. Falling back to full sync."); + } + let chain_sync = match ChainSync::new( + chain_sync_mode(self.config.mode), + self.client.clone(), + self.config.max_parallel_downloads, + self.config.max_blocks_per_request, + self.config.state_request_protocol_name.clone(), + self.config.block_downloader.clone(), + self.config.metrics_registry.as_ref(), + self.peer_best_blocks.iter().map(|(peer_id, (best_hash, best_number))| { + (*peer_id, *best_hash, *best_number) + }), + ) { + Ok(chain_sync) => chain_sync, + Err(e) => { + error!(target: LOG_TARGET, "Failed to start `ChainSync`."); + return Err(e); + }, + }; + + self.state = None; + self.chain_sync = Some(chain_sync); + Ok(()) + } else { + unreachable!("Only warp & state strategies can finish; qed") + } + } +} diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index d69ab3e2d535e..1abbb96ccd907 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -19,18 +19,22 @@ //! State sync strategy. use crate::{ - schema::v1::StateResponse, + schema::v1::{StateRequest, StateResponse}, + service::network::NetworkServiceHandle, strategy::{ disconnected_peers::DisconnectedPeers, state_sync::{ImportResult, StateSync, StateSyncProvider}, + StrategyKey, SyncingAction, }, - types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, + types::{BadPeer, SyncState, SyncStatus}, LOG_TARGET, }; +use futures::{channel::oneshot, FutureExt}; use log::{debug, error, trace}; +use prost::Message; use sc_client_api::ProofProvider; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; -use sc_network::ProtocolName; +use sc_network::{IfDisconnected, ProtocolName}; use sc_network_common::sync::message::BlockAnnounce; use sc_network_types::PeerId; use sp_consensus::BlockOrigin; @@ -38,7 +42,7 @@ use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor}, Justifications, SaturatedConversion, }; -use std::{collections::HashMap, sync::Arc}; +use std::{any::Any, collections::HashMap, sync::Arc}; mod rep { use sc_network::ReputationChange as Rep; @@ -50,18 +54,6 @@ mod rep { pub const BAD_STATE: Rep = Rep::new(-(1 << 29), "Bad state"); } -/// Action that should be performed on [`StateStrategy`]'s behalf. -pub enum StateStrategyAction { - /// Send state request to peer. - SendStateRequest { peer_id: PeerId, protocol_name: ProtocolName, request: OpaqueStateRequest }, - /// Disconnect and report peer. - DropPeer(BadPeer), - /// Import blocks. - ImportBlocks { origin: BlockOrigin, blocks: Vec> }, - /// State sync has finished. - Finished, -} - enum PeerState { Available, DownloadingState, @@ -83,12 +75,15 @@ pub struct StateStrategy { state_sync: Box>, peers: HashMap>, disconnected_peers: DisconnectedPeers, - actions: Vec>, + actions: Vec>, protocol_name: ProtocolName, succeeded: bool, } impl StateStrategy { + /// Strategy key used by state sync. + pub const STRATEGY_KEY: StrategyKey = StrategyKey::new("State"); + /// Create a new instance. pub fn new( client: Arc, @@ -123,10 +118,11 @@ impl StateStrategy { } } - // Create a new instance with a custom state sync provider. - // Used in tests. - #[cfg(test)] - fn new_with_provider( + /// Create a new instance with a custom state sync provider. + /// + /// Note: In most cases, users should use [`StateStrategy::new`]. + /// This method is intended for custom sync strategies and advanced use cases. + pub fn new_with_provider( state_sync_provider: Box>, initial_peers: impl Iterator)>, protocol_name: ProtocolName, @@ -157,7 +153,7 @@ impl StateStrategy { if let Some(bad_peer) = self.disconnected_peers.on_disconnect_during_request(*peer_id) { - self.actions.push(StateStrategyAction::DropPeer(bad_peer)); + self.actions.push(SyncingAction::DropPeer(bad_peer)); } } } @@ -185,30 +181,32 @@ impl StateStrategy { } /// Process state response. - pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { - if let Err(bad_peer) = self.on_state_response_inner(peer_id, response) { - self.actions.push(StateStrategyAction::DropPeer(bad_peer)); + pub fn on_state_response(&mut self, peer_id: &PeerId, response: Vec) { + if let Err(bad_peer) = self.on_state_response_inner(peer_id, &response) { + self.actions.push(SyncingAction::DropPeer(bad_peer)); } } fn on_state_response_inner( &mut self, - peer_id: PeerId, - response: OpaqueStateResponse, + peer_id: &PeerId, + response: &[u8], ) -> Result<(), BadPeer> { if let Some(peer) = self.peers.get_mut(&peer_id) { peer.state = PeerState::Available; } - let response: Box = response.0.downcast().map_err(|_error| { - error!( - target: LOG_TARGET, - "Failed to downcast opaque state response, this is an implementation bug." - ); - debug_assert!(false); + let response = match StateResponse::decode(response) { + Ok(response) => response, + Err(error) => { + debug!( + target: LOG_TARGET, + "Failed to decode state response from peer {peer_id:?}: {error:?}.", + ); - BadPeer(peer_id, rep::BAD_RESPONSE) - })?; + return Err(BadPeer(*peer_id, rep::BAD_RESPONSE)); + }, + }; debug!( target: LOG_TARGET, @@ -218,7 +216,7 @@ impl StateStrategy { response.proof.len(), ); - match self.state_sync.import(*response) { + match self.state_sync.import(response) { ImportResult::Import(hash, header, state, body, justifications) => { let origin = BlockOrigin::NetworkInitialSync; let block = IncomingBlock { @@ -234,14 +232,13 @@ impl StateStrategy { state: Some(state), }; debug!(target: LOG_TARGET, "State download is complete. Import is queued"); - self.actions - .push(StateStrategyAction::ImportBlocks { origin, blocks: vec![block] }); + self.actions.push(SyncingAction::ImportBlocks { origin, blocks: vec![block] }); Ok(()) }, ImportResult::Continue => Ok(()), ImportResult::BadResponse => { debug!(target: LOG_TARGET, "Bad state data received from {peer_id}"); - Err(BadPeer(peer_id, rep::BAD_STATE)) + Err(BadPeer(*peer_id, rep::BAD_STATE)) }, } } @@ -281,12 +278,12 @@ impl StateStrategy { ); }); self.succeeded |= results.into_iter().any(|result| result.is_ok()); - self.actions.push(StateStrategyAction::Finished); + self.actions.push(SyncingAction::Finished); } } /// Produce state request. - fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { + fn state_request(&mut self) -> Option<(PeerId, StateRequest)> { if self.state_sync.is_complete() { return None } @@ -307,7 +304,7 @@ impl StateStrategy { target: LOG_TARGET, "New state request to {peer_id}: {request:?}.", ); - Some((peer_id, OpaqueStateRequest(Box::new(request)))) + Some((peer_id, request)) } fn schedule_next_peer( @@ -352,14 +349,33 @@ impl StateStrategy { } } - /// Get actions that should be performed by the owner on [`WarpSync`]'s behalf + /// Get actions that should be performed. #[must_use] - pub fn actions(&mut self) -> impl Iterator> { + pub fn actions( + &mut self, + network_service: &NetworkServiceHandle, + ) -> impl Iterator> { let state_request = self.state_request().into_iter().map(|(peer_id, request)| { - StateStrategyAction::SendStateRequest { + let (tx, rx) = oneshot::channel(); + + network_service.start_request( + peer_id, + self.protocol_name.clone(), + request.encode_to_vec(), + tx, + IfDisconnected::ImmediateError, + ); + + SyncingAction::StartRequest { peer_id, - protocol_name: self.protocol_name.clone(), - request, + key: Self::STRATEGY_KEY, + request: async move { + Ok(rx.await?.and_then(|(response, protocol_name)| { + Ok((Box::new(response) as Box, protocol_name)) + })) + } + .boxed(), + remove_obsolete: false, } }); self.actions.extend(state_request); @@ -379,6 +395,7 @@ mod test { use super::*; use crate::{ schema::v1::{StateRequest, StateResponse}, + service::network::NetworkServiceProvider, strategy::state_sync::{ImportResult, StateSyncProgress, StateSyncProvider}, }; use codec::Decode; @@ -579,8 +596,7 @@ mod test { ProtocolName::Static(""), ); - let (_peer_id, mut opaque_request) = state_strategy.state_request().unwrap(); - let request: &mut StateRequest = opaque_request.0.downcast_mut().unwrap(); + let (_peer_id, request) = state_strategy.state_request().unwrap(); let hash = Hash::decode(&mut &*request.block).unwrap(); assert_eq!(hash, target_block.header().hash()); @@ -631,8 +647,8 @@ mod test { // Manually set the peer's state. state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; - let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); - state_strategy.on_state_response(peer_id, dummy_response); + let dummy_response = StateResponse::default().encode_to_vec(); + state_strategy.on_state_response(&peer_id, dummy_response); assert!(state_strategy.peers.get(&peer_id).unwrap().state.is_available()); } @@ -651,10 +667,10 @@ mod test { ); // Manually set the peer's state. state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; - let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); + let dummy_response = StateResponse::default().encode_to_vec(); // Receiving response drops the peer. assert!(matches!( - state_strategy.on_state_response_inner(peer_id, dummy_response), + state_strategy.on_state_response_inner(&peer_id, &dummy_response), Err(BadPeer(id, _rep)) if id == peer_id, )); } @@ -674,8 +690,8 @@ mod test { // Manually set the peer's state . state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; - let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); - state_strategy.on_state_response(peer_id, dummy_response); + let dummy_response = StateResponse::default().encode_to_vec(); + state_strategy.on_state_response(&peer_id, dummy_response); // No actions generated. assert_eq!(state_strategy.actions.len(), 0) @@ -737,13 +753,13 @@ mod test { state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; // Receive response. - let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); - state_strategy.on_state_response(peer_id, dummy_response); + let dummy_response = StateResponse::default().encode_to_vec(); + state_strategy.on_state_response(&peer_id, dummy_response); assert_eq!(state_strategy.actions.len(), 1); assert!(matches!( &state_strategy.actions[0], - StateStrategyAction::ImportBlocks { origin, blocks } + SyncingAction::ImportBlocks { origin, blocks } if *origin == expected_origin && *blocks == expected_blocks, )); } @@ -799,7 +815,7 @@ mod test { // Strategy finishes. assert_eq!(state_strategy.actions.len(), 1); - assert!(matches!(&state_strategy.actions[0], StateStrategyAction::Finished)); + assert!(matches!(&state_strategy.actions[0], SyncingAction::Finished)); } #[test] @@ -826,7 +842,7 @@ mod test { // Strategy finishes. assert_eq!(state_strategy.actions.len(), 1); - assert!(matches!(&state_strategy.actions[0], StateStrategyAction::Finished)); + assert!(matches!(&state_strategy.actions[0], SyncingAction::Finished)); } #[test] @@ -854,12 +870,15 @@ mod test { )], ); + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); + // Strategy finishes. - let actions = state_strategy.actions().collect::>(); + let actions = state_strategy.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - assert!(matches!(&actions[0], StateStrategyAction::Finished)); + assert!(matches!(&actions[0], SyncingAction::Finished)); // No more actions generated. - assert_eq!(state_strategy.actions().count(), 0); + assert_eq!(state_strategy.actions(&network_handle).count(), 0); } } diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index 0c71dd3c6aeee..673bc1688ecc6 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -21,13 +21,19 @@ pub use sp_consensus_grandpa::{AuthorityList, SetId}; use crate::{ - strategy::{chain_sync::validate_blocks, disconnected_peers::DisconnectedPeers}, + block_relay_protocol::{BlockDownloader, BlockResponseError}, + service::network::NetworkServiceHandle, + strategy::{ + chain_sync::validate_blocks, disconnected_peers::DisconnectedPeers, StrategyKey, + SyncingAction, + }, types::{BadPeer, SyncState, SyncStatus}, LOG_TARGET, }; use codec::{Decode, Encode}; +use futures::{channel::oneshot, FutureExt}; use log::{debug, error, trace, warn}; -use sc_network::ProtocolName; +use sc_network::{IfDisconnected, ProtocolName}; use sc_network_common::sync::message::{ BlockAnnounce, BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, }; @@ -37,7 +43,7 @@ use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor, Zero}, Justifications, SaturatedConversion, }; -use std::{collections::HashMap, fmt, sync::Arc}; +use std::{any::Any, collections::HashMap, fmt, sync::Arc}; /// Number of peers that need to be connected before warp sync is started. const MIN_PEERS_TO_START_WARP_SYNC: usize = 3; @@ -97,6 +103,9 @@ mod rep { /// Reputation change for peers which send us a block which we fail to verify. pub const VERIFICATION_FAIL: Rep = Rep::new(-(1 << 29), "Block verification failed"); + + /// We received a message that failed to decode. + pub const BAD_MESSAGE: Rep = Rep::new(-(1 << 12), "Bad message"); } /// Reported warp sync phase. @@ -186,22 +195,6 @@ struct Peer { state: PeerState, } -/// Action that should be performed on [`WarpSync`]'s behalf. -pub enum WarpSyncAction { - /// Send warp proof request to peer. - SendWarpProofRequest { - peer_id: PeerId, - protocol_name: ProtocolName, - request: WarpProofRequest, - }, - /// Send block request to peer. Always implies dropping a stale block request to the same peer. - SendBlockRequest { peer_id: PeerId, request: BlockRequest }, - /// Disconnect and report peer. - DropPeer(BadPeer), - /// Warp sync has finished. - Finished, -} - pub struct WarpSyncResult { pub target_header: B::Header, pub target_body: Option>, @@ -217,7 +210,8 @@ pub struct WarpSync { peers: HashMap>, disconnected_peers: DisconnectedPeers, protocol_name: Option, - actions: Vec>, + block_downloader: Arc>, + actions: Vec>, result: Option>, } @@ -226,6 +220,9 @@ where B: BlockT, Client: HeaderBackend + 'static, { + /// Strategy key used by warp sync. + pub const STRATEGY_KEY: StrategyKey = StrategyKey::new("Warp"); + /// Create a new instance. When passing a warp sync provider we will be checking for proof and /// authorities. Alternatively we can pass a target block when we want to skip downloading /// proofs, in this case we will continue polling until the target block is known. @@ -233,6 +230,7 @@ where client: Arc, warp_sync_config: WarpSyncConfig, protocol_name: Option, + block_downloader: Arc>, ) -> Self { if client.info().finalized_state.is_some() { error!( @@ -247,7 +245,8 @@ where peers: HashMap::new(), disconnected_peers: DisconnectedPeers::new(), protocol_name, - actions: vec![WarpSyncAction::Finished], + block_downloader, + actions: vec![SyncingAction::Finished], result: None, } } @@ -266,6 +265,7 @@ where peers: HashMap::new(), disconnected_peers: DisconnectedPeers::new(), protocol_name, + block_downloader, actions: Vec::new(), result: None, } @@ -285,7 +285,7 @@ where if let Some(bad_peer) = self.disconnected_peers.on_disconnect_during_request(*peer_id) { - self.actions.push(WarpSyncAction::DropPeer(bad_peer)); + self.actions.push(SyncingAction::DropPeer(bad_peer)); } } } @@ -329,6 +329,58 @@ where trace!(target: LOG_TARGET, "Started warp sync with {} peers.", self.peers.len()); } + pub fn on_generic_response( + &mut self, + peer_id: &PeerId, + protocol_name: ProtocolName, + response: Box, + ) { + if &protocol_name == self.block_downloader.protocol_name() { + let Ok(response) = response + .downcast::<(BlockRequest, Result>, BlockResponseError>)>() + else { + warn!(target: LOG_TARGET, "Failed to downcast block response"); + debug_assert!(false); + return; + }; + + let (request, response) = *response; + let blocks = match response { + Ok(blocks) => blocks, + Err(BlockResponseError::DecodeFailed(e)) => { + debug!( + target: LOG_TARGET, + "Failed to decode block response from peer {:?}: {:?}.", + peer_id, + e + ); + self.actions.push(SyncingAction::DropPeer(BadPeer(*peer_id, rep::BAD_MESSAGE))); + return; + }, + Err(BlockResponseError::ExtractionFailed(e)) => { + debug!( + target: LOG_TARGET, + "Failed to extract blocks from peer response {:?}: {:?}.", + peer_id, + e + ); + self.actions.push(SyncingAction::DropPeer(BadPeer(*peer_id, rep::BAD_MESSAGE))); + return; + }, + }; + + self.on_block_response(*peer_id, request, blocks); + } else { + let Ok(response) = response.downcast::>() else { + warn!(target: LOG_TARGET, "Failed to downcast warp sync response"); + debug_assert!(false); + return; + }; + + self.on_warp_proof_response(peer_id, EncodedProof(*response)); + } + } + /// Process warp proof response. pub fn on_warp_proof_response(&mut self, peer_id: &PeerId, response: EncodedProof) { if let Some(peer) = self.peers.get_mut(peer_id) { @@ -340,7 +392,7 @@ where else { debug!(target: LOG_TARGET, "Unexpected warp proof response"); self.actions - .push(WarpSyncAction::DropPeer(BadPeer(*peer_id, rep::UNEXPECTED_RESPONSE))); + .push(SyncingAction::DropPeer(BadPeer(*peer_id, rep::UNEXPECTED_RESPONSE))); return }; @@ -348,7 +400,7 @@ where Err(e) => { debug!(target: LOG_TARGET, "Bad warp proof response: {}", e); self.actions - .push(WarpSyncAction::DropPeer(BadPeer(*peer_id, rep::BAD_WARP_PROOF))) + .push(SyncingAction::DropPeer(BadPeer(*peer_id, rep::BAD_WARP_PROOF))) }, Ok(VerificationResult::Partial(new_set_id, new_authorities, new_last_hash)) => { log::debug!(target: LOG_TARGET, "Verified partial proof, set_id={:?}", new_set_id); @@ -379,7 +431,7 @@ where blocks: Vec>, ) { if let Err(bad_peer) = self.on_block_response_inner(peer_id, request, blocks) { - self.actions.push(WarpSyncAction::DropPeer(bad_peer)); + self.actions.push(SyncingAction::DropPeer(bad_peer)); } } @@ -449,7 +501,7 @@ where target_justifications: block.justifications, }); self.phase = Phase::Complete; - self.actions.push(WarpSyncAction::Finished); + self.actions.push(SyncingAction::Finished); Ok(()) } @@ -606,17 +658,67 @@ where /// Get actions that should be performed by the owner on [`WarpSync`]'s behalf #[must_use] - pub fn actions(&mut self) -> impl Iterator> { + pub fn actions( + &mut self, + network_service: &NetworkServiceHandle, + ) -> impl Iterator> { let warp_proof_request = self.warp_proof_request().into_iter().map(|(peer_id, protocol_name, request)| { - WarpSyncAction::SendWarpProofRequest { peer_id, protocol_name, request } + trace!( + target: LOG_TARGET, + "Created `WarpProofRequest` to {}, request: {:?}.", + peer_id, + request, + ); + + let (tx, rx) = oneshot::channel(); + + network_service.start_request( + peer_id, + protocol_name, + request.encode(), + tx, + IfDisconnected::ImmediateError, + ); + + SyncingAction::StartRequest { + peer_id, + key: Self::STRATEGY_KEY, + request: async move { + Ok(rx.await?.and_then(|(response, protocol_name)| { + Ok((Box::new(response) as Box, protocol_name)) + })) + } + .boxed(), + remove_obsolete: false, + } }); self.actions.extend(warp_proof_request); - let target_block_request = self - .target_block_request() - .into_iter() - .map(|(peer_id, request)| WarpSyncAction::SendBlockRequest { peer_id, request }); + let target_block_request = + self.target_block_request().into_iter().map(|(peer_id, request)| { + let downloader = self.block_downloader.clone(); + + SyncingAction::StartRequest { + peer_id, + key: Self::STRATEGY_KEY, + request: async move { + Ok(downloader.download_blocks(peer_id, request.clone()).await?.and_then( + |(response, protocol_name)| { + let decoded_response = + downloader.block_response_into_blocks(&request, response); + let result = + Box::new((request, decoded_response)) as Box; + Ok((result, protocol_name)) + }, + )) + } + .boxed(), + // Sending block request implies dropping obsolete pending response as we are + // not interested in it anymore. + remove_obsolete: true, + } + }); self.actions.extend(target_block_request); std::mem::take(&mut self.actions).into_iter() @@ -632,6 +734,7 @@ where #[cfg(test)] mod test { use super::*; + use crate::{mock::MockBlockDownloader, service::network::NetworkServiceProvider}; use sc_block_builder::BlockBuilderBuilder; use sp_blockchain::{BlockStatus, Error as BlockchainError, HeaderBackend, Info}; use sp_consensus_grandpa::{AuthorityList, SetId}; @@ -716,12 +819,16 @@ mod test { let client = mock_client_with_state(); let provider = MockWarpSyncProvider::::new(); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); + + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); // Warp sync instantly finishes - let actions = warp_sync.actions().collect::>(); + let actions = warp_sync.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - assert!(matches!(actions[0], WarpSyncAction::Finished)); + assert!(matches!(actions[0], SyncingAction::Finished)); // ... with no result. assert!(warp_sync.take_result().is_none()); @@ -737,12 +844,16 @@ mod test { Default::default(), Default::default(), )); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); + + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); // Warp sync instantly finishes - let actions = warp_sync.actions().collect::>(); + let actions = warp_sync.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - assert!(matches!(actions[0], WarpSyncAction::Finished)); + assert!(matches!(actions[0], SyncingAction::Finished)); // ... with no result. assert!(warp_sync.take_result().is_none()); @@ -753,10 +864,14 @@ mod test { let client = mock_client_without_state(); let provider = MockWarpSyncProvider::::new(); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); + + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); // No actions are emitted. - assert_eq!(warp_sync.actions().count(), 0) + assert_eq!(warp_sync.actions(&network_handle).count(), 0) } #[test] @@ -769,10 +884,14 @@ mod test { Default::default(), Default::default(), )); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); + + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); // No actions are emitted. - assert_eq!(warp_sync.actions().count(), 0) + assert_eq!(warp_sync.actions(&network_handle).count(), 0) } #[test] @@ -784,7 +903,8 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); // Warp sync is not started when there is not enough peers. for _ in 0..(MIN_PEERS_TO_START_WARP_SYNC - 1) { @@ -802,7 +922,8 @@ mod test { let client = mock_client_without_state(); let provider = MockWarpSyncProvider::::new(); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); assert!(warp_sync.schedule_next_peer(PeerState::DownloadingProofs, None).is_none()); } @@ -826,7 +947,8 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); for best_number in 1..11 { warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); @@ -847,7 +969,8 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); for best_number in 1..11 { warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); @@ -867,7 +990,8 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); for best_number in 1..11 { warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); @@ -911,7 +1035,12 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, Some(ProtocolName::Static(""))); + let mut warp_sync = WarpSync::new( + Arc::new(client), + config, + Some(ProtocolName::Static("")), + Arc::new(MockBlockDownloader::new()), + ); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -940,7 +1069,12 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, Some(ProtocolName::Static(""))); + let mut warp_sync = WarpSync::new( + Arc::new(client), + config, + Some(ProtocolName::Static("")), + Arc::new(MockBlockDownloader::new()), + ); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -971,7 +1105,12 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, Some(ProtocolName::Static(""))); + let mut warp_sync = WarpSync::new( + Arc::new(client), + config, + Some(ProtocolName::Static("")), + Arc::new(MockBlockDownloader::new()), + ); // Make sure we have enough peers to make requests. for best_number in 1..11 { @@ -998,7 +1137,12 @@ mod test { Err(Box::new(std::io::Error::new(ErrorKind::Other, "test-verification-failure"))) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, Some(ProtocolName::Static(""))); + let mut warp_sync = WarpSync::new( + Arc::new(client), + config, + Some(ProtocolName::Static("")), + Arc::new(MockBlockDownloader::new()), + ); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1006,11 +1150,13 @@ mod test { } assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); + // Consume `SendWarpProofRequest` action. - let actions = warp_sync.actions().collect::>(); + let actions = warp_sync.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] - else { + let SyncingAction::StartRequest { peer_id: request_peer_id, .. } = actions[0] else { panic!("Invalid action"); }; @@ -1021,7 +1167,7 @@ mod test { assert_eq!(actions.len(), 1); assert!(matches!( actions[0], - WarpSyncAction::DropPeer(BadPeer(peer_id, _rep)) if peer_id == request_peer_id + SyncingAction::DropPeer(BadPeer(peer_id, _rep)) if peer_id == request_peer_id )); assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); } @@ -1039,7 +1185,12 @@ mod test { Ok(VerificationResult::Partial(set_id, authorities, Hash::random())) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, Some(ProtocolName::Static(""))); + let mut warp_sync = WarpSync::new( + Arc::new(client), + config, + Some(ProtocolName::Static("")), + Arc::new(MockBlockDownloader::new()), + ); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1047,11 +1198,13 @@ mod test { } assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); + // Consume `SendWarpProofRequest` action. - let actions = warp_sync.actions().collect::>(); + let actions = warp_sync.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] - else { + let SyncingAction::StartRequest { peer_id: request_peer_id, .. } = actions[0] else { panic!("Invalid action"); }; @@ -1083,7 +1236,12 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, Some(ProtocolName::Static(""))); + let mut warp_sync = WarpSync::new( + client, + config, + Some(ProtocolName::Static("")), + Arc::new(MockBlockDownloader::new()), + ); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1091,11 +1249,13 @@ mod test { } assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); + // Consume `SendWarpProofRequest` action. - let actions = warp_sync.actions().collect::>(); + let actions = warp_sync.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] - else { + let SyncingAction::StartRequest { peer_id: request_peer_id, .. } = actions[0] else { panic!("Invalid action."); }; @@ -1116,7 +1276,8 @@ mod test { .once() .return_const(AuthorityList::default()); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(Arc::new(client), config, None); + let mut warp_sync = + WarpSync::new(Arc::new(client), config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1151,7 +1312,8 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1183,7 +1345,8 @@ mod test { .block; let target_header = target_block.header().clone(); let config = WarpSyncConfig::WithTarget(target_header); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1223,7 +1386,8 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1261,7 +1425,8 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1315,7 +1480,8 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1392,7 +1558,8 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1445,7 +1612,8 @@ mod test { Ok(VerificationResult::Complete(set_id, authorities, target_header)) }); let config = WarpSyncConfig::WithProvider(Arc::new(provider)); - let mut warp_sync = WarpSync::new(client, config, None); + let mut warp_sync = + WarpSync::new(client, config, None, Arc::new(MockBlockDownloader::new())); // Make sure we have enough peers to make a request. for best_number in 1..11 { @@ -1473,10 +1641,13 @@ mod test { assert!(warp_sync.on_block_response_inner(peer_id, request, response).is_ok()); + let network_provider = NetworkServiceProvider::new(); + let network_handle = network_provider.handle(); + // Strategy finishes. - let actions = warp_sync.actions().collect::>(); + let actions = warp_sync.actions(&network_handle).collect::>(); assert_eq!(actions.len(), 1); - assert!(matches!(actions[0], WarpSyncAction::Finished)); + assert!(matches!(actions[0], SyncingAction::Finished)); // With correct result. let result = warp_sync.take_result().unwrap(); diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index c3403fe1e5f75..5745a34378df6 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -23,11 +23,10 @@ use sc_network_common::{role::Roles, types::ReputationChange}; use crate::strategy::{state_sync::StateSyncProgress, warp::WarpSyncProgress}; -use sc_network_common::sync::message::BlockRequest; use sc_network_types::PeerId; use sp_runtime::traits::{Block as BlockT, NumberFor}; -use std::{any::Any, fmt, fmt::Formatter, pin::Pin, sync::Arc}; +use std::{fmt, pin::Pin, sync::Arc}; /// The sync status of a peer we are trying to sync with #[derive(Debug)] @@ -107,52 +106,6 @@ impl fmt::Display for BadPeer { impl std::error::Error for BadPeer {} -#[derive(Debug)] -pub enum PeerRequest { - Block(BlockRequest), - State, - WarpProof, -} - -#[derive(Debug)] -pub enum PeerRequestType { - Block, - State, - WarpProof, -} - -impl PeerRequest { - pub fn get_type(&self) -> PeerRequestType { - match self { - PeerRequest::Block(_) => PeerRequestType::Block, - PeerRequest::State => PeerRequestType::State, - PeerRequest::WarpProof => PeerRequestType::WarpProof, - } - } -} - -/// Wrapper for implementation-specific state request. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueStateRequest(pub Box); - -impl fmt::Debug for OpaqueStateRequest { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueStateRequest").finish() - } -} - -/// Wrapper for implementation-specific state response. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueStateResponse(pub Box); - -impl fmt::Debug for OpaqueStateResponse { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueStateResponse").finish() - } -} - /// Provides high-level status of syncing. #[async_trait::async_trait] pub trait SyncStatusProvider: Send + Sync { diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index 06e243342fb2d..825481314c672 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -67,11 +67,11 @@ use sc_network_sync::{ service::{network::NetworkServiceProvider, syncing_service::SyncingService}, state_request_handler::StateRequestHandler, strategy::{ + polkadot::{PolkadotSyncingStrategy, PolkadotSyncingStrategyConfig}, warp::{ AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncConfig, WarpSyncProvider, }, - PolkadotSyncingStrategy, SyncingConfig, }, warp_request_handler, }; @@ -833,8 +833,8 @@ pub trait TestNetFactory: Default + Sized + Send { let fork_id = Some(String::from("test-fork-id")); - let (chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); + let chain_sync_network_provider = NetworkServiceProvider::new(); + let chain_sync_network_handle = chain_sync_network_provider.handle(); let mut block_relay_params = BlockRequestHandler::new::>( chain_sync_network_handle.clone(), &protocol_id, @@ -908,12 +908,13 @@ pub trait TestNetFactory: Default + Sized + Send { ::Hash, >>::register_notification_metrics(None); - let syncing_config = SyncingConfig { + let syncing_config = PolkadotSyncingStrategyConfig { mode: network_config.sync_mode, max_parallel_downloads: network_config.max_parallel_downloads, max_blocks_per_request: network_config.max_blocks_per_request, metrics_registry: None, state_request_protocol_name: state_request_protocol_config.name.clone(), + block_downloader: block_relay_params.downloader, }; // Initialize syncing strategy. let syncing_strategy = Box::new( @@ -934,16 +935,14 @@ pub trait TestNetFactory: Default + Sized + Send { metrics, &full_net_config, protocol_id.clone(), - &fork_id, + fork_id.as_deref(), block_announce_validator, syncing_strategy, chain_sync_network_handle, import_queue.service(), - block_relay_params.downloader, peer_store_handle.clone(), ) .unwrap(); - let sync_service_import_queue = Box::new(sync_service.clone()); let sync_service = Arc::new(sync_service.clone()); for config in config.request_response_protocols { @@ -987,8 +986,12 @@ pub trait TestNetFactory: Default + Sized + Send { chain_sync_network_provider.run(service).await; }); - tokio::spawn(async move { - import_queue.run(sync_service_import_queue).await; + tokio::spawn({ + let sync_service = sync_service.clone(); + + async move { + import_queue.run(sync_service.as_ref()).await; + } }); tokio::spawn(async move { diff --git a/substrate/client/network/test/src/service.rs b/substrate/client/network/test/src/service.rs index ad2d1d9ec24de..688b569c32228 100644 --- a/substrate/client/network/test/src/service.rs +++ b/substrate/client/network/test/src/service.rs @@ -32,9 +32,9 @@ use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ block_request_handler::BlockRequestHandler, engine::SyncingEngine, - service::network::{NetworkServiceHandle, NetworkServiceProvider}, + service::network::NetworkServiceProvider, state_request_handler::StateRequestHandler, - strategy::{PolkadotSyncingStrategy, SyncingConfig}, + strategy::polkadot::{PolkadotSyncingStrategy, PolkadotSyncingStrategyConfig}, }; use sp_blockchain::HeaderBackend; use sp_runtime::traits::{Block as BlockT, Zero}; @@ -78,7 +78,7 @@ struct TestNetworkBuilder { client: Option>, listen_addresses: Vec, set_config: Option, - chain_sync_network: Option<(NetworkServiceProvider, NetworkServiceHandle)>, + chain_sync_network: Option, notification_protocols: Vec, config: Option, } @@ -157,8 +157,9 @@ impl TestNetworkBuilder { let fork_id = Some(String::from("test-fork-id")); let mut full_net_config = FullNetworkConfiguration::new(&network_config, None); - let (chain_sync_network_provider, chain_sync_network_handle) = + let chain_sync_network_provider = self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); + let chain_sync_network_handle = chain_sync_network_provider.handle(); let mut block_relay_params = BlockRequestHandler::new::< NetworkWorker< @@ -203,12 +204,13 @@ impl TestNetworkBuilder { let peer_store_handle: Arc = Arc::new(peer_store.handle()); tokio::spawn(peer_store.run().boxed()); - let syncing_config = SyncingConfig { + let syncing_config = PolkadotSyncingStrategyConfig { mode: network_config.sync_mode, max_parallel_downloads: network_config.max_parallel_downloads, max_blocks_per_request: network_config.max_blocks_per_request, metrics_registry: None, state_request_protocol_name: state_request_protocol_config.name.clone(), + block_downloader: block_relay_params.downloader, }; // Initialize syncing strategy. let syncing_strategy = Box::new( @@ -222,12 +224,11 @@ impl TestNetworkBuilder { NotificationMetrics::new(None), &full_net_config, protocol_id.clone(), - &None, + None, Box::new(sp_consensus::block_validation::DefaultBlockAnnounceValidator), syncing_strategy, chain_sync_network_handle, import_queue.service(), - block_relay_params.downloader, Arc::clone(&peer_store_handle), ) .unwrap(); diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index f27b7ec6fbad8..ce4ce7c082483 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -35,7 +35,7 @@ use sc_client_api::{ BlockBackend, BlockchainEvents, ExecutorProvider, ForkBlocks, StorageProvider, UsageProvider, }; use sc_client_db::{Backend, BlocksPruning, DatabaseSettings, PruningMode}; -use sc_consensus::import_queue::ImportQueue; +use sc_consensus::import_queue::{ImportQueue, ImportQueueService}; use sc_executor::{ sp_wasm_interface::HostFunctions, HeapAllocStrategy, NativeExecutionDispatch, RuntimeVersionOf, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY, @@ -50,15 +50,18 @@ use sc_network::{ }, NetworkBackend, NetworkStateInfo, }; -use sc_network_common::role::Roles; +use sc_network_common::role::{Role, Roles}; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ - block_relay_protocol::BlockRelayParams, + block_relay_protocol::{BlockDownloader, BlockRelayParams}, block_request_handler::BlockRequestHandler, engine::SyncingEngine, - service::network::NetworkServiceProvider, + service::network::{NetworkServiceHandle, NetworkServiceProvider}, state_request_handler::StateRequestHandler, - strategy::{PolkadotSyncingStrategy, SyncingConfig, SyncingStrategy}, + strategy::{ + polkadot::{PolkadotSyncingStrategy, PolkadotSyncingStrategyConfig}, + SyncingStrategy, + }, warp_request_handler::RequestHandler as WarpSyncRequestHandler, SyncingService, WarpSyncConfig, }; @@ -780,7 +783,7 @@ where Ok(rpc_api) } -/// Parameters to pass into `build_network`. +/// Parameters to pass into [`build_network`]. pub struct BuildNetworkParams<'a, Block, Net, TxPool, IQ, Client> where Block: BlockT, @@ -802,8 +805,8 @@ where pub block_announce_validator_builder: Option< Box) -> Box + Send> + Send>, >, - /// Syncing strategy to use in syncing engine. - pub syncing_strategy: Box>, + /// Optional warp sync config. + pub warp_sync_config: Option>, /// User specified block relay params. If not specified, the default /// block request handler will be used. pub block_relay: Option>, @@ -847,100 +850,217 @@ where spawn_handle, import_queue, block_announce_validator_builder, - syncing_strategy, + warp_sync_config, block_relay, metrics, } = params; - let protocol_id = config.protocol_id(); - let genesis_hash = client.info().genesis_hash; - let block_announce_validator = if let Some(f) = block_announce_validator_builder { f(client.clone()) } else { Box::new(DefaultBlockAnnounceValidator) }; - let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut block_server, block_downloader, block_request_protocol_config) = match block_relay { - Some(params) => (params.server, params.downloader, params.request_response_config), - None => { - // Custom protocol was not specified, use the default block handler. - // Allow both outgoing and incoming requests. - let params = BlockRequestHandler::new::( - chain_sync_network_handle.clone(), - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - config.network.default_peers_set.in_peers as usize + - config.network.default_peers_set.out_peers as usize, - ); - (params.server, params.downloader, params.request_response_config) + let network_service_provider = NetworkServiceProvider::new(); + let protocol_id = config.protocol_id(); + let fork_id = config.chain_spec.fork_id(); + let metrics_registry = config.prometheus_config.as_ref().map(|config| &config.registry); + + let block_downloader = match block_relay { + Some(params) => { + let BlockRelayParams { mut server, downloader, request_response_config } = params; + + net_config.add_request_response_protocol(request_response_config); + + spawn_handle.spawn("block-request-handler", Some("networking"), async move { + server.run().await; + }); + + downloader }, + None => build_default_block_downloader( + &protocol_id, + fork_id, + &mut net_config, + network_service_provider.handle(), + Arc::clone(&client), + config.network.default_peers_set.in_peers as usize + + config.network.default_peers_set.out_peers as usize, + &spawn_handle, + ), }; - spawn_handle.spawn("block-request-handler", Some("networking"), async move { - block_server.run().await; - }); + + let syncing_strategy = build_polkadot_syncing_strategy( + protocol_id.clone(), + fork_id, + &mut net_config, + warp_sync_config, + block_downloader, + client.clone(), + &spawn_handle, + metrics_registry, + )?; + + let (syncing_engine, sync_service, block_announce_config) = SyncingEngine::new( + Roles::from(&config.role), + Arc::clone(&client), + metrics_registry, + metrics.clone(), + &net_config, + protocol_id.clone(), + fork_id, + block_announce_validator, + syncing_strategy, + network_service_provider.handle(), + import_queue.service(), + net_config.peer_store_handle(), + )?; + + spawn_handle.spawn_blocking("syncing", None, syncing_engine.run()); + + build_network_advanced(BuildNetworkAdvancedParams { + role: config.role, + protocol_id, + fork_id, + ipfs_server: config.network.ipfs_server, + announce_block: config.announce_block, + net_config, + client, + transaction_pool, + spawn_handle, + import_queue, + sync_service, + block_announce_config, + network_service_provider, + metrics_registry, + metrics, + }) +} + +/// Parameters to pass into [`build_network_advanced`]. +pub struct BuildNetworkAdvancedParams<'a, Block, Net, TxPool, IQ, Client> +where + Block: BlockT, + Net: NetworkBackend::Hash>, +{ + /// Role of the local node. + pub role: Role, + /// Protocol name prefix. + pub protocol_id: ProtocolId, + /// Fork ID. + pub fork_id: Option<&'a str>, + /// Enable serving block data over IPFS bitswap. + pub ipfs_server: bool, + /// Announce block automatically after they have been imported. + pub announce_block: bool, + /// Full network configuration. + pub net_config: FullNetworkConfiguration::Hash, Net>, + /// A shared client returned by `new_full_parts`. + pub client: Arc, + /// A shared transaction pool. + pub transaction_pool: Arc, + /// A handle for spawning tasks. + pub spawn_handle: SpawnTaskHandle, + /// An import queue. + pub import_queue: IQ, + /// Syncing service to communicate with syncing engine. + pub sync_service: SyncingService, + /// Block announce config. + pub block_announce_config: Net::NotificationProtocolConfig, + /// Network service provider to drive with network internally. + pub network_service_provider: NetworkServiceProvider, + /// Prometheus metrics registry. + pub metrics_registry: Option<&'a Registry>, + /// Metrics. + pub metrics: NotificationMetrics, +} + +/// Build the network service, the network status sinks and an RPC sender, this is a lower-level +/// version of [`build_network`] for those needing more control. +pub fn build_network_advanced( + params: BuildNetworkAdvancedParams, +) -> Result< + ( + Arc, + TracingUnboundedSender>, + sc_network_transactions::TransactionsHandlerController<::Hash>, + NetworkStarter, + Arc>, + ), + Error, +> +where + Block: BlockT, + Client: ProvideRuntimeApi + + HeaderMetadata + + Chain + + BlockBackend + + BlockIdTo + + ProofProvider + + HeaderBackend + + BlockchainEvents + + 'static, + TxPool: TransactionPool::Hash> + 'static, + IQ: ImportQueue + 'static, + Net: NetworkBackend::Hash>, +{ + let BuildNetworkAdvancedParams { + role, + protocol_id, + fork_id, + ipfs_server, + announce_block, + mut net_config, + client, + transaction_pool, + spawn_handle, + import_queue, + sync_service, + block_announce_config, + network_service_provider, + metrics_registry, + metrics, + } = params; + + let genesis_hash = client.info().genesis_hash; let light_client_request_protocol_config = { // Allow both outgoing and incoming requests. - let (handler, protocol_config) = LightClientRequestHandler::new::( - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - ); + let (handler, protocol_config) = + LightClientRequestHandler::new::(&protocol_id, fork_id, client.clone()); spawn_handle.spawn("light-client-request-handler", Some("networking"), handler.run()); protocol_config }; // install request handlers to `FullNetworkConfiguration` - net_config.add_request_response_protocol(block_request_protocol_config); net_config.add_request_response_protocol(light_client_request_protocol_config); - let bitswap_config = config.network.ipfs_server.then(|| { + let bitswap_config = ipfs_server.then(|| { let (handler, config) = Net::bitswap_server(client.clone()); spawn_handle.spawn("bitswap-request-handler", Some("networking"), handler); config }); - // create transactions protocol and add it to the list of supported protocols of - let peer_store_handle = net_config.peer_store_handle(); + // Create transactions protocol and add it to the list of supported protocols of let (transactions_handler_proto, transactions_config) = sc_network_transactions::TransactionsHandlerPrototype::new::<_, Block, Net>( protocol_id.clone(), genesis_hash, - config.chain_spec.fork_id(), + fork_id, metrics.clone(), - Arc::clone(&peer_store_handle), + net_config.peer_store_handle(), ); net_config.add_notification_protocol(transactions_config); // Start task for `PeerStore` let peer_store = net_config.take_peer_store(); - let peer_store_handle = peer_store.handle(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - let (engine, sync_service, block_announce_config) = SyncingEngine::new( - Roles::from(&config.role), - client.clone(), - config.prometheus_config.as_ref().map(|config| config.registry.clone()).as_ref(), - metrics.clone(), - &net_config, - protocol_id.clone(), - &config.chain_spec.fork_id().map(ToOwned::to_owned), - block_announce_validator, - syncing_strategy, - chain_sync_network_handle, - import_queue.service(), - block_downloader, - Arc::clone(&peer_store_handle), - )?; - let sync_service_import_queue = sync_service.clone(); let sync_service = Arc::new(sync_service); let network_params = sc_network::config::Params::::Hash, Net> { - role: config.role, + role, executor: { let spawn_handle = Clone::clone(&spawn_handle); Box::new(move |fut| { @@ -950,8 +1070,8 @@ where network_config: net_config, genesis_hash, protocol_id, - fork_id: config.chain_spec.fork_id().map(ToOwned::to_owned), - metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), + fork_id: fork_id.map(ToOwned::to_owned), + metrics_registry: metrics_registry.cloned(), block_announce_config, bitswap_config, notification_metrics: metrics, @@ -965,7 +1085,7 @@ where network.clone(), sync_service.clone(), Arc::new(TransactionPoolAdapter { pool: transaction_pool, client: client.clone() }), - config.prometheus_config.as_ref().map(|config| &config.registry), + metrics_registry, )?; spawn_handle.spawn_blocking( "network-transactions-handler", @@ -976,17 +1096,20 @@ where spawn_handle.spawn_blocking( "chain-sync-network-service-provider", Some("networking"), - chain_sync_network_provider.run(Arc::new(network.clone())), + network_service_provider.run(Arc::new(network.clone())), ); - spawn_handle.spawn("import-queue", None, import_queue.run(Box::new(sync_service_import_queue))); - spawn_handle.spawn_blocking("syncing", None, engine.run()); + spawn_handle.spawn("import-queue", None, { + let sync_service = sync_service.clone(); + + async move { import_queue.run(sync_service.as_ref()).await } + }); let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc", 10_000); spawn_handle.spawn( "system-rpc-handler", Some("networking"), build_system_rpc_future::<_, _, ::Hash>( - config.role, + role, network_mut.network_service(), sync_service.clone(), client.clone(), @@ -999,7 +1122,7 @@ where network_mut, client, sync_service.clone(), - config.announce_block, + announce_block, ); // TODO: Normally, one is supposed to pass a list of notifications protocols supported by the @@ -1047,12 +1170,154 @@ where )) } +/// Configuration for [`build_default_syncing_engine`]. +pub struct DefaultSyncingEngineConfig<'a, Block, Client, Net> +where + Block: BlockT, + Net: NetworkBackend::Hash>, +{ + /// Role of the local node. + pub role: Role, + /// Protocol name prefix. + pub protocol_id: ProtocolId, + /// Fork ID. + pub fork_id: Option<&'a str>, + /// Full network configuration. + pub net_config: &'a mut FullNetworkConfiguration::Hash, Net>, + /// Validator for incoming block announcements. + pub block_announce_validator: Box + Send>, + /// Handle to communicate with `NetworkService`. + pub network_service_handle: NetworkServiceHandle, + /// Warp sync configuration (when used). + pub warp_sync_config: Option>, + /// A shared client returned by `new_full_parts`. + pub client: Arc, + /// Blocks import queue API. + pub import_queue_service: Box>, + /// Expected max total number of peer connections (in + out). + pub num_peers_hint: usize, + /// A handle for spawning tasks. + pub spawn_handle: &'a SpawnTaskHandle, + /// Prometheus metrics registry. + pub metrics_registry: Option<&'a Registry>, + /// Metrics. + pub metrics: NotificationMetrics, +} + +/// Build default syncing engine using [`build_default_block_downloader`] and +/// [`build_polkadot_syncing_strategy`] internally. +pub fn build_default_syncing_engine( + config: DefaultSyncingEngineConfig, +) -> Result<(SyncingService, Net::NotificationProtocolConfig), Error> +where + Block: BlockT, + Client: HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, + Net: NetworkBackend::Hash>, +{ + let DefaultSyncingEngineConfig { + role, + protocol_id, + fork_id, + net_config, + block_announce_validator, + network_service_handle, + warp_sync_config, + client, + import_queue_service, + num_peers_hint, + spawn_handle, + metrics_registry, + metrics, + } = config; + + let block_downloader = build_default_block_downloader( + &protocol_id, + fork_id, + net_config, + network_service_handle.clone(), + client.clone(), + num_peers_hint, + spawn_handle, + ); + let syncing_strategy = build_polkadot_syncing_strategy( + protocol_id.clone(), + fork_id, + net_config, + warp_sync_config, + block_downloader, + client.clone(), + spawn_handle, + metrics_registry, + )?; + + let (syncing_engine, sync_service, block_announce_config) = SyncingEngine::new( + Roles::from(&role), + client, + metrics_registry, + metrics, + &net_config, + protocol_id, + fork_id, + block_announce_validator, + syncing_strategy, + network_service_handle, + import_queue_service, + net_config.peer_store_handle(), + )?; + + spawn_handle.spawn_blocking("syncing", None, syncing_engine.run()); + + Ok((sync_service, block_announce_config)) +} + +/// Build default block downloader +pub fn build_default_block_downloader( + protocol_id: &ProtocolId, + fork_id: Option<&str>, + net_config: &mut FullNetworkConfiguration::Hash, Net>, + network_service_handle: NetworkServiceHandle, + client: Arc, + num_peers_hint: usize, + spawn_handle: &SpawnTaskHandle, +) -> Arc> +where + Block: BlockT, + Client: HeaderBackend + BlockBackend + Send + Sync + 'static, + Net: NetworkBackend::Hash>, +{ + // Custom protocol was not specified, use the default block handler. + // Allow both outgoing and incoming requests. + let BlockRelayParams { mut server, downloader, request_response_config } = + BlockRequestHandler::new::( + network_service_handle, + &protocol_id, + fork_id, + client.clone(), + num_peers_hint, + ); + + spawn_handle.spawn("block-request-handler", Some("networking"), async move { + server.run().await; + }); + + net_config.add_request_response_protocol(request_response_config); + + downloader +} + /// Build standard polkadot syncing strategy pub fn build_polkadot_syncing_strategy( protocol_id: ProtocolId, fork_id: Option<&str>, net_config: &mut FullNetworkConfiguration::Hash, Net>, warp_sync_config: Option>, + block_downloader: Arc>, client: Arc, spawn_handle: &SpawnTaskHandle, metrics_registry: Option<&Registry>, @@ -1066,7 +1331,6 @@ where + Send + Sync + 'static, - Net: NetworkBackend::Hash>, { if warp_sync_config.is_none() && net_config.network_config.sync_mode.is_warp() { @@ -1117,12 +1381,13 @@ where net_config.add_request_response_protocol(config); } - let syncing_config = SyncingConfig { + let syncing_config = PolkadotSyncingStrategyConfig { mode: net_config.network_config.sync_mode, max_parallel_downloads: net_config.network_config.max_parallel_downloads, max_blocks_per_request: net_config.network_config.max_blocks_per_request, metrics_registry: metrics_registry.cloned(), state_request_protocol_name, + block_downloader, }; Ok(Box::new(PolkadotSyncingStrategy::new( syncing_config, diff --git a/substrate/client/service/src/chain_ops/import_blocks.rs b/substrate/client/service/src/chain_ops/import_blocks.rs index 661fc09a8f19e..8e759faa0775d 100644 --- a/substrate/client/service/src/chain_ops/import_blocks.rs +++ b/substrate/client/service/src/chain_ops/import_blocks.rs @@ -37,6 +37,10 @@ use sp_runtime::{ use std::{ io::Read, pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, task::Poll, time::{Duration, Instant}, }; @@ -50,8 +54,6 @@ const DELAY_TIME: u64 = 200; /// Number of milliseconds that must have passed between two updates. const TIME_BETWEEN_UPDATES: u64 = 3_000; -use std::sync::Arc; - /// Build a chain spec json pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result { spec.as_json(raw).map_err(Into::into) @@ -301,29 +303,29 @@ where IQ: ImportQueue + 'static, { struct WaitLink { - imported_blocks: u64, - has_error: bool, + imported_blocks: AtomicU64, + has_error: AtomicBool, } impl WaitLink { fn new() -> WaitLink { - WaitLink { imported_blocks: 0, has_error: false } + WaitLink { imported_blocks: AtomicU64::new(0), has_error: AtomicBool::new(false) } } } impl Link for WaitLink { fn blocks_processed( - &mut self, + &self, imported: usize, _num_expected_blocks: usize, results: Vec<(Result>, BlockImportError>, B::Hash)>, ) { - self.imported_blocks += imported as u64; + self.imported_blocks.fetch_add(imported as u64, Ordering::AcqRel); for result in results { if let (Err(err), hash) = result { warn!("There was an error importing block with hash {:?}: {}", hash, err); - self.has_error = true; + self.has_error.store(true, Ordering::Release); break } } @@ -373,7 +375,9 @@ where let read_block_count = block_iter.read_block_count(); match block_result { Ok(block) => { - if read_block_count - link.imported_blocks >= MAX_PENDING_BLOCKS { + if read_block_count - link.imported_blocks.load(Ordering::Acquire) >= + MAX_PENDING_BLOCKS + { // The queue is full, so do not add this block and simply wait // until the queue has made some progress. let delay = Delay::new(Duration::from_millis(DELAY_TIME)); @@ -399,7 +403,9 @@ where }, ImportState::WaitingForImportQueueToCatchUp { block_iter, mut delay, block } => { let read_block_count = block_iter.read_block_count(); - if read_block_count - link.imported_blocks >= MAX_PENDING_BLOCKS { + if read_block_count - link.imported_blocks.load(Ordering::Acquire) >= + MAX_PENDING_BLOCKS + { // Queue is still full, so wait until there is room to insert our block. match Pin::new(&mut delay).poll(cx) { Poll::Pending => { @@ -433,7 +439,11 @@ where } => { // All the blocks have been added to the queue, which doesn't mean they // have all been properly imported. - if importing_is_done(num_expected_blocks, read_block_count, link.imported_blocks) { + if importing_is_done( + num_expected_blocks, + read_block_count, + link.imported_blocks.load(Ordering::Acquire), + ) { // Importing is done, we can log the result and return. info!( "🎉 Imported {} blocks. Best: #{}", @@ -472,10 +482,10 @@ where let best_number = client.info().best_number; speedometer.notify_user(best_number); - if link.has_error { + if link.has_error.load(Ordering::Acquire) { return Poll::Ready(Err(Error::Other(format!( "Stopping after #{} blocks because of an error", - link.imported_blocks + link.imported_blocks.load(Ordering::Acquire) )))) } diff --git a/substrate/client/service/src/lib.rs b/substrate/client/service/src/lib.rs index 3df9020b04180..ee4f4e7622e74 100644 --- a/substrate/client/service/src/lib.rs +++ b/substrate/client/service/src/lib.rs @@ -59,11 +59,13 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; pub use self::{ builder::{ - build_network, build_polkadot_syncing_strategy, gen_rpc_module, init_telemetry, new_client, - new_db_backend, new_full_client, new_full_parts, new_full_parts_record_import, + build_default_block_downloader, build_default_syncing_engine, build_network, + build_network_advanced, build_polkadot_syncing_strategy, gen_rpc_module, init_telemetry, + new_client, new_db_backend, new_full_client, new_full_parts, new_full_parts_record_import, new_full_parts_with_genesis_builder, new_wasm_executor, - propagate_transaction_notifications, spawn_tasks, BuildNetworkParams, KeystoreContainer, - NetworkStarter, SpawnTasksParams, TFullBackend, TFullCallExecutor, TFullClient, + propagate_transaction_notifications, spawn_tasks, BuildNetworkAdvancedParams, + BuildNetworkParams, DefaultSyncingEngineConfig, KeystoreContainer, NetworkStarter, + SpawnTasksParams, TFullBackend, TFullCallExecutor, TFullClient, }, client::{ClientConfig, LocalCallExecutor}, error::Error, diff --git a/substrate/frame/election-provider-support/benchmarking/src/inner.rs b/substrate/frame/election-provider-support/benchmarking/src/inner.rs index 8cca0d459eac3..7fb8c1bdb7290 100644 --- a/substrate/frame/election-provider-support/benchmarking/src/inner.rs +++ b/substrate/frame/election-provider-support/benchmarking/src/inner.rs @@ -20,17 +20,19 @@ use alloc::vec::Vec; use codec::Decode; -use frame_benchmarking::v1::benchmarks; +use frame_benchmarking::v2::*; use frame_election_provider_support::{NposSolver, PhragMMS, SequentialPhragmen}; - -pub struct Pallet(frame_system::Pallet); -pub trait Config: frame_system::Config {} +use sp_runtime::Perbill; const VOTERS: [u32; 2] = [1_000, 2_000]; const TARGETS: [u32; 2] = [500, 1_000]; const VOTES_PER_VOTER: [u32; 2] = [5, 16]; - const SEED: u32 = 999; + +pub trait Config: frame_system::Config {} + +pub struct Pallet(frame_system::Pallet); + fn set_up_voters_targets( voters_len: u32, targets_len: u32, @@ -54,36 +56,47 @@ fn set_up_voters_targets( (voters, targets) } -benchmarks! { - phragmen { - // number of votes in snapshot. - let v in (VOTERS[0]) .. VOTERS[1]; - // number of targets in snapshot. - let t in (TARGETS[0]) .. TARGETS[1]; - // number of votes per voter (ie the degree). - let d in (VOTES_PER_VOTER[0]) .. VOTES_PER_VOTER[1]; - - let (voters, targets) = set_up_voters_targets::(v, t, d as usize); - }: { - assert!( - SequentialPhragmen:: - ::solve(d as usize, targets, voters).is_ok() - ); +#[benchmarks] +mod benchmarks { + use super::*; + + #[benchmark] + fn phragmen( + // Number of votes in snapshot. + v: Linear<{ VOTERS[0] }, { VOTERS[1] }>, + // Number of targets in snapshot. + t: Linear<{ TARGETS[0] }, { TARGETS[1] }>, + // Number of votes per voter (ie the degree). + d: Linear<{ VOTES_PER_VOTER[0] }, { VOTES_PER_VOTER[1] }>, + ) { + let (voters, targets) = set_up_voters_targets::(v, t, d as _); + let result; + + #[block] + { + result = SequentialPhragmen::::solve(d as _, targets, voters); + } + + assert!(result.is_ok()); } - phragmms { - // number of votes in snapshot. - let v in (VOTERS[0]) .. VOTERS[1]; - // number of targets in snapshot. - let t in (TARGETS[0]) .. TARGETS[1]; - // number of votes per voter (ie the degree). - let d in (VOTES_PER_VOTER[0]) .. VOTES_PER_VOTER[1]; - - let (voters, targets) = set_up_voters_targets::(v, t, d as usize); - }: { - assert!( - PhragMMS:: - ::solve(d as usize, targets, voters).is_ok() - ); + #[benchmark] + fn phragmms( + // Number of votes in snapshot. + v: Linear<{ VOTERS[0] }, { VOTERS[1] }>, + // Number of targets in snapshot. + t: Linear<{ TARGETS[0] }, { TARGETS[1] }>, + // Number of votes per voter (ie the degree). + d: Linear<{ VOTES_PER_VOTER[0] }, { VOTES_PER_VOTER[1] }>, + ) { + let (voters, targets) = set_up_voters_targets::(v, t, d as _); + let result; + + #[block] + { + result = PhragMMS::::solve(d as _, targets, voters); + } + + assert!(result.is_ok()); } } diff --git a/substrate/frame/elections-phragmen/src/benchmarking.rs b/substrate/frame/elections-phragmen/src/benchmarking.rs index 8e762f667b2a6..60771fa89ad7e 100644 --- a/substrate/frame/elections-phragmen/src/benchmarking.rs +++ b/substrate/frame/elections-phragmen/src/benchmarking.rs @@ -19,47 +19,47 @@ #![cfg(feature = "runtime-benchmarks")] -use super::*; - -use frame_benchmarking::v1::{account, benchmarks, whitelist, BenchmarkError, BenchmarkResult}; +use frame_benchmarking::v2::*; use frame_support::{dispatch::DispatchResultWithPostInfo, traits::OnInitialize}; use frame_system::RawOrigin; -use crate::Pallet as Elections; +#[cfg(test)] +use crate::tests::MEMBERS; +use crate::*; const BALANCE_FACTOR: u32 = 250; -/// grab new account with infinite balance. +// grab new account with infinite balance. fn endowed_account(name: &'static str, index: u32) -> T::AccountId { let account: T::AccountId = account(name, index, 0); // Fund each account with at-least their stake but still a sane amount as to not mess up // the vote calculation. let amount = default_stake::(T::MaxVoters::get()) * BalanceOf::::from(BALANCE_FACTOR); let _ = T::Currency::make_free_balance_be(&account, amount); - // important to increase the total issuance since T::CurrencyToVote will need it to be sane for - // phragmen to work. + // Important to increase the total issuance since `T::CurrencyToVote` will need it to be sane + // for phragmen to work. let _ = T::Currency::issue(amount); account } -/// Account to lookup type of system trait. +// Account to lookup type of system trait. fn as_lookup(account: T::AccountId) -> AccountIdLookupOf { T::Lookup::unlookup(account) } -/// Get a reasonable amount of stake based on the execution trait's configuration +// Get a reasonable amount of stake based on the execution trait's configuration. fn default_stake(num_votes: u32) -> BalanceOf { let min = T::Currency::minimum_balance(); - Elections::::deposit_of(num_votes as usize).max(min) + Pallet::::deposit_of(num_votes as usize).max(min) } -/// Get the current number of candidates. +// Get the current number of candidates. fn candidate_count() -> u32 { Candidates::::decode_len().unwrap_or(0usize) as u32 } -/// Add `c` new candidates. +// Add `c` new candidates. fn submit_candidates( c: u32, prefix: &'static str, @@ -67,7 +67,7 @@ fn submit_candidates( (0..c) .map(|i| { let account = endowed_account::(prefix, i); - Elections::::submit_candidacy( + Pallet::::submit_candidacy( RawOrigin::Signed(account.clone()).into(), candidate_count::(), ) @@ -77,7 +77,7 @@ fn submit_candidates( .collect::>() } -/// Add `c` new candidates with self vote. +// Add `c` new candidates with self vote. fn submit_candidates_with_self_vote( c: u32, prefix: &'static str, @@ -90,17 +90,17 @@ fn submit_candidates_with_self_vote( Ok(candidates) } -/// Submit one voter. +// Submit one voter. fn submit_voter( caller: T::AccountId, votes: Vec, stake: BalanceOf, ) -> DispatchResultWithPostInfo { - Elections::::vote(RawOrigin::Signed(caller).into(), votes, stake) + Pallet::::vote(RawOrigin::Signed(caller).into(), votes, stake) } -/// create `num_voter` voters who randomly vote for at most `votes` of `all_candidates` if -/// available. +// Create `num_voter` voters who randomly vote for at most `votes` of `all_candidates` if +// available. fn distribute_voters( mut all_candidates: Vec, num_voters: u32, @@ -117,12 +117,12 @@ fn distribute_voters( Ok(()) } -/// Fill the seats of members and runners-up up until `m`. Note that this might include either only -/// members, or members and runners-up. +// Fill the seats of members and runners-up up until `m`. Note that this might include either only +// members, or members and runners-up. fn fill_seats_up_to(m: u32) -> Result, &'static str> { let _ = submit_candidates_with_self_vote::(m, "fill_seats_up_to")?; assert_eq!(Candidates::::get().len() as u32, m, "wrong number of candidates."); - Elections::::do_phragmen(); + Pallet::::do_phragmen(); assert_eq!(Candidates::::get().len(), 0, "some candidates remaining."); assert_eq!( Members::::get().len() + RunnersUp::::get().len(), @@ -136,7 +136,7 @@ fn fill_seats_up_to(m: u32) -> Result, &'static str .collect()) } -/// removes all the storage items to reverse any genesis state. +// Removes all the storage items to reverse any genesis state. fn clean() { Members::::kill(); Candidates::::kill(); @@ -145,10 +145,13 @@ fn clean() { Voting::::remove_all(None); } -benchmarks! { +#[benchmarks] +mod benchmarks { + use super::*; + // -- Signed ones - vote_equal { - let v in 1 .. T::MaxVotesPerVoter::get(); + #[benchmark] + fn vote_equal(v: Linear<1, { T::MaxVotesPerVoter::get() }>) -> Result<(), BenchmarkError> { clean::(); // create a bunch of candidates. @@ -157,65 +160,81 @@ benchmarks! { let caller = endowed_account::("caller", 0); let stake = default_stake::(v); - // original votes. + // Original votes. let mut votes = all_candidates; submit_voter::(caller.clone(), votes.clone(), stake)?; - // new votes. + // New votes. votes.rotate_left(1); whitelist!(caller); - }: vote(RawOrigin::Signed(caller), votes, stake) - vote_more { - let v in 2 .. T::MaxVotesPerVoter::get(); + #[extrinsic_call] + vote(RawOrigin::Signed(caller), votes, stake); + + Ok(()) + } + + #[benchmark] + fn vote_more(v: Linear<2, { T::MaxVotesPerVoter::get() }>) -> Result<(), BenchmarkError> { clean::(); - // create a bunch of candidates. + // Create a bunch of candidates. let all_candidates = submit_candidates::(v, "candidates")?; let caller = endowed_account::("caller", 0); // Multiply the stake with 10 since we want to be able to divide it by 10 again. - let stake = default_stake::(v) * BalanceOf::::from(10u32); + let stake = default_stake::(v) * BalanceOf::::from(10_u32); - // original votes. + // Original votes. let mut votes = all_candidates.iter().skip(1).cloned().collect::>(); - submit_voter::(caller.clone(), votes.clone(), stake / BalanceOf::::from(10u32))?; + submit_voter::(caller.clone(), votes.clone(), stake / BalanceOf::::from(10_u32))?; - // new votes. + // New votes. votes = all_candidates; assert!(votes.len() > Voting::::get(caller.clone()).votes.len()); whitelist!(caller); - }: vote(RawOrigin::Signed(caller), votes, stake / BalanceOf::::from(10u32)) - vote_less { - let v in 2 .. T::MaxVotesPerVoter::get(); + #[extrinsic_call] + vote(RawOrigin::Signed(caller), votes, stake / BalanceOf::::from(10_u32)); + + Ok(()) + } + + #[benchmark] + fn vote_less(v: Linear<2, { T::MaxVotesPerVoter::get() }>) -> Result<(), BenchmarkError> { clean::(); - // create a bunch of candidates. + // Create a bunch of candidates. let all_candidates = submit_candidates::(v, "candidates")?; let caller = endowed_account::("caller", 0); let stake = default_stake::(v); - // original votes. + // Original votes. let mut votes = all_candidates; submit_voter::(caller.clone(), votes.clone(), stake)?; - // new votes. + // New votes. votes = votes.into_iter().skip(1).collect::>(); assert!(votes.len() < Voting::::get(caller.clone()).votes.len()); whitelist!(caller); - }: vote(RawOrigin::Signed(caller), votes, stake) - remove_voter { - // we fix the number of voted candidates to max + #[extrinsic_call] + vote(RawOrigin::Signed(caller), votes, stake); + + Ok(()) + } + + #[benchmark] + fn remove_voter() -> Result<(), BenchmarkError> { + // We fix the number of voted candidates to max. let v = T::MaxVotesPerVoter::get(); clean::(); - // create a bunch of candidates. + // Create a bunch of candidates. let all_candidates = submit_candidates::(v, "candidates")?; let caller = endowed_account::("caller", 0); @@ -224,207 +243,245 @@ benchmarks! { submit_voter::(caller.clone(), all_candidates, stake)?; whitelist!(caller); - }: _(RawOrigin::Signed(caller)) - submit_candidacy { - // number of already existing candidates. - let c in 1 .. T::MaxCandidates::get(); - // we fix the number of members to the number of desired members and runners-up. We'll be in - // this state almost always. + #[extrinsic_call] + _(RawOrigin::Signed(caller)); + + Ok(()) + } + + #[benchmark] + fn submit_candidacy( + // Number of already existing candidates. + c: Linear<1, { T::MaxCandidates::get() }>, + ) -> Result<(), BenchmarkError> { + // We fix the number of members to the number of desired members and runners-up. + // We'll be in this state almost always. let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); clean::(); - let stake = default_stake::(c); - // create m members and runners combined. + // Create `m` members and runners combined. let _ = fill_seats_up_to::(m)?; - // create previous candidates; + // Create previous candidates. let _ = submit_candidates::(c, "candidates")?; - // we assume worse case that: extrinsic is successful and candidate is not duplicate. + // We assume worse case that: extrinsic is successful and candidate is not duplicate. let candidate_account = endowed_account::("caller", 0); whitelist!(candidate_account); - }: _(RawOrigin::Signed(candidate_account.clone()), candidate_count::()) - verify { + + #[extrinsic_call] + _(RawOrigin::Signed(candidate_account), candidate_count::()); + + // Reset members in between benchmark tests. #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } + MEMBERS.with(|m| *m.borrow_mut() = vec![]); + + Ok(()) } - renounce_candidacy_candidate { - // this will check members, runners-up and candidate for removal. Members and runners-up are - // limited by the runtime bound, nonetheless we fill them by `m`. - // number of already existing candidates. - let c in 1 .. T::MaxCandidates::get(); - // we fix the number of members to the number of desired members and runners-up. We'll be in - // this state almost always. + #[benchmark] + fn renounce_candidacy_candidate( + // This will check members, runners-up and candidate for removal. + // Members and runners-up are limited by the runtime bound, nonetheless we fill them by + // `m`. + // Number of already existing candidates. + c: Linear<1, { T::MaxCandidates::get() }>, + ) -> Result<(), BenchmarkError> { + // We fix the number of members to the number of desired members and runners-up. + // We'll be in this state almost always. let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); clean::(); - // create m members and runners combined. + // Create `m` members and runners combined. let _ = fill_seats_up_to::(m)?; let all_candidates = submit_candidates::(c, "caller")?; let bailing = all_candidates[0].clone(); // Should be ("caller", 0) let count = candidate_count::(); whitelist!(bailing); - }: renounce_candidacy(RawOrigin::Signed(bailing), Renouncing::Candidate(count)) - verify { + + #[extrinsic_call] + renounce_candidacy(RawOrigin::Signed(bailing), Renouncing::Candidate(count)); + + // Reset members in between benchmark tests. #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } + MEMBERS.with(|m| *m.borrow_mut() = vec![]); + + Ok(()) } - renounce_candidacy_members { - // removing members and runners will be cheaper than a candidate. - // we fix the number of members to when members and runners-up to the desired. We'll be in - // this state almost always. + #[benchmark] + fn renounce_candidacy_members() -> Result<(), BenchmarkError> { + // Removing members and runners will be cheaper than a candidate. + // We fix the number of members to when members and runners-up to the desired. + // We'll be in this state almost always. let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); clean::(); - // create m members and runners combined. + // Create `m` members and runners combined. let members_and_runners_up = fill_seats_up_to::(m)?; let bailing = members_and_runners_up[0].clone(); - assert!(Elections::::is_member(&bailing)); + assert!(Pallet::::is_member(&bailing)); whitelist!(bailing); - }: renounce_candidacy(RawOrigin::Signed(bailing.clone()), Renouncing::Member) - verify { + + #[extrinsic_call] + renounce_candidacy(RawOrigin::Signed(bailing.clone()), Renouncing::Member); + + // Reset members in between benchmark tests. #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } + MEMBERS.with(|m| *m.borrow_mut() = vec![]); + + Ok(()) } - renounce_candidacy_runners_up { - // removing members and runners will be cheaper than a candidate. - // we fix the number of members to when members and runners-up to the desired. We'll be in - // this state almost always. + #[benchmark] + fn renounce_candidacy_runners_up() -> Result<(), BenchmarkError> { + // Removing members and runners will be cheaper than a candidate. + // We fix the number of members to when members and runners-up to the desired. + // We'll be in this state almost always. let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); clean::(); - // create m members and runners combined. + // Create `m` members and runners combined. let members_and_runners_up = fill_seats_up_to::(m)?; let bailing = members_and_runners_up[T::DesiredMembers::get() as usize + 1].clone(); - assert!(Elections::::is_runner_up(&bailing)); + assert!(Pallet::::is_runner_up(&bailing)); whitelist!(bailing); - }: renounce_candidacy(RawOrigin::Signed(bailing.clone()), Renouncing::RunnerUp) - verify { + + #[extrinsic_call] + renounce_candidacy(RawOrigin::Signed(bailing.clone()), Renouncing::RunnerUp); + + // Reset members in between benchmark tests. #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } + MEMBERS.with(|m| *m.borrow_mut() = vec![]); + + Ok(()) } // We use the max block weight for this extrinsic for now. See below. - remove_member_without_replacement {}: { - Err(BenchmarkError::Override( - BenchmarkResult::from_weight(T::BlockWeights::get().max_block) - ))?; + #[benchmark] + fn remove_member_without_replacement() -> Result<(), BenchmarkError> { + #[block] + { + Err(BenchmarkError::Override(BenchmarkResult::from_weight( + T::BlockWeights::get().max_block, + )))?; + } + + Ok(()) } - remove_member_with_replacement { - // easy case. We have a runner up. Nothing will have that much of an impact. m will be - // number of members and runners. There is always at least one runner. + #[benchmark] + fn remove_member_with_replacement() -> Result<(), BenchmarkError> { + // Easy case. + // We have a runner up. + // Nothing will have that much of an impact. + // `m` will be number of members and runners. + // There is always at least one runner. let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); clean::(); let _ = fill_seats_up_to::(m)?; - let removing = as_lookup::(Elections::::members_ids()[0].clone()); - }: remove_member(RawOrigin::Root, removing, true, false) - verify { - // must still have enough members. + let removing = as_lookup::(Pallet::::members_ids()[0].clone()); + + #[extrinsic_call] + remove_member(RawOrigin::Root, removing, true, false); + + // Must still have enough members. assert_eq!(Members::::get().len() as u32, T::DesiredMembers::get()); + + // Reset members in between benchmark tests. #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } - } + MEMBERS.with(|m| *m.borrow_mut() = vec![]); - clean_defunct_voters { - // total number of voters. - let v in (T::MaxVoters::get() / 2) .. T::MaxVoters::get(); - // those that are defunct and need removal. - let d in 0 .. (T::MaxVoters::get() / 2); + Ok(()) + } - // remove any previous stuff. + #[benchmark] + fn clean_defunct_voters( + // Total number of voters. + v: Linear<{ T::MaxVoters::get() / 2 }, { T::MaxVoters::get() }>, + // Those that are defunct and need removal. + d: Linear<0, { T::MaxVoters::get() / 2 }>, + ) -> Result<(), BenchmarkError> { + // Remove any previous stuff. clean::(); let all_candidates = submit_candidates::(T::MaxCandidates::get(), "candidates")?; distribute_voters::(all_candidates, v, T::MaxVotesPerVoter::get() as usize)?; - // all candidates leave. + // All candidates leave. Candidates::::kill(); - // now everyone is defunct - assert!(Voting::::iter().all(|(_, v)| Elections::::is_defunct_voter(&v.votes))); + // Now everyone is defunct. + assert!(Voting::::iter().all(|(_, v)| Pallet::::is_defunct_voter(&v.votes))); assert_eq!(Voting::::iter().count() as u32, v); - let root = RawOrigin::Root; - }: _(root, v, d) - verify { + + #[extrinsic_call] + _(RawOrigin::Root, v, d); + assert_eq!(Voting::::iter().count() as u32, v - d); + + Ok(()) } - election_phragmen { - // This is just to focus on phragmen in the context of this module. We always select 20 - // members, this is hard-coded in the runtime and cannot be trivially changed at this stage. - // Yet, change the number of voters, candidates and edge per voter to see the impact. Note - // that we give all candidates a self vote to make sure they are all considered. - let c in 1 .. T::MaxCandidates::get(); - let v in 1 .. T::MaxVoters::get(); - let e in (T::MaxVoters::get()) .. T::MaxVoters::get() * T::MaxVotesPerVoter::get(); + #[benchmark] + fn election_phragmen( + // This is just to focus on phragmen in the context of this module. + // We always select 20 members, this is hard-coded in the runtime and cannot be trivially + // changed at this stage. Yet, change the number of voters, candidates and edge per voter + // to see the impact. Note that we give all candidates a self vote to make sure they are + // all considered. + c: Linear<1, { T::MaxCandidates::get() }>, + v: Linear<1, { T::MaxVoters::get() }>, + e: Linear<{ T::MaxVoters::get() }, { T::MaxVoters::get() * T::MaxVotesPerVoter::get() }>, + ) -> Result<(), BenchmarkError> { clean::(); - // so we have a situation with v and e. we want e to basically always be in the range of `e - // -> e * T::MaxVotesPerVoter::get()`, but we cannot express that now with the benchmarks. - // So what we do is: when c is being iterated, v, and e are max and fine. when v is being - // iterated, e is being set to max and this is a problem. In these cases, we cap e to a - // lower value, namely v * T::MaxVotesPerVoter::get(). when e is being iterated, v is at - // max, and again fine. all in all, votes_per_voter can never be more than - // T::MaxVotesPerVoter::get(). Note that this might cause `v` to be an overestimate. + // So we have a situation with `v` and `e`. + // We want `e` to basically always be in the range of + // `e -> e * T::MaxVotesPerVoter::get()`, but we cannot express that now with the + // benchmarks. So what we do is: when `c` is being iterated, `v`, and `e` are max and + // fine. When `v` is being iterated, `e` is being set to max and this is a problem. + // In these cases, we cap `e` to a lower value, namely `v * T::MaxVotesPerVoter::get()`. + // When `e` is being iterated, `v` is at max, and again fine. + // All in all, `votes_per_voter` can never be more than `T::MaxVotesPerVoter::get()`. + // Note that this might cause `v` to be an overestimate. let votes_per_voter = (e / v).min(T::MaxVotesPerVoter::get()); let all_candidates = submit_candidates_with_self_vote::(c, "candidates")?; - let _ = distribute_voters::(all_candidates, v.saturating_sub(c), votes_per_voter as usize)?; - }: { - Elections::::on_initialize(T::TermDuration::get()); - } - verify { + let _ = + distribute_voters::(all_candidates, v.saturating_sub(c), votes_per_voter as usize)?; + + #[block] + { + Pallet::::on_initialize(T::TermDuration::get()); + } + assert_eq!(Members::::get().len() as u32, T::DesiredMembers::get().min(c)); assert_eq!( RunnersUp::::get().len() as u32, T::DesiredRunnersUp::get().min(c.saturating_sub(T::DesiredMembers::get())), ); + // reset members in between benchmark tests. #[cfg(test)] - { - // reset members in between benchmark tests. - use crate::tests::MEMBERS; - MEMBERS.with(|m| *m.borrow_mut() = vec![]); - } + MEMBERS.with(|m| *m.borrow_mut() = vec![]); + + Ok(()) } - impl_benchmark_test_suite!( - Elections, - crate::tests::ExtBuilder::default().desired_members(13).desired_runners_up(7), - crate::tests::Test, + impl_benchmark_test_suite! { + Pallet, + tests::ExtBuilder::default().desired_members(13).desired_runners_up(7), + tests::Test, exec_name = build_and_execute, - ); + } } diff --git a/substrate/frame/identity/src/lib.rs b/substrate/frame/identity/src/lib.rs index 11b43f958c4eb..6a71e831cca11 100644 --- a/substrate/frame/identity/src/lib.rs +++ b/substrate/frame/identity/src/lib.rs @@ -421,6 +421,10 @@ pub mod pallet { RegistrarAdded { registrar_index: RegistrarIndex }, /// A sub-identity was added to an identity and the deposit paid. SubIdentityAdded { sub: T::AccountId, main: T::AccountId, deposit: BalanceOf }, + /// An account's sub-identities were set (in bulk). + SubIdentitiesSet { main: T::AccountId, number_of_subs: u32, new_deposit: BalanceOf }, + /// A given sub-account's associated name was changed by its super-identity. + SubIdentityRenamed { sub: T::AccountId, main: T::AccountId }, /// A sub-identity was removed from an identity and the deposit freed. SubIdentityRemoved { sub: T::AccountId, main: T::AccountId, deposit: BalanceOf }, /// A sub-identity was cleared, and the given deposit repatriated from the @@ -591,6 +595,12 @@ pub mod pallet { SubsOf::::insert(&sender, (new_deposit, ids)); } + Self::deposit_event(Event::SubIdentitiesSet { + main: sender, + number_of_subs: new_subs as u32, + new_deposit, + }); + Ok(Some( T::WeightInfo::set_subs_old(old_ids.len() as u32) // P: Real number of old accounts removed. // S: New subs added @@ -992,7 +1002,9 @@ pub mod pallet { let sub = T::Lookup::lookup(sub)?; ensure!(IdentityOf::::contains_key(&sender), Error::::NoIdentity); ensure!(SuperOf::::get(&sub).map_or(false, |x| x.0 == sender), Error::::NotOwned); - SuperOf::::insert(&sub, (sender, data)); + SuperOf::::insert(&sub, (&sender, data)); + + Self::deposit_event(Event::SubIdentityRenamed { main: sender, sub }); Ok(()) } diff --git a/substrate/frame/identity/src/tests.rs b/substrate/frame/identity/src/tests.rs index a095085a81882..7bf5b2a727607 100644 --- a/substrate/frame/identity/src/tests.rs +++ b/substrate/frame/identity/src/tests.rs @@ -273,6 +273,10 @@ fn editing_subaccounts_should_work() { // rename first sub account assert_ok!(Identity::rename_sub(RuntimeOrigin::signed(ten.clone()), one.clone(), data(11))); + System::assert_last_event(tests::RuntimeEvent::Identity(Event::SubIdentityRenamed { + main: ten.clone(), + sub: one.clone(), + })); assert_eq!(SuperOf::::get(one.clone()), Some((ten.clone(), data(11)))); assert_eq!(SuperOf::::get(two.clone()), Some((ten.clone(), data(2)))); assert_eq!(Balances::free_balance(ten.clone()), 1000 - id_deposit - 2 * sub_deposit); @@ -546,6 +550,13 @@ fn setting_subaccounts_should_work() { assert_ok!(Identity::set_identity(RuntimeOrigin::signed(ten.clone()), Box::new(ten_info))); assert_eq!(Balances::free_balance(ten.clone()), 1000 - id_deposit); assert_ok!(Identity::set_subs(RuntimeOrigin::signed(ten.clone()), subs.clone())); + + System::assert_last_event(tests::RuntimeEvent::Identity(Event::SubIdentitiesSet { + main: ten.clone(), + number_of_subs: 1, + new_deposit: sub_deposit, + })); + assert_eq!(Balances::free_balance(ten.clone()), 1000 - id_deposit - sub_deposit); assert_eq!( SubsOf::::get(ten.clone()), diff --git a/substrate/frame/indices/src/benchmarking.rs b/substrate/frame/indices/src/benchmarking.rs index bd173815cb347..28f5e3bf5cf08 100644 --- a/substrate/frame/indices/src/benchmarking.rs +++ b/substrate/frame/indices/src/benchmarking.rs @@ -19,26 +19,31 @@ #![cfg(feature = "runtime-benchmarks")] -use super::*; -use frame_benchmarking::v1::{account, benchmarks, whitelisted_caller}; +use crate::*; +use frame_benchmarking::v2::*; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; -use crate::Pallet as Indices; - const SEED: u32 = 0; -benchmarks! { - claim { +#[benchmarks] +mod benchmarks { + use super::*; + + #[benchmark] + fn claim() { let account_index = T::AccountIndex::from(SEED); let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - }: _(RawOrigin::Signed(caller.clone()), account_index) - verify { + + #[extrinsic_call] + _(RawOrigin::Signed(caller.clone()), account_index); + assert_eq!(Accounts::::get(account_index).unwrap().0, caller); } - transfer { + #[benchmark] + fn transfer() -> Result<(), BenchmarkError> { let account_index = T::AccountIndex::from(SEED); // Setup accounts let caller: T::AccountId = whitelisted_caller(); @@ -47,25 +52,33 @@ benchmarks! { let recipient_lookup = T::Lookup::unlookup(recipient.clone()); T::Currency::make_free_balance_be(&recipient, BalanceOf::::max_value()); // Claim the index - Indices::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; - }: _(RawOrigin::Signed(caller.clone()), recipient_lookup, account_index) - verify { + Pallet::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; + + #[extrinsic_call] + _(RawOrigin::Signed(caller.clone()), recipient_lookup, account_index); + assert_eq!(Accounts::::get(account_index).unwrap().0, recipient); + Ok(()) } - free { + #[benchmark] + fn free() -> Result<(), BenchmarkError> { let account_index = T::AccountIndex::from(SEED); // Setup accounts let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); // Claim the index - Indices::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; - }: _(RawOrigin::Signed(caller.clone()), account_index) - verify { + Pallet::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; + + #[extrinsic_call] + _(RawOrigin::Signed(caller.clone()), account_index); + assert_eq!(Accounts::::get(account_index), None); + Ok(()) } - force_transfer { + #[benchmark] + fn force_transfer() -> Result<(), BenchmarkError> { let account_index = T::AccountIndex::from(SEED); // Setup accounts let original: T::AccountId = account("original", 0, SEED); @@ -74,25 +87,32 @@ benchmarks! { let recipient_lookup = T::Lookup::unlookup(recipient.clone()); T::Currency::make_free_balance_be(&recipient, BalanceOf::::max_value()); // Claim the index - Indices::::claim(RawOrigin::Signed(original).into(), account_index)?; - }: _(RawOrigin::Root, recipient_lookup, account_index, false) - verify { + Pallet::::claim(RawOrigin::Signed(original).into(), account_index)?; + + #[extrinsic_call] + _(RawOrigin::Root, recipient_lookup, account_index, false); + assert_eq!(Accounts::::get(account_index).unwrap().0, recipient); + Ok(()) } - freeze { + #[benchmark] + fn freeze() -> Result<(), BenchmarkError> { let account_index = T::AccountIndex::from(SEED); // Setup accounts let caller: T::AccountId = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); // Claim the index - Indices::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; - }: _(RawOrigin::Signed(caller.clone()), account_index) - verify { + Pallet::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; + + #[extrinsic_call] + _(RawOrigin::Signed(caller.clone()), account_index); + assert_eq!(Accounts::::get(account_index).unwrap().2, true); + Ok(()) } // TODO in another PR: lookup and unlookup trait weights (not critical) - impl_benchmark_test_suite!(Indices, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Pallet, mock::new_test_ext(), mock::Test); } diff --git a/substrate/frame/nomination-pools/runtime-api/src/lib.rs b/substrate/frame/nomination-pools/runtime-api/src/lib.rs index d81ad1dd4954e..4138dd22d8987 100644 --- a/substrate/frame/nomination-pools/runtime-api/src/lib.rs +++ b/substrate/frame/nomination-pools/runtime-api/src/lib.rs @@ -69,5 +69,8 @@ sp_api::decl_runtime_apis! { /// Total balance contributed to the pool. fn pool_balance(pool_id: PoolId) -> Balance; + + /// Returns the bonded account and reward account associated with the pool_id. + fn pool_accounts(pool_id: PoolId) -> (AccountId, AccountId); } } diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index 177c5da74d4ff..201b0af1d6080 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -4020,6 +4020,13 @@ impl Pallet { T::StakeAdapter::total_balance(Pool::from(Self::generate_bonded_account(pool_id))) .unwrap_or_default() } + + /// Returns the bonded account and reward account associated with the pool_id. + pub fn api_pool_accounts(pool_id: PoolId) -> (T::AccountId, T::AccountId) { + let bonded_account = Self::generate_bonded_account(pool_id); + let reward_account = Self::generate_reward_account(pool_id); + (bonded_account, reward_account) + } } impl sp_staking::OnStakingUpdate> for Pallet { diff --git a/substrate/frame/revive/Cargo.toml b/substrate/frame/revive/Cargo.toml index 67bc1809cad74..81fbbc8cf38e1 100644 --- a/substrate/frame/revive/Cargo.toml +++ b/substrate/frame/revive/Cargo.toml @@ -40,7 +40,7 @@ frame-benchmarking = { optional = true, workspace = true } frame-support = { workspace = true } frame-system = { workspace = true } pallet-balances = { optional = true, workspace = true } -pallet-revive-fixtures = { workspace = true, default-features = false } +pallet-revive-fixtures = { workspace = true, default-features = false, optional = true } pallet-revive-uapi = { workspace = true, default-features = true } pallet-revive-proc-macro = { workspace = true, default-features = true } pallet-transaction-payment = { workspace = true } @@ -91,7 +91,7 @@ std = [ "log/std", "pallet-balances?/std", "pallet-proxy/std", - "pallet-revive-fixtures/std", + "pallet-revive-fixtures?/std", "pallet-timestamp/std", "pallet-transaction-payment/std", "pallet-utility/std", @@ -121,6 +121,7 @@ runtime-benchmarks = [ "pallet-balances/runtime-benchmarks", "pallet-message-queue/runtime-benchmarks", "pallet-proxy/runtime-benchmarks", + "pallet-revive-fixtures", "pallet-timestamp/runtime-benchmarks", "pallet-transaction-payment/runtime-benchmarks", "pallet-utility/runtime-benchmarks", diff --git a/substrate/frame/staking/src/pallet/impls.rs b/substrate/frame/staking/src/pallet/impls.rs index 6499037411401..d3423d82769dc 100644 --- a/substrate/frame/staking/src/pallet/impls.rs +++ b/substrate/frame/staking/src/pallet/impls.rs @@ -349,6 +349,8 @@ impl Pallet { Self::deposit_event(Event::::PayoutStarted { era_index: era, validator_stash: stash.clone(), + page, + next: EraInfo::::get_next_claimable_page(era, &stash, &ledger), }); let mut total_imbalance = PositiveImbalanceOf::::zero(); diff --git a/substrate/frame/staking/src/pallet/mod.rs b/substrate/frame/staking/src/pallet/mod.rs index 28aa4f89b6227..5210bef853b29 100644 --- a/substrate/frame/staking/src/pallet/mod.rs +++ b/substrate/frame/staking/src/pallet/mod.rs @@ -851,8 +851,13 @@ pub mod pallet { StakingElectionFailed, /// An account has stopped participating as either a validator or nominator. Chilled { stash: T::AccountId }, - /// The stakers' rewards are getting paid. - PayoutStarted { era_index: EraIndex, validator_stash: T::AccountId }, + /// A Page of stakers rewards are getting paid. `next` is `None` if all pages are claimed. + PayoutStarted { + era_index: EraIndex, + validator_stash: T::AccountId, + page: Page, + next: Option, + }, /// A validator has set their preferences. ValidatorPrefsSet { stash: T::AccountId, prefs: ValidatorPrefs }, /// Voters size limit reached. diff --git a/substrate/frame/staking/src/tests.rs b/substrate/frame/staking/src/tests.rs index 639f4096456fb..d1dc6c3db659b 100644 --- a/substrate/frame/staking/src/tests.rs +++ b/substrate/frame/staking/src/tests.rs @@ -3978,6 +3978,7 @@ fn test_multi_page_payout_stakers_by_page() { assert!(matches!( staking_events_since_last_call().as_slice(), &[ + Event::PayoutStarted { era_index: 1, validator_stash: 11, page: 0, next: Some(1) }, .., Event::Rewarded { stash: 1063, dest: RewardDestination::Stash, amount: 111 }, Event::Rewarded { stash: 1064, dest: RewardDestination::Stash, amount: 111 }, @@ -4001,7 +4002,7 @@ fn test_multi_page_payout_stakers_by_page() { assert!(matches!( events.as_slice(), &[ - Event::PayoutStarted { era_index: 1, validator_stash: 11 }, + Event::PayoutStarted { era_index: 1, validator_stash: 11, page: 1, next: None }, Event::Rewarded { stash: 1065, dest: RewardDestination::Stash, amount: 111 }, Event::Rewarded { stash: 1066, dest: RewardDestination::Stash, amount: 111 }, .. @@ -8011,7 +8012,8 @@ mod ledger_recovery { assert_eq!(asset::staked::(&333), lock_333_before); // OK assert_eq!(Bonded::::get(&333), Some(444)); // OK assert!(Payee::::get(&333).is_some()); // OK - // however, ledger associated with its controller was killed. + + // however, ledger associated with its controller was killed. assert!(Ledger::::get(&444).is_none()); // NOK // side effects on 444 - ledger, bonded, payee, lock should be completely removed. diff --git a/substrate/frame/support/src/generate_genesis_config.rs b/substrate/frame/support/src/generate_genesis_config.rs index fc21e76c74273..283840d70c7c4 100644 --- a/substrate/frame/support/src/generate_genesis_config.rs +++ b/substrate/frame/support/src/generate_genesis_config.rs @@ -23,41 +23,44 @@ use alloc::{borrow::Cow, format, string::String}; /// Represents the initialization method of a field within a struct. /// -/// This enum provides information about how it was initialized and the field name (as a `String`). +/// This enum provides information about how it was initialized. /// /// Intended to be used in `build_struct_json_patch` macro. #[derive(Debug)] -pub enum InitializedField<'a> { +pub enum InitilizationType { /// The field was partially initialized (e.g., specific fields within the struct were set /// manually). - Partial(Cow<'a, str>), - /// The field was fully initialized (e.g., using `new()` or `default()` like methods). - Full(Cow<'a, str>), + Partial, + /// The field was fully initialized (e.g., using `new()` or `default()` like methods + Full, } +/// This struct provides information about how the struct field was initialized and the field name +/// (as a `&str`). +/// +/// Intended to be used in `build_struct_json_patch` macro. +#[derive(Debug)] +pub struct InitializedField<'a>(InitilizationType, Cow<'a, str>); + impl<'a> InitializedField<'a> { /// Returns a name of the field. pub fn get_name(&'a self) -> &'a str { - match self { - Self::Partial(s) | Self::Full(s) => s, - } + &self.1 } /// Injects a prefix to the field name. pub fn add_prefix(&mut self, prefix: &str) { - match self { - Self::Partial(s) | Self::Full(s) => *s = format!("{prefix}.{s}").into(), - }; + self.1 = format!("{prefix}.{}", self.1).into() } /// Creates new partial field instiance. pub fn partial(s: &'a str) -> Self { - Self::Partial(s.into()) + Self(InitilizationType::Partial, s.into()) } /// Creates new full field instiance. pub fn full(s: &'a str) -> Self { - Self::Full(s.into()) + Self(InitilizationType::Full, s.into()) } } @@ -73,9 +76,15 @@ impl PartialEq for InitializedField<'_> { .map(|c| c.to_ascii_uppercase()) .eq(camel_chars.map(|c| c.to_ascii_uppercase())) } - match self { - InitializedField::Partial(field_name) | InitializedField::Full(field_name) => - field_name == other || compare_keys(field_name.chars(), other.chars()), + *self.1 == *other || compare_keys(self.1.chars(), other.chars()) + } +} + +impl<'a> From<(InitilizationType, &'a str)> for InitializedField<'a> { + fn from(value: (InitilizationType, &'a str)) -> Self { + match value.0 { + InitilizationType::Full => InitializedField::full(value.1), + InitilizationType::Partial => InitializedField::partial(value.1), } } } @@ -104,8 +113,8 @@ pub fn retain_initialized_fields( let current_key = if current_root.is_empty() { key.clone() } else { format!("{current_root}.{key}") }; match keys_to_retain.iter().find(|key| **key == current_key) { - Some(InitializedField::Full(_)) => true, - Some(InitializedField::Partial(_)) => { + Some(InitializedField(InitilizationType::Full, _)) => true, + Some(InitializedField(InitilizationType::Partial, _)) => { retain_initialized_fields(value, keys_to_retain, current_key.clone()); true }, @@ -208,89 +217,154 @@ pub fn retain_initialized_fields( #[macro_export] macro_rules! build_struct_json_patch { ( - $($struct_type:ident)::+ { $($tail:tt)* } + $($struct_type:ident)::+ { $($body:tt)* } ) => { { - let mut keys = $crate::__private::Vec::<$crate::generate_genesis_config::InitializedField>::default(); + let mut __keys = $crate::__private::Vec::<$crate::generate_genesis_config::InitializedField>::default(); #[allow(clippy::needless_update)] - let struct_instance = $crate::build_struct_json_patch!($($struct_type)::+, keys @ { $($tail)* }); - let mut json_value = - $crate::__private::serde_json::to_value(struct_instance).expect("serialization to json should work. qed"); - $crate::generate_genesis_config::retain_initialized_fields(&mut json_value, &keys, Default::default()); - json_value + let __struct_instance = $crate::build_struct_json_patch!($($struct_type)::+, __keys @ { $($body)* }).0; + let mut __json_value = + $crate::__private::serde_json::to_value(__struct_instance).expect("serialization to json should work. qed"); + $crate::generate_genesis_config::retain_initialized_fields(&mut __json_value, &__keys, Default::default()); + __json_value } }; - ($($struct_type:ident)::+, $all_keys:ident @ { $($tail:tt)* }) => { - $($struct_type)::+ { - ..$crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($tail)*) + ($($struct_type:ident)::+, $all_keys:ident @ { $($body:tt)* }) => { + { + let __value = $crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($body)*); + ( + $($struct_type)::+ { ..__value.0 }, + __value.1 + ) } }; - ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $($type:ident)::+ { $keyi:ident : $value:tt } ) => { - $($struct_type)::+ { - $key: { - $all_keys.push($crate::generate_genesis_config::InitializedField::partial(stringify!($key))); - $all_keys.push( - $crate::generate_genesis_config::InitializedField::full(concat!(stringify!($key), ".", stringify!($keyi))) - ); - $($type)::+ { - $keyi:$value, - ..Default::default() - } + ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $($type:ident)::+ { $($body:tt)* } ) => { + ( + $($struct_type)::+ { + $key: { + let mut __inner_keys = + $crate::__private::Vec::<$crate::generate_genesis_config::InitializedField>::default(); + let __value = $crate::build_struct_json_patch!($($type)::+, __inner_keys @ { $($body)* }); + for i in __inner_keys.iter_mut() { + i.add_prefix(stringify!($key)); + }; + $all_keys.push((__value.1,stringify!($key)).into()); + $all_keys.extend(__inner_keys); + __value.0 + }, + ..Default::default() }, - ..Default::default() + $crate::generate_genesis_config::InitilizationType::Partial + ) + }; + ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $($type:ident)::+ { $($body:tt)* }, $($tail:tt)*) => { + { + let mut __initialization_type; + ( + $($struct_type)::+ { + $key : { + let mut __inner_keys = + $crate::__private::Vec::<$crate::generate_genesis_config::InitializedField>::default(); + let __value = $crate::build_struct_json_patch!($($type)::+, __inner_keys @ { $($body)* }); + $all_keys.push((__value.1,stringify!($key)).into()); + + for i in __inner_keys.iter_mut() { + i.add_prefix(stringify!($key)); + }; + $all_keys.extend(__inner_keys); + __value.0 + }, + .. { + let (__value, __tmp) = + $crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($tail)*); + __initialization_type = __tmp; + __value + } + }, + __initialization_type + ) } }; - ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $($type:ident)::+ { $($body:tt)* } ) => { - $($struct_type)::+ { - $key: { - $all_keys.push($crate::generate_genesis_config::InitializedField::partial(stringify!($key))); - let mut inner_keys = $crate::__private::Vec::<$crate::generate_genesis_config::InitializedField>::default(); - let value = $crate::build_struct_json_patch!($($type)::+, inner_keys @ { $($body)* }); - for i in inner_keys.iter_mut() { - i.add_prefix(stringify!($key)); - }; - $all_keys.extend(inner_keys); - value - }, - ..Default::default() + ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $value:expr, $($tail:tt)* ) => { + { + let mut __initialization_type; + ( + $($struct_type)::+ { + $key: { + $all_keys.push($crate::generate_genesis_config::InitializedField::full( + stringify!($key)) + ); + $value + }, + .. { + let (__value, __tmp) = + $crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($tail)*); + __initialization_type = __tmp; + __value + } + }, + __initialization_type + ) } }; - ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $($type:ident)::+ { $($body:tt)* }, $($tail:tt)* ) => { - $($struct_type)::+ { - $key : { - $all_keys.push($crate::generate_genesis_config::InitializedField::partial(stringify!($key))); - let mut inner_keys = $crate::__private::Vec::<$crate::generate_genesis_config::InitializedField>::default(); - let value = $crate::build_struct_json_patch!($($type)::+, inner_keys @ { $($body)* }); - for i in inner_keys.iter_mut() { - i.add_prefix(stringify!($key)); - }; - $all_keys.extend(inner_keys); - value + ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $value:expr ) => { + ( + $($struct_type)::+ { + $key: { + $all_keys.push($crate::generate_genesis_config::InitializedField::full(stringify!($key))); + $value + }, + ..Default::default() }, - .. $crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($tail)*) + $crate::generate_genesis_config::InitilizationType::Partial + ) + }; + // field init shorthand + ($($struct_type:ident)::+, $all_keys:ident @ $key:ident, $($tail:tt)* ) => { + { + let __update = $crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($tail)*); + ( + $($struct_type)::+ { + $key: { + $all_keys.push($crate::generate_genesis_config::InitializedField::full( + stringify!($key)) + ); + $key + }, + ..__update.0 + }, + __update.1 + ) } }; - ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $value:expr, $($tail:tt)* ) => { - $($struct_type)::+ { - $key: { - $all_keys.push($crate::generate_genesis_config::InitializedField::full(stringify!($key))); - $value + ($($struct_type:ident)::+, $all_keys:ident @ $key:ident ) => { + ( + $($struct_type)::+ { + $key: { + $all_keys.push($crate::generate_genesis_config::InitializedField::full(stringify!($key))); + $key + }, + ..Default::default() }, - ..$crate::build_struct_json_patch!($($struct_type)::+, $all_keys @ $($tail)*) - } + $crate::generate_genesis_config::InitilizationType::Partial + ) }; - ($($struct_type:ident)::+, $all_keys:ident @ $key:ident: $value:expr ) => { - $($struct_type)::+ { - $key: { - $all_keys.push($crate::generate_genesis_config::InitializedField::full(stringify!($key))); - $value + // update struct + ($($struct_type:ident)::+, $all_keys:ident @ ..$update:expr ) => { + ( + $($struct_type)::+ { + ..$update }, - ..Default::default() - } + $crate::generate_genesis_config::InitilizationType::Full + ) }; - ($($struct_type:ident)::+, $all_keys:ident @ $(,)?) => { - $($struct_type)::+ { ..Default::default() } + ( + $($struct_type)::+ { + ..Default::default() + }, + $crate::generate_genesis_config::InitilizationType::Partial + ) }; } @@ -401,11 +475,8 @@ mod test { macro_rules! test { ($($struct:ident)::+ { $($v:tt)* }, { $($j:tt)* } ) => {{ - println!("--"); let expected = serde_json::json!({ $($j)* }); - println!("json: {}", serde_json::to_string_pretty(&expected).unwrap()); let value = build_struct_json_patch!($($struct)::+ { $($v)* }); - println!("gc: {}", serde_json::to_string_pretty(&value).unwrap()); assert_eq!(value, expected); }}; } @@ -415,6 +486,7 @@ mod test { let t = 5; const C: u32 = 5; test!(TestStruct { b: 5 }, { "b": 5 }); + test!(TestStruct { b: 5, }, { "b": 5 }); #[allow(unused_braces)] { test!(TestStruct { b: { 4 + 34 } } , { "b": 38 }); @@ -705,6 +777,324 @@ mod test { ); } + #[test] + fn test_generate_config_macro_field_init_shorthand() { + { + let x = 5; + test!(TestStruct { s: S { x } }, { "s": { "x": 5 } }); + } + { + let s = nested_mod::InsideMod { a: 34, b: 8 }; + test!( + TestStruct { + t: nested_mod::InsideMod { a: 32 }, + u: nested_mod::nested_mod2::nested_mod3::InsideMod3 { + s, + a: 32, + } + }, + { + "t" : { "a": 32 }, + "u" : { "a": 32, "s": { "a": 34, "b": 8} } + } + ); + } + { + let s = nested_mod::InsideMod { a: 34, b: 8 }; + test!( + TestStruct { + t: nested_mod::InsideMod { a: 32 }, + u: nested_mod::nested_mod2::nested_mod3::InsideMod3 { + a: 32, + s, + } + }, + { + "t" : { "a": 32 }, + "u" : { "a": 32, "s": { "a": 34, "b": 8} } + } + ); + } + } + + #[test] + fn test_generate_config_macro_struct_update() { + { + let s = S { x: 5 }; + test!(TestStruct { s: S { ..s } }, { "s": { "x": 5 } }); + } + { + mod nested { + use super::*; + pub fn function() -> S { + S { x: 5 } + } + } + test!(TestStruct { s: S { ..nested::function() } }, { "s": { "x": 5 } }); + } + { + let s = nested_mod::InsideMod { a: 34, b: 8 }; + let s1 = nested_mod::InsideMod { a: 34, b: 8 }; + test!( + TestStruct { + t: nested_mod::InsideMod { ..s1 }, + u: nested_mod::nested_mod2::nested_mod3::InsideMod3 { + s, + a: 32, + } + }, + { + "t" : { "a": 34, "b": 8 }, + "u" : { "a": 32, "s": { "a": 34, "b": 8} } + } + ); + } + { + let i3 = nested_mod::nested_mod2::nested_mod3::InsideMod3 { + a: 1, + b: 2, + s: nested_mod::InsideMod { a: 55, b: 88 }, + }; + test!( + TestStruct { + t: nested_mod::InsideMod { a: 32 }, + u: nested_mod::nested_mod2::nested_mod3::InsideMod3 { + a: 32, + ..i3 + } + }, + { + "t" : { "a": 32 }, + "u" : { "a": 32, "b": 2, "s": { "a": 55, "b": 88} } + } + ); + } + { + let s = nested_mod::InsideMod { a: 34, b: 8 }; + test!( + TestStruct { + t: nested_mod::InsideMod { a: 32 }, + u: nested_mod::nested_mod2::nested_mod3::InsideMod3 { + a: 32, + s: nested_mod::InsideMod { + b: 66, + ..s + } + } + }, + { + "t" : { "a": 32 }, + "u" : { "a": 32, "s": { "a": 34, "b": 66} } + } + ); + } + { + let s = nested_mod::InsideMod { a: 34, b: 8 }; + test!( + TestStruct { + t: nested_mod::InsideMod { a: 32 }, + u: nested_mod::nested_mod2::nested_mod3::InsideMod3 { + s: nested_mod::InsideMod { + b: 66, + ..s + }, + a: 32 + } + }, + { + "t" : { "a": 32 }, + "u" : { "a": 32, "s": { "a": 34, "b": 66} } + } + ); + } + } + + #[test] + fn test_generate_config_macro_with_execution_order() { + #[derive(Debug, Default, serde::Serialize, serde::Deserialize, PartialEq)] + struct X { + x: Vec, + x2: Vec, + y2: Y, + } + #[derive(Debug, Default, serde::Serialize, serde::Deserialize, PartialEq)] + struct Y { + y: Vec, + } + #[derive(Debug, Default, serde::Serialize, serde::Deserialize, PartialEq)] + struct Z { + a: u32, + x: X, + y: Y, + } + { + let v = vec![1, 2, 3]; + test!(Z { a: 0, x: X { x: v }, }, { + "a": 0, "x": { "x": [1,2,3] } + }); + } + { + let v = vec![1, 2, 3]; + test!(Z { a: 3, x: X { x: v.clone() }, y: Y { y: v } }, { + "a": 3, "x": { "x": [1,2,3] }, "y": { "y": [1,2,3] } + }); + } + { + let v = vec![1, 2, 3]; + test!(Z { a: 3, x: X { y2: Y { y: v.clone() }, x: v.clone() }, y: Y { y: v } }, { + "a": 3, "x": { "x": [1,2,3], "y2":{ "y":[1,2,3] } }, "y": { "y": [1,2,3] } + }); + } + { + let v = vec![1, 2, 3]; + test!(Z { a: 3, y: Y { y: v.clone() }, x: X { y2: Y { y: v.clone() }, x: v }, }, { + "a": 3, "x": { "x": [1,2,3], "y2":{ "y":[1,2,3] } }, "y": { "y": [1,2,3] } + }); + } + { + let v = vec![1, 2, 3]; + test!( + Z { + y: Y { + y: v.clone() + }, + x: X { + y2: Y { + y: v.clone() + }, + x: v.clone(), + x2: v.clone() + }, + }, + { + "x": { + "x": [1,2,3], + "x2": [1,2,3], + "y2": { + "y":[1,2,3] + } + }, + "y": { + "y": [1,2,3] + } + }); + } + { + let v = vec![1, 2, 3]; + test!( + Z { + y: Y { + y: v.clone() + }, + x: X { + y2: Y { + y: v.clone() + }, + x: v + }, + }, + { + "x": { + "x": [1,2,3], + "y2": { + "y":[1,2,3] + } + }, + "y": { + "y": [1,2,3] + } + }); + } + { + let mut v = vec![0, 1, 2]; + let f = |vec: &mut Vec| -> Vec { + vec.iter_mut().for_each(|x| *x += 1); + vec.clone() + }; + let z = Z { + a: 0, + y: Y { y: f(&mut v) }, + x: X { y2: Y { y: f(&mut v) }, x: f(&mut v), x2: vec![] }, + }; + let z_expected = Z { + a: 0, + y: Y { y: vec![1, 2, 3] }, + x: X { y2: Y { y: vec![2, 3, 4] }, x: vec![3, 4, 5], x2: vec![] }, + }; + assert_eq!(z, z_expected); + v = vec![0, 1, 2]; + println!("{z:?}"); + test!( + Z { + y: Y { + y: f(&mut v) + }, + x: X { + y2: Y { + y: f(&mut v) + }, + x: f(&mut v) + }, + }, + { + "y": { + "y": [1,2,3] + }, + "x": { + "y2": { + "y":[2,3,4] + }, + "x": [3,4,5], + }, + }); + } + { + let mut v = vec![0, 1, 2]; + let f = |vec: &mut Vec| -> Vec { + vec.iter_mut().for_each(|x| *x += 1); + vec.clone() + }; + let z = Z { + a: 0, + y: Y { y: f(&mut v) }, + x: X { y2: Y { y: f(&mut v) }, x: f(&mut v), x2: f(&mut v) }, + }; + let z_expected = Z { + a: 0, + y: Y { y: vec![1, 2, 3] }, + x: X { y2: Y { y: vec![2, 3, 4] }, x: vec![3, 4, 5], x2: vec![4, 5, 6] }, + }; + assert_eq!(z, z_expected); + v = vec![0, 1, 2]; + println!("{z:?}"); + test!( + Z { + y: Y { + y: f(&mut v) + }, + x: X { + y2: Y { + y: f(&mut v) + }, + x: f(&mut v), + x2: f(&mut v) + }, + }, + { + "y": { + "y": [1,2,3] + }, + "x": { + "y2": { + "y":[2,3,4] + }, + "x": [3,4,5], + "x2": [4,5,6], + }, + }); + } + } + #[test] fn test_generate_config_macro_with_nested_mods() { test!( @@ -797,7 +1187,6 @@ mod retain_keys_test { ( $s:literal ) => { let field = InitializedField::full($s); let cc = inflector::cases::camelcase::to_camel_case($s); - println!("field: {:?}, cc: {}", field, cc); assert_eq!(field,cc); } ; ( &[ $f:literal $(, $r:literal)* ]) => { @@ -808,7 +1197,6 @@ mod retain_keys_test { .map(|s| inflector::cases::camelcase::to_camel_case(s)) .collect::>() .join("."); - println!("field: {:?}, cc: {}", field, cc); assert_eq!(field,cc); } ; ); diff --git a/substrate/frame/support/test/tests/runtime_metadata.rs b/substrate/frame/support/test/tests/runtime_metadata.rs index 81377210eb43f..7523a415d458e 100644 --- a/substrate/frame/support/test/tests/runtime_metadata.rs +++ b/substrate/frame/support/test/tests/runtime_metadata.rs @@ -178,7 +178,7 @@ fn runtime_metadata() { RuntimeApiMethodMetadataIR { name: "wild_card", inputs: vec![RuntimeApiMethodParamMetadataIR:: { - name: "_", + name: "__runtime_api_generated_name_0__", ty: meta_type::(), }], output: meta_type::<()>(), diff --git a/substrate/frame/transaction-storage/src/benchmarking.rs b/substrate/frame/transaction-storage/src/benchmarking.rs index f360e9847a1e1..0b5b0dc994054 100644 --- a/substrate/frame/transaction-storage/src/benchmarking.rs +++ b/substrate/frame/transaction-storage/src/benchmarking.rs @@ -19,16 +19,14 @@ #![cfg(feature = "runtime-benchmarks")] -use super::*; +use crate::*; use alloc::{vec, vec::Vec}; -use frame_benchmarking::v1::{benchmarks, whitelisted_caller}; +use frame_benchmarking::v2::*; use frame_support::traits::{Get, OnFinalize, OnInitialize}; use frame_system::{pallet_prelude::BlockNumberFor, EventRecord, Pallet as System, RawOrigin}; use sp_runtime::traits::{Bounded, CheckedDiv, One, Zero}; use sp_transaction_storage_proof::TransactionStorageProof; -use crate::Pallet as TransactionStorage; - // Proof generated from max size storage: // ``` // let mut transactions = Vec::new(); @@ -122,39 +120,50 @@ pub fn run_to_block(n: frame_system::pallet_prelude::BlockNumberFor) { let caller: T::AccountId = whitelisted_caller(); let initial_balance = BalanceOf::::max_value().checked_div(&2u32.into()).unwrap(); T::Currency::set_balance(&caller, initial_balance); - }: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]) - verify { + + #[extrinsic_call] + _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]); + assert!(!BlockTransactions::::get().is_empty()); assert_last_event::(Event::Stored { index: 0 }.into()); } - renew { + #[benchmark] + fn renew() -> Result<(), BenchmarkError> { let caller: T::AccountId = whitelisted_caller(); let initial_balance = BalanceOf::::max_value().checked_div(&2u32.into()).unwrap(); T::Currency::set_balance(&caller, initial_balance); - TransactionStorage::::store( + Pallet::::store( RawOrigin::Signed(caller.clone()).into(), vec![0u8; T::MaxTransactionSize::get() as usize], )?; run_to_block::(1u32.into()); - }: _(RawOrigin::Signed(caller.clone()), BlockNumberFor::::zero(), 0) - verify { + + #[extrinsic_call] + _(RawOrigin::Signed(caller.clone()), BlockNumberFor::::zero(), 0); + assert_last_event::(Event::Renewed { index: 0 }.into()); + + Ok(()) } - check_proof_max { + #[benchmark] + fn check_proof_max() -> Result<(), BenchmarkError> { run_to_block::(1u32.into()); let caller: T::AccountId = whitelisted_caller(); let initial_balance = BalanceOf::::max_value().checked_div(&2u32.into()).unwrap(); T::Currency::set_balance(&caller, initial_balance); - for _ in 0 .. T::MaxBlockTransactions::get() { - TransactionStorage::::store( + for _ in 0..T::MaxBlockTransactions::get() { + Pallet::::store( RawOrigin::Signed(caller.clone()).into(), vec![0u8; T::MaxTransactionSize::get() as usize], )?; @@ -162,10 +171,14 @@ benchmarks! { run_to_block::(StoragePeriod::::get() + BlockNumberFor::::one()); let encoded_proof = proof(); let proof = TransactionStorageProof::decode(&mut &*encoded_proof).unwrap(); - }: check_proof(RawOrigin::None, proof) - verify { + + #[extrinsic_call] + check_proof(RawOrigin::None, proof); + assert_last_event::(Event::ProofChecked.into()); + + Ok(()) } - impl_benchmark_test_suite!(TransactionStorage, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Pallet, mock::new_test_ext(), mock::Test); } diff --git a/substrate/primitives/api/proc-macro/Cargo.toml b/substrate/primitives/api/proc-macro/Cargo.toml index 659307e7b0f89..191578f432adb 100644 --- a/substrate/primitives/api/proc-macro/Cargo.toml +++ b/substrate/primitives/api/proc-macro/Cargo.toml @@ -20,7 +20,7 @@ proc-macro = true [dependencies] quote = { workspace = true } -syn = { features = ["extra-traits", "fold", "full", "visit"], workspace = true } +syn = { features = ["extra-traits", "fold", "full", "visit", "visit-mut"], workspace = true } proc-macro2 = { workspace = true } blake2 = { workspace = true } proc-macro-crate = { workspace = true } diff --git a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs index cb213f2fd627b..ddca1095a1920 100644 --- a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -32,6 +32,7 @@ use proc_macro2::{Span, TokenStream}; use quote::quote; +use std::collections::{BTreeMap, HashMap}; use syn::{ fold::{self, Fold}, parse::{Error, Parse, ParseStream, Result}, @@ -43,8 +44,6 @@ use syn::{ TraitItem, TraitItemFn, }; -use std::collections::{BTreeMap, HashMap}; - /// The structure used for parsing the runtime api declarations. struct RuntimeApiDecls { decls: Vec, @@ -133,7 +132,7 @@ fn remove_supported_attributes(attrs: &mut Vec) -> HashMap<&'static s /// ``` fn generate_versioned_api_traits( api: ItemTrait, - methods: BTreeMap>, + methods: BTreeMap>, ) -> Vec { let mut result = Vec::::new(); for (version, _) in &methods { @@ -189,15 +188,12 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> Result { extend_generics_with_block(&mut decl.generics); let mod_name = generate_runtime_mod_name_for_trait(&decl.ident); let found_attributes = remove_supported_attributes(&mut decl.attrs); - let api_version = - get_api_version(&found_attributes).map(|v| generate_runtime_api_version(v as u32))?; + let api_version = get_api_version(&found_attributes).map(generate_runtime_api_version)?; let id = generate_runtime_api_id(&decl.ident.to_string()); - let metadata = crate::runtime_metadata::generate_decl_runtime_metadata(&decl); - let trait_api_version = get_api_version(&found_attributes)?; - let mut methods_by_version: BTreeMap> = BTreeMap::new(); + let mut methods_by_version: BTreeMap> = BTreeMap::new(); // Process the items in the declaration. The filter_map function below does a lot of stuff // because the method attributes are stripped at this point @@ -255,6 +251,12 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> Result { _ => (), }); + let versioned_methods_iter = methods_by_version + .iter() + .flat_map(|(&version, methods)| methods.iter().map(move |method| (method, version))); + let metadata = + crate::runtime_metadata::generate_decl_runtime_metadata(&decl, versioned_methods_iter); + let versioned_api_traits = generate_versioned_api_traits(decl.clone(), methods_by_version); let main_api_ident = decl.ident.clone(); @@ -505,7 +507,7 @@ fn generate_runtime_api_version(version: u32) -> TokenStream { } /// Generates the implementation of `RuntimeApiInfo` for the given trait. -fn generate_runtime_info_impl(trait_: &ItemTrait, version: u64) -> TokenStream { +fn generate_runtime_info_impl(trait_: &ItemTrait, version: u32) -> TokenStream { let trait_name = &trait_.ident; let crate_ = generate_crate_access(); let id = generate_runtime_api_id(&trait_name.to_string()); @@ -537,7 +539,7 @@ fn generate_runtime_info_impl(trait_: &ItemTrait, version: u64) -> TokenStream { } /// Get changed in version from the user given attribute or `Ok(None)`, if no attribute was given. -fn get_changed_in(found_attributes: &HashMap<&'static str, Attribute>) -> Result> { +fn get_changed_in(found_attributes: &HashMap<&'static str, Attribute>) -> Result> { found_attributes .get(&CHANGED_IN_ATTRIBUTE) .map(|v| parse_runtime_api_version(v).map(Some)) @@ -545,7 +547,7 @@ fn get_changed_in(found_attributes: &HashMap<&'static str, Attribute>) -> Result } /// Get the api version from the user given attribute or `Ok(1)`, if no attribute was given. -fn get_api_version(found_attributes: &HashMap<&'static str, Attribute>) -> Result { +fn get_api_version(found_attributes: &HashMap<&'static str, Attribute>) -> Result { found_attributes .get(&API_VERSION_ATTRIBUTE) .map(parse_runtime_api_version) @@ -610,7 +612,7 @@ impl CheckTraitDecl { /// /// Any error is stored in `self.errors`. fn check_method_declarations<'a>(&mut self, methods: impl Iterator) { - let mut method_to_signature_changed = HashMap::>>::new(); + let mut method_to_signature_changed = HashMap::>>::new(); methods.into_iter().for_each(|method| { let attributes = remove_supported_attributes(&mut method.attrs.clone()); diff --git a/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs index de922e3253e47..5c9448da2bc7e 100644 --- a/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -15,14 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - common::API_VERSION_ATTRIBUTE, - utils::{ - extract_block_type_from_trait_path, extract_impl_trait, - extract_parameter_names_types_and_borrows, generate_crate_access, - generate_runtime_mod_name_for_trait, parse_runtime_api_version, prefix_function_with_trait, - versioned_trait_name, AllowSelfRefInParameters, RequireQualifiedTraitPath, - }, +use crate::utils::{ + extract_api_version, extract_block_type_from_trait_path, extract_impl_trait, + extract_parameter_names_types_and_borrows, generate_crate_access, + generate_runtime_mod_name_for_trait, prefix_function_with_trait, versioned_trait_name, + AllowSelfRefInParameters, ApiVersion, RequireQualifiedTraitPath, }; use proc_macro2::{Span, TokenStream}; @@ -31,11 +28,11 @@ use quote::quote; use syn::{ fold::{self, Fold}, - parenthesized, parse::{Error, Parse, ParseStream, Result}, parse_macro_input, parse_quote, spanned::Spanned, - Attribute, Ident, ImplItem, ItemImpl, LitInt, LitStr, Path, Signature, Type, TypePath, + visit_mut::{self, VisitMut}, + Attribute, Ident, ImplItem, ItemImpl, Path, Signature, Type, TypePath, }; use std::collections::HashMap; @@ -227,34 +224,34 @@ fn generate_wasm_interface(impls: &[ItemImpl]) -> Result { let c = generate_crate_access(); let impl_calls = - generate_impl_calls(impls, &input)? - .into_iter() - .map(|(trait_, fn_name, impl_, attrs)| { - let fn_name = - Ident::new(&prefix_function_with_trait(&trait_, &fn_name), Span::call_site()); - - quote!( - #c::std_disabled! { - #( #attrs )* - #[no_mangle] - #[cfg_attr(any(target_arch = "riscv32", target_arch = "riscv64"), #c::__private::polkavm_export(abi = #c::__private::polkavm_abi))] - pub unsafe extern fn #fn_name(input_data: *mut u8, input_len: usize) -> u64 { - let mut #input = if input_len == 0 { - &[0u8; 0] - } else { - unsafe { - ::core::slice::from_raw_parts(input_data, input_len) - } - }; - - #c::init_runtime_logger(); - - let output = (move || { #impl_ })(); - #c::to_substrate_wasm_fn_return_value(&output) - } - } - ) - }); + generate_impl_calls(impls, &input)? + .into_iter() + .map(|(trait_, fn_name, impl_, attrs)| { + let fn_name = + Ident::new(&prefix_function_with_trait(&trait_, &fn_name), Span::call_site()); + + quote!( + #c::std_disabled! { + #( #attrs )* + #[no_mangle] + #[cfg_attr(any(target_arch = "riscv32", target_arch = "riscv64"), #c::__private::polkavm_export(abi = #c::__private::polkavm_abi))] + pub unsafe extern fn #fn_name(input_data: *mut u8, input_len: usize) -> u64 { + let mut #input = if input_len == 0 { + &[0u8; 0] + } else { + unsafe { + ::core::slice::from_raw_parts(input_data, input_len) + } + }; + + #c::init_runtime_logger(); + + let output = (move || { #impl_ })(); + #c::to_substrate_wasm_fn_return_value(&output) + } + } + ) + }); Ok(quote!( #( #impl_calls )* )) } @@ -396,10 +393,10 @@ fn generate_runtime_api_base_structures() -> Result { impl> RuntimeApiImpl { fn commit_or_rollback_transaction(&self, commit: bool) { let proof = "\ - We only close a transaction when we opened one ourself. - Other parts of the runtime that make use of transactions (state-machine) - also balance their transactions. The runtime cannot close client initiated - transactions; qed"; + We only close a transaction when we opened one ourself. + Other parts of the runtime that make use of transactions (state-machine) + also balance their transactions. The runtime cannot close client initiated + transactions; qed"; let res = if commit { let res = if let Some(recorder) = &self.recorder { @@ -466,7 +463,7 @@ fn extend_with_runtime_decl_path(mut trait_: Path) -> Path { trait_ } -fn extend_with_api_version(mut trait_: Path, version: Option) -> Path { +fn extend_with_api_version(mut trait_: Path, version: Option) -> Path { let version = if let Some(v) = version { v } else { @@ -740,8 +737,8 @@ fn generate_runtime_api_versions(impls: &[ItemImpl]) -> Result { let mut error = Error::new( span, "Two traits with the same name detected! \ - The trait name is used to generate its ID. \ - Please rename one trait at the declaration!", + The trait name is used to generate its ID. \ + Please rename one trait at the declaration!", ); error.combine(Error::new(other_span, "First trait implementation.")); @@ -787,17 +784,50 @@ fn generate_runtime_api_versions(impls: &[ItemImpl]) -> Result { )) } +/// replaces `Self` with explicit `ItemImpl.self_ty`. +struct ReplaceSelfImpl { + self_ty: Box, +} + +impl ReplaceSelfImpl { + /// Replace `Self` with `ItemImpl.self_ty` + fn replace(&mut self, trait_: &mut ItemImpl) { + visit_mut::visit_item_impl_mut(self, trait_) + } +} + +impl VisitMut for ReplaceSelfImpl { + fn visit_type_mut(&mut self, ty: &mut syn::Type) { + match ty { + Type::Path(p) if p.path.is_ident("Self") => { + *ty = *self.self_ty.clone(); + }, + ty => syn::visit_mut::visit_type_mut(self, ty), + } + } +} + +/// Rename `Self` to `ItemImpl.self_ty` in all items. +fn rename_self_in_trait_impls(impls: &mut [ItemImpl]) { + impls.iter_mut().for_each(|i| { + let mut checker = ReplaceSelfImpl { self_ty: i.self_ty.clone() }; + checker.replace(i); + }); +} + /// The implementation of the `impl_runtime_apis!` macro. pub fn impl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { // Parse all impl blocks - let RuntimeApiImpls { impls: api_impls } = parse_macro_input!(input as RuntimeApiImpls); + let RuntimeApiImpls { impls: mut api_impls } = parse_macro_input!(input as RuntimeApiImpls); - impl_runtime_apis_impl_inner(&api_impls) + impl_runtime_apis_impl_inner(&mut api_impls) .unwrap_or_else(|e| e.to_compile_error()) .into() } -fn impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result { +fn impl_runtime_apis_impl_inner(api_impls: &mut [ItemImpl]) -> Result { + rename_self_in_trait_impls(api_impls); + let dispatch_impl = generate_dispatch_function(api_impls)?; let api_impls_for_runtime = generate_api_impl_for_runtime(api_impls)?; let base_runtime_api = generate_runtime_api_base_structures()?; @@ -841,88 +871,6 @@ fn filter_cfg_attrs(attrs: &[Attribute]) -> Vec { attrs.iter().filter(|a| a.path().is_ident("cfg")).cloned().collect() } -/// Parse feature flagged api_version. -/// E.g. `#[cfg_attr(feature = "enable-staging-api", api_version(99))]` -fn extract_cfg_api_version(attrs: &Vec, span: Span) -> Result> { - let cfg_attrs = attrs.iter().filter(|a| a.path().is_ident("cfg_attr")).collect::>(); - - let mut cfg_api_version_attr = Vec::new(); - for cfg_attr in cfg_attrs { - let mut feature_name = None; - let mut api_version = None; - cfg_attr.parse_nested_meta(|m| { - if m.path.is_ident("feature") { - let a = m.value()?; - let b: LitStr = a.parse()?; - feature_name = Some(b.value()); - } else if m.path.is_ident(API_VERSION_ATTRIBUTE) { - let content; - parenthesized!(content in m.input); - let ver: LitInt = content.parse()?; - api_version = Some(ver.base10_parse::()?); - } - Ok(()) - })?; - - // If there is a cfg attribute containing api_version - save if for processing - if let (Some(feature_name), Some(api_version)) = (feature_name, api_version) { - cfg_api_version_attr.push((feature_name, api_version, cfg_attr.span())); - } - } - - if cfg_api_version_attr.len() > 1 { - let mut err = Error::new(span, format!("Found multiple feature gated api versions (cfg attribute with nested `{}` attribute). This is not supported.", API_VERSION_ATTRIBUTE)); - for (_, _, attr_span) in cfg_api_version_attr { - err.combine(Error::new(attr_span, format!("`{}` found here", API_VERSION_ATTRIBUTE))); - } - - return Err(err); - } - - Ok(cfg_api_version_attr - .into_iter() - .next() - .map(|(feature, name, _)| (feature, name))) -} - -/// Represents an API version. -struct ApiVersion { - /// Corresponds to `#[api_version(X)]` attribute. - pub custom: Option, - /// Corresponds to `#[cfg_attr(feature = "enable-staging-api", api_version(99))]` - /// attribute. `String` is the feature name, `u64` the staging api version. - pub feature_gated: Option<(String, u64)>, -} - -// Extracts the value of `API_VERSION_ATTRIBUTE` and handles errors. -// Returns: -// - Err if the version is malformed -// - `ApiVersion` on success. If a version is set or not is determined by the fields of `ApiVersion` -fn extract_api_version(attrs: &Vec, span: Span) -> Result { - // First fetch all `API_VERSION_ATTRIBUTE` values (should be only one) - let api_ver = attrs - .iter() - .filter(|a| a.path().is_ident(API_VERSION_ATTRIBUTE)) - .collect::>(); - - if api_ver.len() > 1 { - return Err(Error::new( - span, - format!( - "Found multiple #[{}] attributes for an API implementation. \ - Each runtime API can have only one version.", - API_VERSION_ATTRIBUTE - ), - )); - } - - // Parse the runtime version if there exists one. - Ok(ApiVersion { - custom: api_ver.first().map(|v| parse_runtime_api_version(v)).transpose()?, - feature_gated: extract_cfg_api_version(attrs, span)?, - }) -} - #[cfg(test)] mod tests { use super::*; @@ -945,4 +893,34 @@ mod tests { assert_eq!(cfg_std, filtered[0]); assert_eq!(cfg_benchmarks, filtered[1]); } + + #[test] + fn impl_trait_rename_self_param() { + let code = quote::quote! { + impl client::Core for Runtime { + fn initialize_block(header: &HeaderFor) -> Output { + let _: HeaderFor = header.clone(); + example_fn::(header) + } + } + }; + let expected = quote::quote! { + impl client::Core for Runtime { + fn initialize_block(header: &HeaderFor) -> Output { + let _: HeaderFor = header.clone(); + example_fn::(header) + } + } + }; + + // Parse the items + let RuntimeApiImpls { impls: mut api_impls } = + syn::parse2::(code).unwrap(); + + // Run the renamer which is being run first in the `impl_runtime_apis!` macro. + rename_self_in_trait_impls(&mut api_impls); + let result: TokenStream = quote::quote! { #(#api_impls)* }; + + assert_eq!(result.to_string(), expected.to_string()); + } } diff --git a/substrate/primitives/api/proc-macro/src/runtime_metadata.rs b/substrate/primitives/api/proc-macro/src/runtime_metadata.rs index 4cba524dbe253..6be3963392598 100644 --- a/substrate/primitives/api/proc-macro/src/runtime_metadata.rs +++ b/substrate/primitives/api/proc-macro/src/runtime_metadata.rs @@ -17,14 +17,11 @@ use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use syn::{parse_quote, ItemImpl, ItemTrait, Result}; - -use crate::{ - common::CHANGED_IN_ATTRIBUTE, - utils::{ - extract_impl_trait, filter_cfg_attributes, generate_crate_access, - generate_runtime_mod_name_for_trait, get_doc_literals, RequireQualifiedTraitPath, - }, +use syn::{parse_quote, spanned::Spanned, ItemImpl, ItemTrait, Result}; + +use crate::utils::{ + extract_api_version, extract_impl_trait, filter_cfg_attributes, generate_crate_access, + generate_runtime_mod_name_for_trait, get_doc_literals, RequireQualifiedTraitPath, }; /// Get the type parameter argument without lifetime or mutability @@ -72,7 +69,10 @@ fn collect_docs(attrs: &[syn::Attribute], crate_: &TokenStream2) -> TokenStream2 /// /// The metadata is exposed as a generic function on the hidden module /// of the trait generated by the `decl_runtime_apis`. -pub fn generate_decl_runtime_metadata(decl: &ItemTrait) -> TokenStream2 { +pub fn generate_decl_runtime_metadata<'a>( + decl: &ItemTrait, + versioned_methods_iter: impl Iterator, +) -> TokenStream2 { let crate_ = generate_crate_access(); let mut methods = Vec::new(); @@ -86,17 +86,7 @@ pub fn generate_decl_runtime_metadata(decl: &ItemTrait) -> TokenStream2 { // This restricts the bounds at the metadata level, without needing to modify the `BlockT` // itself, since the concrete implementations are already satisfying `TypeInfo`. let mut where_clause = Vec::new(); - for item in &decl.items { - // Collect metadata for methods only. - let syn::TraitItem::Fn(method) = item else { continue }; - - // Collect metadata only for the latest methods. - let is_changed_in = - method.attrs.iter().any(|attr| attr.path().is_ident(CHANGED_IN_ATTRIBUTE)); - if is_changed_in { - continue; - } - + for (method, version) in versioned_methods_iter { let mut inputs = Vec::new(); let signature = &method.sig; for input in &signature.inputs { @@ -135,14 +125,21 @@ pub fn generate_decl_runtime_metadata(decl: &ItemTrait) -> TokenStream2 { Ok(deprecation) => deprecation, Err(e) => return e.into_compile_error(), }; + + // Methods are filtered so that only those whose version is <= the `impl_version` passed to + // `runtime_metadata` are kept in the metadata we hand back. methods.push(quote!( #( #attrs )* - #crate_::metadata_ir::RuntimeApiMethodMetadataIR { - name: #method_name, - inputs: #crate_::vec![ #( #inputs, )* ], - output: #output, - docs: #docs, - deprecation_info: #deprecation, + if #version <= impl_version { + Some(#crate_::metadata_ir::RuntimeApiMethodMetadataIR { + name: #method_name, + inputs: #crate_::vec![ #( #inputs, )* ], + output: #output, + docs: #docs, + deprecation_info: #deprecation, + }) + } else { + None } )); } @@ -176,12 +173,15 @@ pub fn generate_decl_runtime_metadata(decl: &ItemTrait) -> TokenStream2 { #crate_::frame_metadata_enabled! { #( #attrs )* #[inline(always)] - pub fn runtime_metadata #impl_generics () -> #crate_::metadata_ir::RuntimeApiMetadataIR + pub fn runtime_metadata #impl_generics (impl_version: u32) -> #crate_::metadata_ir::RuntimeApiMetadataIR #where_clause { #crate_::metadata_ir::RuntimeApiMetadataIR { name: #trait_name, - methods: #crate_::vec![ #( #methods, )* ], + methods: [ #( #methods, )* ] + .into_iter() + .filter_map(|maybe_m| maybe_m) + .collect(), docs: #docs, deprecation_info: #deprecation, } @@ -242,10 +242,43 @@ pub fn generate_impl_runtime_metadata(impls: &[ItemImpl]) -> Result Result { +pub fn parse_runtime_api_version(version: &Attribute) -> Result { let version = version.parse_args::().map_err(|_| { Error::new( version.span(), @@ -231,7 +231,7 @@ pub fn parse_runtime_api_version(version: &Attribute) -> Result { } /// Each versioned trait is named 'ApiNameVN' where N is the specific version. E.g. ParachainHostV2 -pub fn versioned_trait_name(trait_ident: &Ident, version: u64) -> Ident { +pub fn versioned_trait_name(trait_ident: &Ident, version: u32) -> Ident { format_ident!("{}V{}", trait_ident, version) } @@ -334,6 +334,89 @@ pub fn get_deprecation(crate_: &TokenStream, attrs: &[syn::Attribute]) -> Result .unwrap_or_else(|| Ok(quote! {#crate_::metadata_ir::DeprecationStatusIR::NotDeprecated})) } +/// Represents an API version. +pub struct ApiVersion { + /// Corresponds to `#[api_version(X)]` attribute. + pub custom: Option, + /// Corresponds to `#[cfg_attr(feature = "enable-staging-api", api_version(99))]` + /// attribute. `String` is the feature name, `u32` the staging api version. + pub feature_gated: Option<(String, u32)>, +} + +/// Extracts the value of `API_VERSION_ATTRIBUTE` and handles errors. +/// Returns: +/// - Err if the version is malformed +/// - `ApiVersion` on success. If a version is set or not is determined by the fields of +/// `ApiVersion` +pub fn extract_api_version(attrs: &[Attribute], span: Span) -> Result { + // First fetch all `API_VERSION_ATTRIBUTE` values (should be only one) + let api_ver = attrs + .iter() + .filter(|a| a.path().is_ident(API_VERSION_ATTRIBUTE)) + .collect::>(); + + if api_ver.len() > 1 { + return Err(Error::new( + span, + format!( + "Found multiple #[{}] attributes for an API implementation. \ + Each runtime API can have only one version.", + API_VERSION_ATTRIBUTE + ), + )); + } + + // Parse the runtime version if there exists one. + Ok(ApiVersion { + custom: api_ver.first().map(|v| parse_runtime_api_version(v)).transpose()?, + feature_gated: extract_cfg_api_version(attrs, span)?, + }) +} + +/// Parse feature flagged api_version. +/// E.g. `#[cfg_attr(feature = "enable-staging-api", api_version(99))]` +fn extract_cfg_api_version(attrs: &[Attribute], span: Span) -> Result> { + let cfg_attrs = attrs.iter().filter(|a| a.path().is_ident("cfg_attr")).collect::>(); + + let mut cfg_api_version_attr = Vec::new(); + for cfg_attr in cfg_attrs { + let mut feature_name = None; + let mut api_version = None; + cfg_attr.parse_nested_meta(|m| { + if m.path.is_ident("feature") { + let a = m.value()?; + let b: LitStr = a.parse()?; + feature_name = Some(b.value()); + } else if m.path.is_ident(API_VERSION_ATTRIBUTE) { + let content; + parenthesized!(content in m.input); + let ver: LitInt = content.parse()?; + api_version = Some(ver.base10_parse::()?); + } + Ok(()) + })?; + + // If there is a cfg attribute containing api_version - save if for processing + if let (Some(feature_name), Some(api_version)) = (feature_name, api_version) { + cfg_api_version_attr.push((feature_name, api_version, cfg_attr.span())); + } + } + + if cfg_api_version_attr.len() > 1 { + let mut err = Error::new(span, format!("Found multiple feature gated api versions (cfg attribute with nested `{}` attribute). This is not supported.", API_VERSION_ATTRIBUTE)); + for (_, _, attr_span) in cfg_api_version_attr { + err.combine(Error::new(attr_span, format!("`{}` found here", API_VERSION_ATTRIBUTE))); + } + + return Err(err); + } + + Ok(cfg_api_version_attr + .into_iter() + .next() + .map(|(feature, name, _)| (feature, name))) +} + #[cfg(test)] mod tests { use assert_matches::assert_matches; diff --git a/substrate/primitives/api/test/tests/decl_and_impl.rs b/substrate/primitives/api/test/tests/decl_and_impl.rs index 211a08561fd4b..890cf6eccdbcb 100644 --- a/substrate/primitives/api/test/tests/decl_and_impl.rs +++ b/substrate/primitives/api/test/tests/decl_and_impl.rs @@ -24,7 +24,7 @@ use substrate_test_runtime_client::runtime::{Block, Hash}; /// The declaration of the `Runtime` type is done by the `construct_runtime!` macro in a real /// runtime. -pub enum Runtime {} +pub struct Runtime {} decl_runtime_apis! { pub trait Api { @@ -306,3 +306,62 @@ fn mock_runtime_api_works_with_advanced() { mock.wild_card(Hash::repeat_byte(0x01), 1).unwrap_err().to_string(), ); } + +#[test] +fn runtime_api_metadata_matches_version_implemented() { + let rt = Runtime {}; + let runtime_metadata = rt.runtime_metadata(); + + // Check that the metadata for some runtime API matches expectation. + let assert_has_api_with_methods = |api_name: &str, api_methods: &[&str]| { + let Some(api) = runtime_metadata.iter().find(|api| api.name == api_name) else { + panic!("Can't find runtime API '{api_name}'"); + }; + if api.methods.len() != api_methods.len() { + panic!( + "Wrong number of methods in '{api_name}'; expected {} methods but got {}: {:?}", + api_methods.len(), + api.methods.len(), + api.methods + ); + } + for expected_name in api_methods { + if !api.methods.iter().any(|method| &method.name == expected_name) { + panic!("Can't find API method '{expected_name}' in '{api_name}'"); + } + } + }; + + assert_has_api_with_methods("ApiWithCustomVersion", &["same_name"]); + + assert_has_api_with_methods("ApiWithMultipleVersions", &["stable_one", "new_one"]); + + assert_has_api_with_methods( + "ApiWithStagingMethod", + &[ + "stable_one", + #[cfg(feature = "enable-staging-api")] + "staging_one", + ], + ); + + assert_has_api_with_methods( + "ApiWithStagingAndVersionedMethods", + &[ + "stable_one", + "new_one", + #[cfg(feature = "enable-staging-api")] + "staging_one", + ], + ); + + assert_has_api_with_methods( + "ApiWithStagingAndChangedBase", + &[ + "stable_one", + "new_one", + #[cfg(feature = "enable-staging-api")] + "staging_one", + ], + ); +} diff --git a/substrate/primitives/consensus/babe/src/lib.rs b/substrate/primitives/consensus/babe/src/lib.rs index ee07da6829f52..163fbafa8dd4c 100644 --- a/substrate/primitives/consensus/babe/src/lib.rs +++ b/substrate/primitives/consensus/babe/src/lib.rs @@ -134,7 +134,7 @@ pub enum ConsensusLog { } /// Configuration data used by the BABE consensus engine. -#[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] +#[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct BabeConfigurationV1 { /// The slot duration in milliseconds for BABE. Currently, only /// the value provided by this type at genesis will be used. diff --git a/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl b/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl index bc587b0447746..cca1f544b3506 100644 --- a/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl +++ b/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl @@ -35,7 +35,10 @@ bob: reports block height is at least {{DB_BLOCK_HEIGHT}} within 10 seconds alice: reports substrate_beefy_best_block is at least {{200*180/6}} within 180 seconds bob: reports substrate_beefy_best_block is at least {{200*180/6}} within 180 seconds -alice: count of log lines containing "error" is 0 within 10 seconds +# Validators started without public addresses must emit an error. +# Double check the error is the expected one. +alice: log line matches "No public addresses configured and no global listen addresses found" within 60 seconds +alice: count of log lines containing "error" is 1 within 10 seconds bob: count of log lines containing "verification failed" is 0 within 10 seconds # new blocks were built diff --git a/templates/minimal/node/src/service.rs b/templates/minimal/node/src/service.rs index f9a9d1e0f3cfe..b4e6fc0b728b6 100644 --- a/templates/minimal/node/src/service.rs +++ b/templates/minimal/node/src/service.rs @@ -21,9 +21,7 @@ use minimal_template_runtime::{interface::OpaqueBlock as Block, RuntimeApi}; use polkadot_sdk::{ sc_client_api::backend::Backend, sc_executor::WasmExecutor, - sc_service::{ - build_polkadot_syncing_strategy, error::Error as ServiceError, Configuration, TaskManager, - }, + sc_service::{error::Error as ServiceError, Configuration, TaskManager}, sc_telemetry::{Telemetry, TelemetryWorker}, sc_transaction_pool_api::OffchainTransactionPoolFactory, sp_runtime::traits::Block as BlockT, @@ -124,7 +122,7 @@ pub fn new_full::Ha other: mut telemetry, } = new_partial(&config)?; - let mut net_config = sc_network::config::FullNetworkConfiguration::< + let net_config = sc_network::config::FullNetworkConfiguration::< Block, ::Hash, Network, @@ -136,26 +134,16 @@ pub fn new_full::Ha config.prometheus_config.as_ref().map(|cfg| &cfg.registry), ); - let syncing_strategy = build_polkadot_syncing_strategy( - config.protocol_id(), - config.chain_spec.fork_id(), - &mut net_config, - None, - client.clone(), - &task_manager.spawn_handle(), - config.prometheus_config.as_ref().map(|config| &config.registry), - )?; - let (network, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, + net_config, client: client.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, - net_config, block_announce_validator_builder: None, - syncing_strategy, + warp_sync_config: None, block_relay: None, metrics, })?; diff --git a/templates/minimal/runtime/src/lib.rs b/templates/minimal/runtime/src/lib.rs index ecdba739c50eb..7b8449f2abe48 100644 --- a/templates/minimal/runtime/src/lib.rs +++ b/templates/minimal/runtime/src/lib.rs @@ -51,17 +51,14 @@ pub mod genesis_config_presets { /// Returns a development genesis config preset. pub fn development_config_genesis() -> Value { let endowment = >::get().max(1) * 1000; - let config = RuntimeGenesisConfig { + frame_support::build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: AccountKeyring::iter() .map(|a| (a.to_account_id(), endowment)) .collect::>(), }, sudo: SudoConfig { key: Some(AccountKeyring::Alice.to_account_id()) }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + }) } /// Get the set of the available genesis config presets. diff --git a/templates/parachain/runtime/src/genesis_config_presets.rs b/templates/parachain/runtime/src/genesis_config_presets.rs index 77ae85c0b174c..aa1ff7895eb82 100644 --- a/templates/parachain/runtime/src/genesis_config_presets.rs +++ b/templates/parachain/runtime/src/genesis_config_presets.rs @@ -8,6 +8,7 @@ use alloc::{vec, vec::Vec}; use polkadot_sdk::{staging_xcm as xcm, *}; use cumulus_primitives_core::ParaId; +use frame_support::build_struct_json_patch; use parachains_common::AuraId; use serde_json::Value; use sp_genesis_builder::PresetId; @@ -31,7 +32,7 @@ fn testnet_genesis( root: AccountId, id: ParaId, ) -> Value { - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -39,11 +40,10 @@ fn testnet_genesis( .map(|k| (k, 1u128 << 60)) .collect::>(), }, - parachain_info: ParachainInfoConfig { parachain_id: id, ..Default::default() }, + parachain_info: ParachainInfoConfig { parachain_id: id }, collator_selection: CollatorSelectionConfig { invulnerables: invulnerables.iter().cloned().map(|(acc, _)| acc).collect::>(), candidacy_bond: EXISTENTIAL_DEPOSIT * 16, - ..Default::default() }, session: SessionConfig { keys: invulnerables @@ -56,17 +56,10 @@ fn testnet_genesis( ) }) .collect::>(), - ..Default::default() - }, - polkadot_xcm: PolkadotXcmConfig { - safe_xcm_version: Some(SAFE_XCM_VERSION), - ..Default::default() }, + polkadot_xcm: PolkadotXcmConfig { safe_xcm_version: Some(SAFE_XCM_VERSION) }, sudo: SudoConfig { key: Some(root) }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + }) } fn local_testnet_genesis() -> Value { diff --git a/templates/solochain/node/Cargo.toml b/templates/solochain/node/Cargo.toml index 8a3c7d0ac7800..4c0ab31df95e2 100644 --- a/templates/solochain/node/Cargo.toml +++ b/templates/solochain/node/Cargo.toml @@ -30,9 +30,9 @@ sc-telemetry = { workspace = true, default-features = true } sc-transaction-pool = { workspace = true, default-features = true } sc-transaction-pool-api = { workspace = true, default-features = true } sc-offchain = { workspace = true, default-features = true } +sc-consensus = { workspace = true, default-features = true } sc-consensus-aura = { workspace = true, default-features = true } sp-consensus-aura = { workspace = true, default-features = true } -sc-consensus = { workspace = true, default-features = true } sc-consensus-grandpa = { workspace = true, default-features = true } sp-consensus-grandpa = { workspace = true, default-features = true } sp-genesis-builder = { workspace = true, default-features = true } diff --git a/templates/solochain/node/src/service.rs b/templates/solochain/node/src/service.rs index 2524906fd508f..d6fcebe239f7e 100644 --- a/templates/solochain/node/src/service.rs +++ b/templates/solochain/node/src/service.rs @@ -4,10 +4,7 @@ use futures::FutureExt; use sc_client_api::{Backend, BlockBackend}; use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams}; use sc_consensus_grandpa::SharedVoterState; -use sc_service::{ - build_polkadot_syncing_strategy, error::Error as ServiceError, Configuration, TaskManager, - WarpSyncConfig, -}; +use sc_service::{error::Error as ServiceError, Configuration, TaskManager, WarpSyncConfig}; use sc_telemetry::{Telemetry, TelemetryWorker}; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use solochain_template_runtime::{self, apis::RuntimeApi, opaque::Block}; @@ -172,16 +169,6 @@ pub fn new_full< Vec::default(), )); - let syncing_strategy = build_polkadot_syncing_strategy( - config.protocol_id(), - config.chain_spec.fork_id(), - &mut net_config, - Some(WarpSyncConfig::WithProvider(warp_sync)), - client.clone(), - &task_manager.spawn_handle(), - config.prometheus_config.as_ref().map(|config| &config.registry), - )?; - let (network, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, @@ -191,7 +178,7 @@ pub fn new_full< spawn_handle: task_manager.spawn_handle(), import_queue, block_announce_validator_builder: None, - syncing_strategy, + warp_sync_config: Some(WarpSyncConfig::WithProvider(warp_sync)), block_relay: None, metrics, })?; diff --git a/templates/solochain/runtime/src/genesis_config_presets.rs b/templates/solochain/runtime/src/genesis_config_presets.rs index 7c444456a600e..049f4593451b9 100644 --- a/templates/solochain/runtime/src/genesis_config_presets.rs +++ b/templates/solochain/runtime/src/genesis_config_presets.rs @@ -17,6 +17,7 @@ use crate::{AccountId, BalancesConfig, RuntimeGenesisConfig, SudoConfig}; use alloc::{vec, vec::Vec}; +use frame_support::build_struct_json_patch; use serde_json::Value; use sp_consensus_aura::sr25519::AuthorityId as AuraId; use sp_consensus_grandpa::AuthorityId as GrandpaId; @@ -29,7 +30,7 @@ fn testnet_genesis( endowed_accounts: Vec, root: AccountId, ) -> Value { - let config = RuntimeGenesisConfig { + build_struct_json_patch!(RuntimeGenesisConfig { balances: BalancesConfig { balances: endowed_accounts .iter() @@ -42,13 +43,9 @@ fn testnet_genesis( }, grandpa: pallet_grandpa::GenesisConfig { authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect::>(), - ..Default::default() }, sudo: SudoConfig { key: Some(root) }, - ..Default::default() - }; - - serde_json::to_value(config).expect("Could not build genesis config.") + }) } /// Return the development genesis config.