From a3bdb2c01ed8baedf1fb7509a680a1b933f12934 Mon Sep 17 00:00:00 2001 From: Raj Raorane <41839716+Raj-RR1@users.noreply.github.com> Date: Wed, 23 Aug 2023 15:16:48 +0530 Subject: [PATCH] aura changes --- node/consensus-transition/aura/src/lib.rs | 530 ++++++++++++++++------ 1 file changed, 400 insertions(+), 130 deletions(-) diff --git a/node/consensus-transition/aura/src/lib.rs b/node/consensus-transition/aura/src/lib.rs index 89d6e22d..f6b1b37d 100644 --- a/node/consensus-transition/aura/src/lib.rs +++ b/node/consensus-transition/aura/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. +// Copyright (C) Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,11 +33,10 @@ use std::{fmt::Debug, hash::Hash, marker::PhantomData, pin::Pin, sync::Arc}; use futures::prelude::*; -use log::{debug, trace}; -use parity_scale_codec::{Codec, Decode, Encode}; +use codec::{Codec, Decode, Encode}; -use sc_client_api::{backend::AuxStore, BlockOf, UsageProvider}; +use sc_client_api::{backend::AuxStore, BlockOf}; use sc_consensus::{BlockImport, BlockImportParams, ForkChoiceStrategy, StateAction}; use sc_consensus_slots::{ BackoffAuthoringBlocksStrategy, InherentDataProviderExt, SimpleSlotWorkerToSlotWorker, @@ -45,23 +44,19 @@ use sc_consensus_slots::{ }; use sc_telemetry::TelemetryHandle; use sp_api::{Core, ProvideRuntimeApi}; -use sp_application_crypto::{AppKey, AppPublic}; -use sp_blockchain::{HeaderBackend, Result as CResult}; -use sp_consensus::{ - BlockOrigin, CanAuthorWith, Environment, Error as ConsensusError, Proposer, SelectChain, -}; +use sp_application_crypto::AppPublic; +use sp_blockchain::HeaderBackend; +use sp_consensus::{BlockOrigin, Environment, Error as ConsensusError, Proposer, SelectChain}; use sp_consensus_slots::Slot; -use sp_core::crypto::{ByteArray, Pair, Public}; +use sp_core::crypto::{Pair, Public}; use sp_inherents::CreateInherentDataProviders; -use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Header, Member, NumberFor, Zero}, - DigestItem, -}; +use sp_keystore::KeystorePtr; +use sp_runtime::traits::{Block as BlockT, Header, Member, NumberFor}; mod import_queue; +pub mod standalone; +pub use crate::standalone::{find_pre_digest, slot_duration}; pub use import_queue::{ build_verifier, import_queue, AuraVerifier, BuildVerifierParams, CheckForEquivocation, ImportQueueParams, @@ -74,6 +69,8 @@ pub use sp_consensus_aura::{ AuraApi, ConsensusLog, SlotDuration, AURA_ENGINE_ID, }; +const LOG_TARGET: &str = "aura"; + type AuthorityId

=

::Public; /// Run `AURA` in a compatibility mode. @@ -86,21 +83,23 @@ pub enum CompatibilityMode { None, /// Call `initialize_block` before doing any runtime calls. /// - /// The node would execute `initialize_block` before fetchting the authorities + /// Previously the node would execute `initialize_block` before fetchting the authorities /// from the runtime. This behaviour changed in: /// /// By calling `initialize_block` before fetching the authorities, on a block that /// would enact a new validator set, the block would already be build/sealed by an - /// authority of the new set. A block that enacts a new set, should not be sealed/build - /// by an authority of the new set. This isn't done anymore. However, to make new nodes - /// being able to sync the old chain this compatibility mode exists. + /// authority of the new set. With this mode disabled (the default) a block that enacts a new + /// set isn't sealed/built by an authority of the new set, however to make new nodes be able to + /// sync old chains this compatibility mode exists. UseInitializeBlock { /// The block number until this compatibility mode should be executed. The first runtime - /// call in the context (importing it/building it) of the `until` block should disable the - /// compatibility mode. This number should be of a block in the future! It should be a - /// block number on that all nodes have upgraded to a release that runs with the - /// compatibility mode. After this block there will be a hard fork when the authority set - /// changes, between the old nodes (running with `initialize_block`) and the new nodes. + /// call in the context of the `until` block (importing it/building it) will disable the + /// compatibility mode (i.e. at `until` the default rules will apply). When enabling this + /// compatibility mode the `until` block should be a future block on which all nodes will + /// have upgraded to a release that includes the updated compatibility mode configuration. + /// At `until` block there will be a hard fork when the authority set changes, between the + /// old nodes (running with `initialize_block`, i.e. without the compatibility mode + /// configuration) and the new nodes. until: N, }, } @@ -143,7 +142,7 @@ fn slot_author(slot: Slot, authorities: &[AuthorityId

]) -> Option<&A } /// Parameters of [`start_aura`]. -pub struct StartAuraParams { +pub struct StartAuraParams { /// The duration of a slot. pub slot_duration: SlotDuration, /// The client to interact with the chain. @@ -165,9 +164,7 @@ pub struct StartAuraParams { /// The backoff strategy when we miss slots. pub backoff_authoring_blocks: Option, /// The keystore used by the node. - pub keystore: SyncCryptoStorePtr, - /// Can we author a block with this node? - pub can_author_with: CAW, + pub keystore: KeystorePtr, /// The proportion of the slot dedicated to proposing. /// /// The block proposing will be limited to this proportion of the slot from the starting of the @@ -186,7 +183,7 @@ pub struct StartAuraParams { } /// Start the aura worker. The returned future should be run in a futures executor. -pub fn start_aura( +pub fn start_aura( StartAuraParams { slot_duration, client, @@ -199,13 +196,12 @@ pub fn start_aura( force_authoring, backoff_authoring_blocks, keystore, - can_author_with, block_proposal_slot_portion, max_block_proposal_slot_portion, telemetry, compatibility_mode, - }: StartAuraParams>, -) -> Result, sp_consensus::Error> + }: StartAuraParams>, +) -> Result, ConsensusError> where P: Pair + Send + Sync, P::Public: AppPublic + Hash + Member + Encode + Decode, @@ -219,11 +215,10 @@ where PF::Proposer: Proposer>, SO: SyncOracle + Send + Sync + Clone, L: sc_consensus::JustificationSyncLink, - CIDP: CreateInherentDataProviders + Send, + CIDP: CreateInherentDataProviders + Send + 'static, CIDP::InherentDataProviders: InherentDataProviderExt + Send, BS: BackoffAuthoringBlocksStrategy> + Send + Sync + 'static, - CAW: CanAuthorWith + Send, - Error: std::error::Error + Send + From + 'static, + Error: std::error::Error + Send + From + 'static, { let worker = build_aura_worker::(BuildAuraWorkerParams { client, @@ -243,21 +238,16 @@ where Ok(sc_consensus_slots::start_slot_worker( slot_duration, select_chain, - worker, + SimpleSlotWorkerToSlotWorker(worker), sync_oracle, create_inherent_data_providers, - can_author_with, )) } /// Parameters of [`build_aura_worker`]. -pub struct BuildAuraWorkerParams { - /// The duration of a slot. - pub slot_duration: SlotDuration, +pub struct BuildAuraWorkerParams { /// The client to interact with the chain. pub client: Arc, - /// A select chain implementation to select the best block. - pub select_chain: SC, /// The block import. pub block_import: I, /// The proposer factory to build proposer instances. @@ -266,14 +256,12 @@ pub struct BuildAuraWorkerParams { pub sync_oracle: SO, /// Hook into the sync module to control the justification sync process. pub justification_sync_link: L, - /// Something that can create the inherent data providers. - pub create_inherent_data_providers: CIDP, /// Should we force the authoring of blocks? pub force_authoring: bool, /// The backoff strategy when we miss slots. pub backoff_authoring_blocks: Option, /// The keystore used by the node. - pub keystore: SyncCryptoStorePtr, + pub keystore: KeystorePtr, /// The proportion of the slot dedicated to proposing. /// /// The block proposing will be limited to this proportion of the slot from the starting of the @@ -309,15 +297,15 @@ pub fn build_aura_worker( force_authoring, compatibility_mode, }: BuildAuraWorkerParams>, -) -> impl sc_consensus_slots::SlotWorker>, - > - +) -> impl sc_consensus_slots::SimpleSlotWorker< + B, + Proposer = PF::Proposer, + BlockImport = I, + SyncOracle = SO, + JustificationSyncLink = L, + Claim = P::Public, + AuxData = Vec>, +> where B: BlockT, C: ProvideRuntimeApi + BlockOf + AuxStore + HeaderBackend + Send + Sync, @@ -333,7 +321,7 @@ where L: sc_consensus::JustificationSyncLink, BS: BackoffAuthoringBlocksStrategy> + Send + Sync + 'static, { - SimpleSlotWorkerToSlotWorker(AuraWorker { + AuraWorker { client, block_import, env: proposer_factory, @@ -347,14 +335,14 @@ where max_block_proposal_slot_portion, compatibility_mode, _key_type: PhantomData::

, - }) + } } struct AuraWorker { client: Arc, block_import: I, env: E, - keystore: SyncCryptoStorePtr, + keystore: KeystorePtr, sync_oracle: SO, justification_sync_link: L, force_authoring: bool, @@ -382,16 +370,16 @@ where SO: SyncOracle + Send + Clone + Sync, L: sc_consensus::JustificationSyncLink, BS: BackoffAuthoringBlocksStrategy> + Send + Sync + 'static, - Error: std::error::Error + Send + From + 'static, + Error: std::error::Error + Send + From + 'static, { type BlockImport = I; type SyncOracle = SO; type JustificationSyncLink = L; type CreateProposer = - Pin> + Send + 'static>>; + Pin> + Send + 'static>>; type Proposer = E::Proposer; type Claim = P::Public; - type EpochData = Vec>; + type AuxData = Vec>; fn logging_target(&self) -> &'static str { "aura" @@ -401,11 +389,7 @@ where &mut self.block_import } - fn epoch_data( - &self, - header: &B::Header, - _slot: Slot, - ) -> Result { + fn aux_data(&self, header: &B::Header, _slot: Slot) -> Result { authorities( self.client.as_ref(), header.hash(), @@ -414,31 +398,21 @@ where ) } - fn authorities_len(&self, epoch_data: &Self::EpochData) -> Option { - Some(epoch_data.len()) + fn authorities_len(&self, authorities: &Self::AuxData) -> Option { + Some(authorities.len()) } async fn claim_slot( &self, _header: &B::Header, slot: Slot, - epoch_data: &Self::EpochData, + authorities: &Self::AuxData, ) -> Option { - let expected_author = slot_author::

(slot, epoch_data); - expected_author.and_then(|p| { - if SyncCryptoStore::has_keys( - &*self.keystore, - &[(p.to_raw_vec(), sp_application_crypto::key_types::AURA)], - ) { - Some(p.clone()) - } else { - None - } - }) + crate::standalone::claim_slot::

(slot, authorities, &self.keystore).await } fn pre_digest_data(&self, slot: Slot, _claim: &Self::Claim) -> Vec { - vec![>::aura_pre_digest(slot)] + vec![crate::standalone::pre_digest::

(slot)] } async fn block_import_params( @@ -448,35 +422,13 @@ where body: Vec, storage_changes: StorageChanges<>::Transaction, B>, public: Self::Claim, - _epoch: Self::EpochData, + _authorities: Self::AuxData, ) -> Result< sc_consensus::BlockImportParams>::Transaction>, - sp_consensus::Error, + ConsensusError, > { - // sign the pre-sealed hash of the block and then - // add it to a digest item. - let public_type_pair = public.to_public_crypto_pair(); - let public = public.to_raw_vec(); - let signature = SyncCryptoStore::sign_with( - &*self.keystore, - as AppKey>::ID, - &public_type_pair, - header_hash.as_ref(), - ) - .map_err(|e| sp_consensus::Error::CannotSign(public.clone(), e.to_string()))? - .ok_or_else(|| { - sp_consensus::Error::CannotSign( - public.clone(), - "Could not find key in keystore.".into(), - ) - })?; - let signature = signature - .clone() - .try_into() - .map_err(|_| sp_consensus::Error::InvalidSignature(signature, public))?; - let signature_digest_item = - >::aura_seal(signature); + crate::standalone::seal::<_, P>(header_hash, &public, &self.keystore)?; let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); import_block.post_digests.push(signature_digest_item); @@ -518,7 +470,7 @@ where fn proposer(&mut self, block: &B::Header) -> Self::CreateProposer { self.env .init(block) - .map_err(|e| sp_consensus::Error::ClientImport(format!("{:?}", e))) + .map_err(|e| ConsensusError::ClientImport(format!("{:?}", e))) .boxed() } @@ -540,11 +492,6 @@ where } } -fn aura_err(error: Error) -> Error { - debug!(target: "aura", "{}", error); - error -} - /// Aura Errors #[derive(Debug, thiserror::Error)] pub enum Error { @@ -583,22 +530,13 @@ impl From> for String { } } -/// Get pre-digests from the header -pub fn find_pre_digest(header: &B::Header) -> Result> { - if header.number().is_zero() { - return Ok(0.into()) - } - - let mut pre_digest: Option = None; - for log in header.digest().logs() { - trace!(target: "aura", "Checking log {:?}", log); - match (CompatibleDigestItem::::as_aura_pre_digest(log), pre_digest.is_some()) { - (Some(_), true) => return Err(aura_err(Error::MultipleHeaders)), - (None, _) => trace!(target: "aura", "Ignoring digest not meant for us"), - (s, false) => pre_digest = s, +impl From for Error { + fn from(e: crate::standalone::PreDigestLookupError) -> Self { + match e { + crate::standalone::PreDigestLookupError::MultipleHeaders => Error::MultipleHeaders, + crate::standalone::PreDigestLookupError::NoDigestFound => Error::NoDigestFound, } } - pre_digest.ok_or_else(|| aura_err(Error::NoDigestFound)) } fn authorities( @@ -622,7 +560,7 @@ where if *until > context_block_number { runtime_api .initialize_block( - &BlockId::Hash(parent_hash), + parent_hash, &B::Header::new( context_block_number, Default::default(), @@ -631,13 +569,345 @@ where Default::default(), ), ) - .map_err(|_| sp_consensus::Error::InvalidAuthoritiesSet)?; + .map_err(|_| ConsensusError::InvalidAuthoritiesSet)?; }, } runtime_api - .authorities(&BlockId::Hash(parent_hash)) + .authorities(parent_hash) .ok() - .ok_or(sp_consensus::Error::InvalidAuthoritiesSet) + .ok_or(ConsensusError::InvalidAuthoritiesSet) } +#[cfg(test)] +mod tests { + use super::*; + use parking_lot::Mutex; + use sc_block_builder::BlockBuilderProvider; + use sc_client_api::BlockchainEvents; + use sc_consensus::BoxJustificationImport; + use sc_consensus_slots::{BackoffAuthoringOnFinalizedHeadLagging, SimpleSlotWorker}; + use sc_keystore::LocalKeystore; + use sc_network_test::{Block as TestBlock, *}; + use sp_application_crypto::{key_types::AURA, AppCrypto}; + use sp_consensus::{DisableProofRecording, NoNetwork as DummyOracle, Proposal}; + use sp_consensus_aura::sr25519::AuthorityPair; + use sp_inherents::InherentData; + use sp_keyring::sr25519::Keyring; + use sp_keystore::Keystore; + use sp_runtime::{ + traits::{Block as BlockT, Header as _}, + Digest, + }; + use sp_timestamp::Timestamp; + use std::{ + task::Poll, + time::{Duration, Instant}, + }; + use substrate_test_runtime_client::{ + runtime::{Header, H256}, + TestClient, + }; + + const SLOT_DURATION_MS: u64 = 1000; + + type Error = sp_blockchain::Error; + + struct DummyFactory(Arc); + struct DummyProposer(u64, Arc); + + impl Environment for DummyFactory { + type Proposer = DummyProposer; + type CreateProposer = futures::future::Ready>; + type Error = Error; + + fn init(&mut self, parent_header: &::Header) -> Self::CreateProposer { + futures::future::ready(Ok(DummyProposer(parent_header.number + 1, self.0.clone()))) + } + } + + impl Proposer for DummyProposer { + type Error = Error; + type Transaction = + sc_client_api::TransactionFor; + type Proposal = future::Ready, Error>>; + type ProofRecording = DisableProofRecording; + type Proof = (); + + fn propose( + self, + _: InherentData, + digests: Digest, + _: Duration, + _: Option, + ) -> Self::Proposal { + let r = self.1.new_block(digests).unwrap().build().map_err(|e| e.into()); + + future::ready(r.map(|b| Proposal { + block: b.block, + proof: (), + storage_changes: b.storage_changes, + })) + } + } + + type AuraVerifier = import_queue::AuraVerifier< + PeersFullClient, + AuthorityPair, + Box< + dyn CreateInherentDataProviders< + TestBlock, + (), + InherentDataProviders = (InherentDataProvider,), + >, + >, + u64, + >; + type AuraPeer = Peer<(), PeersClient>; + + #[derive(Default)] + pub struct AuraTestNet { + peers: Vec, + } + + impl TestNetFactory for AuraTestNet { + type Verifier = AuraVerifier; + type PeerData = (); + type BlockImport = PeersClient; + + fn make_verifier(&self, client: PeersClient, _peer_data: &()) -> Self::Verifier { + let client = client.as_client(); + let slot_duration = slot_duration(&*client).expect("slot duration available"); + + assert_eq!(slot_duration.as_millis() as u64, SLOT_DURATION_MS); + import_queue::AuraVerifier::new( + client, + Box::new(|_, _| async { + let slot = InherentDataProvider::from_timestamp_and_slot_duration( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MS), + ); + Ok((slot,)) + }), + CheckForEquivocation::Yes, + None, + CompatibilityMode::None, + ) + } + + fn make_block_import( + &self, + client: PeersClient, + ) -> ( + BlockImportAdapter, + Option>, + Self::PeerData, + ) { + (client.as_block_import(), None, ()) + } + + fn peer(&mut self, i: usize) -> &mut AuraPeer { + &mut self.peers[i] + } + + fn peers(&self) -> &Vec { + &self.peers + } + + fn peers_mut(&mut self) -> &mut Vec { + &mut self.peers + } + + fn mut_peers)>(&mut self, closure: F) { + closure(&mut self.peers); + } + } + + #[tokio::test] + async fn authoring_blocks() { + sp_tracing::try_init_simple(); + let net = AuraTestNet::new(3); + + let peers = &[(0, Keyring::Alice), (1, Keyring::Bob), (2, Keyring::Charlie)]; + + let net = Arc::new(Mutex::new(net)); + let mut import_notifications = Vec::new(); + let mut aura_futures = Vec::new(); + + let mut keystore_paths = Vec::new(); + for (peer_id, key) in peers { + let mut net = net.lock(); + let peer = net.peer(*peer_id); + let client = peer.client().as_client(); + let select_chain = peer.select_chain().expect("full client has a select chain"); + let keystore_path = tempfile::tempdir().expect("Creates keystore path"); + let keystore = Arc::new( + LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore."), + ); + + keystore + .sr25519_generate_new(AURA, Some(&key.to_seed())) + .expect("Creates authority key"); + keystore_paths.push(keystore_path); + + let environ = DummyFactory(client.clone()); + import_notifications.push( + client + .import_notification_stream() + .take_while(|n| { + future::ready(!(n.origin != BlockOrigin::Own && n.header.number() < &5)) + }) + .for_each(move |_| future::ready(())), + ); + + let slot_duration = slot_duration(&*client).expect("slot duration available"); + + aura_futures.push( + start_aura::(StartAuraParams { + slot_duration, + block_import: client.clone(), + select_chain, + client, + proposer_factory: environ, + sync_oracle: DummyOracle, + justification_sync_link: (), + create_inherent_data_providers: |_, _| async { + let slot = InherentDataProvider::from_timestamp_and_slot_duration( + Timestamp::current(), + SlotDuration::from_millis(SLOT_DURATION_MS), + ); + + Ok((slot,)) + }, + force_authoring: false, + backoff_authoring_blocks: Some( + BackoffAuthoringOnFinalizedHeadLagging::default(), + ), + keystore, + block_proposal_slot_portion: SlotProportion::new(0.5), + max_block_proposal_slot_portion: None, + telemetry: None, + compatibility_mode: CompatibilityMode::None, + }) + .expect("Starts aura"), + ); + } + + future::select( + future::poll_fn(move |cx| { + net.lock().poll(cx); + Poll::<()>::Pending + }), + future::select(future::join_all(aura_futures), future::join_all(import_notifications)), + ) + .await; + } + + #[tokio::test] + async fn current_node_authority_should_claim_slot() { + let net = AuraTestNet::new(4); + + let mut authorities = vec![ + Keyring::Alice.public().into(), + Keyring::Bob.public().into(), + Keyring::Charlie.public().into(), + ]; + + let keystore_path = tempfile::tempdir().expect("Creates keystore path"); + let keystore = LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore."); + let public = keystore + .sr25519_generate_new(AuthorityPair::ID, None) + .expect("Key should be created"); + authorities.push(public.into()); + + let net = Arc::new(Mutex::new(net)); + + let mut net = net.lock(); + let peer = net.peer(3); + let client = peer.client().as_client(); + let environ = DummyFactory(client.clone()); + + let worker = AuraWorker { + client: client.clone(), + block_import: client, + env: environ, + keystore: keystore.into(), + sync_oracle: DummyOracle, + justification_sync_link: (), + force_authoring: false, + backoff_authoring_blocks: Some(BackoffAuthoringOnFinalizedHeadLagging::default()), + telemetry: None, + _key_type: PhantomData::, + block_proposal_slot_portion: SlotProportion::new(0.5), + max_block_proposal_slot_portion: None, + compatibility_mode: Default::default(), + }; + + let head = Header::new( + 1, + H256::from_low_u64_be(0), + H256::from_low_u64_be(0), + Default::default(), + Default::default(), + ); + assert!(worker.claim_slot(&head, 0.into(), &authorities).await.is_none()); + assert!(worker.claim_slot(&head, 1.into(), &authorities).await.is_none()); + assert!(worker.claim_slot(&head, 2.into(), &authorities).await.is_none()); + assert!(worker.claim_slot(&head, 3.into(), &authorities).await.is_some()); + assert!(worker.claim_slot(&head, 4.into(), &authorities).await.is_none()); + assert!(worker.claim_slot(&head, 5.into(), &authorities).await.is_none()); + assert!(worker.claim_slot(&head, 6.into(), &authorities).await.is_none()); + assert!(worker.claim_slot(&head, 7.into(), &authorities).await.is_some()); + } + + #[tokio::test] + async fn on_slot_returns_correct_block() { + let net = AuraTestNet::new(4); + + let keystore_path = tempfile::tempdir().expect("Creates keystore path"); + let keystore = LocalKeystore::open(keystore_path.path(), None).expect("Creates keystore."); + keystore + .sr25519_generate_new(AuthorityPair::ID, Some(&Keyring::Alice.to_seed())) + .expect("Key should be created"); + + let net = Arc::new(Mutex::new(net)); + + let mut net = net.lock(); + let peer = net.peer(3); + let client = peer.client().as_client(); + let environ = DummyFactory(client.clone()); + + let mut worker = AuraWorker { + client: client.clone(), + block_import: client.clone(), + env: environ, + keystore: keystore.into(), + sync_oracle: DummyOracle, + justification_sync_link: (), + force_authoring: false, + backoff_authoring_blocks: Option::<()>::None, + telemetry: None, + _key_type: PhantomData::, + block_proposal_slot_portion: SlotProportion::new(0.5), + max_block_proposal_slot_portion: None, + compatibility_mode: Default::default(), + }; + + let head = client.expect_header(client.info().genesis_hash).unwrap(); + + let res = worker + .on_slot(SlotInfo { + slot: 0.into(), + ends_at: Instant::now() + Duration::from_secs(100), + create_inherent_data: Box::new(()), + duration: Duration::from_millis(1000), + chain_head: head, + block_size_limit: None, + }) + .await + .unwrap(); + + // The returned block should be imported and we should be able to get its header by now. + assert!(client.header(res.block.hash()).unwrap().is_some()); + } +} \ No newline at end of file