From a7ad4ae0324bc2e50d9191b3259444c9baa5a6b1 Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Wed, 16 Jun 2021 15:09:33 -0700 Subject: [PATCH] feat: add documentation on block processing --- specs/ChainSpec/BlockProcessing.md | 87 ++++++++++++++ specs/DataStructures/Block.md | 179 +++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 specs/ChainSpec/BlockProcessing.md create mode 100644 specs/DataStructures/Block.md diff --git a/specs/ChainSpec/BlockProcessing.md b/specs/ChainSpec/BlockProcessing.md new file mode 100644 index 000000000..ae81bd7b4 --- /dev/null +++ b/specs/ChainSpec/BlockProcessing.md @@ -0,0 +1,87 @@ +# Block Processing + +This section covers how blocks are processed once they arrive from the network. + +## Data Structures + +Please refer to [this section](../DataStructures/Block.md) for details about blocks and chunks. + +## Validity of Block Header + +A block header is invalid if any of the following holds: +- `timestamp` is invalid due to one of the following: + * It is more than 120s ahead of the local time of the machine. + * It is smaller than the timestamp of the previous block. +- Its signature is not valid, i.e, verifying the signature using the public key of the block producer fails +- `epoch_id` is invalid, i.e, it does not match the epoch id of the block computed locally. +- `next_bp_hash` is invalid. This could mean one of the following: + * `epoch_id == prev_block.epoch_id && next_bp_hash != prev_block.next_bp_hash` + * `epoch_id != prev_block.epoch_id && next_bp_hash != compute_bp_hash(next_epoch_id, prev_hash)` where `compute_bp_hash` computes the hash of next epoch's validators. +- `chunk_mask` does not match the size of the chunk mask is not the same as the number of shards +- Approval or finality information is invalid. See [consensus](Consensus.md) for more details. + + +## Validity of Block + +A block is invalid if any of the following holds: +- Any of the chunk headers included in the block has an invalid signature. +- State root computed from chunk headers does not match `state_root` in the header. +- Receipts root computed from chunk headers does not match `chunk_receipts_root` in the header. +- Chunk headers root computed from chunk headers does not match `chunk_headers_root` in the header. +- Transactions root computed from chunk headers does not match `chunk_tx_root` in the header. +- For some index `i`, `chunk_mask[i]` does not match whether a new chunk from shard `i` is included in the block. +- Its vrf output is invalid +- Its gas price is invalid, i.e, gas priced computed from previous gas price and gas usage from chunks included in the block according to the formula described in [economoics](../Economics/README.md) does not match the gas price in the header. +- Its `validator_proposals` is not valid, which means that it does not match the concatenation of validator proposals from the chunk headers included in the block. + +## Process a new block + +When a new block `B` is received from the network, the node first check whether it is ready to be processed: +- Its parent has already been processed. +- Header and the block itself are valid +- All the chunk parts are received. More specifically, this means + * For any shard that the node tracks, if there is a new chunk from that shard included in `B`, the node has the entire chunk. + * For block producers, they have their chunk parts for shards they do not track to guarantee data availability. + +Once all the checks are done, the chunks included in `B` can be applied. + +If the chunks are successfully applied, `B` is saved and if the height of `B` is higher than the highest known block (head of the chain), +the head is updated. + +## Apply a chunk + +In order to apply a chunk, we first check that the chunk is valid: + +```python +def validate_chunk(chunk): + # get the local result of applying the previous chunk (chunk_extra) + prev_chunk_extra = get_chunk_extra(chunk.prev_block_hash, chunk.shard_id) + # check that the post state root of applying the previous chunk matches the prev state root in the current chunk + assert prev_chunk_extra.state_root == chunk.state_root + # check that the merkle root of execution outcomes match + assert prev_chunk_extra.outcome_root == chunk.outcome_root + # check that validator proposals match + assert prev_chunk_extra.validator_proposals == chunk.validator_proposals + # check that gas usage matches + assert prev_chunk_extra.gas_used == chunk.gas_used + # check that balance burnt matches + assert prev_chunk_extra.balance_burnt == chunk.balance_burnt + # check outgoing receipt root matches + assert prev_chunk_extra.outgoing_receipt_root == chunk.outgoing_receipt_root +``` + +After this we apply transactions and receipts: +```python +# get the incoming receipts for this shard +incoming_receipts = get_incoming_receipts(block_hash, shard_id) +# apply transactions and receipts and obtain the result, which includes state changes and execution outcomes +apply_result = apply_transactions(shard_id, state_root, chunk.transactions, incoming_receipts, other_block_info) +# save apply result locally +save_result(apply_result) +``` + +## Catchup + +If a node validates shard `X` in epoch `T` and needs to validate shard `Y` in epoch `T+1` due to validator rotation, it has to download the state of that shard before epoch `T+1` starts to be able to do so. +To accomplish this, the node will start downloading the state of shard `Y` at the beginning of epoch `T` and, after it has successfully downloaded the state, will apply all the chunks for shard `Y` in epoch `T` until the current block. +From there the node will apply chunks for both shard `X` and shard `Y` for the rest of the epoch. diff --git a/specs/DataStructures/Block.md b/specs/DataStructures/Block.md new file mode 100644 index 000000000..39786d078 --- /dev/null +++ b/specs/DataStructures/Block.md @@ -0,0 +1,179 @@ +# Block and Block Header + +The data structures for block and block headers are + +```rust +pub struct Block { + /// Block Header + pub header: BlockHeader, + /// Headers of chunk in the block + pub chunks: Vec, + /// Challenges, but they are not used today + pub challenges: Challenges, + + /// Data to confirm the correctness of randomness beacon output + pub vrf_value: [u8; 32], + pub vrf_proof: [u8; 64], +} +``` + +```rust +pub struct BlockHeader { + pub prev_hash: CryptoHash, + + /// Inner part of the block header that gets hashed, split into two parts, one that is sent + /// to light clients, and the rest + pub inner_lite: BlockHeaderInnerLite, + pub inner_rest: BlockHeaderInnerRest, + + /// Signature of the block producer. + pub signature: Signature, + + /// Cached value of hash for this block. + pub hash: CryptoHash, +} +``` + +where `BlockHeaderInnerLite` and `BlockHeaderInnerRest` are + +```rust +pub struct BlockHeaderInnerLite { + /// Height of this block. + pub height: u64, + /// Epoch start hash of this block's epoch. + /// Used for retrieving validator information + pub epoch_id: EpochId, + pub next_epoch_id: EpochId, + /// Root hash of the state at the previous block. + pub prev_state_root: CryptoHash, + /// Root of the outcomes of transactions and receipts. + pub outcome_root: CryptoHash, + /// Timestamp at which the block was built (number of non-leap-nanoseconds since January 1, 1970 0:00:00 UTC). + pub timestamp: u64, + /// Hash of the next epoch block producers set + pub next_bp_hash: CryptoHash, + /// Merkle root of block hashes up to the current block. + pub block_merkle_root: CryptoHash, +} +``` + +```rust +pub struct BlockHeaderInnerRest { + /// Root hash of the chunk receipts in the given block. + pub chunk_receipts_root: CryptoHash, + /// Root hash of the chunk headers in the given block. + pub chunk_headers_root: CryptoHash, + /// Root hash of the chunk transactions in the given block. + pub chunk_tx_root: CryptoHash, + /// Root hash of the challenges in the given block. + pub challenges_root: CryptoHash, + /// The output of the randomness beacon + pub random_value: CryptoHash, + /// Validator proposals. + pub validator_proposals: Vec, + /// Mask for new chunks included in the block + pub chunk_mask: Vec, + /// Gas price. Same for all chunks + pub gas_price: u128, + /// Total supply of tokens in the system + pub total_supply: u128, + /// List of challenges result from previous block. + pub challenges_result: ChallengesResult, + + /// Last block that has full BFT finality + pub last_final_block: CryptoHash, + /// Last block that has doomslug finality + pub last_ds_final_block: CryptoHash, + + /// The ordinal of the Block on the Canonical Chain + pub block_ordinal: u64, + + /// All the approvals included in this block + pub approvals: Vec>, + + /// Latest protocol version that this block producer has. + pub latest_protocol_version: u32, +} +``` + +Here `CryptoHash` is a 32-byte hash and `EpochId` is a 32-byte identifier. + +## Block Hash + +The hash of a block is computed by +```rust +sha256(concat( + sha256(concat( + sha256(borsh(inner_lite)), + sha256(borsh(inner_rest)) + )), + prev_hash +)) +``` + +# Chunk and Chunk Header + +The data structures for chunk and chunk header are + +```rust +pub struct ShardChunkHeader { + pub inner: ShardChunkHeaderInner, + + pub height_included: BlockHeight, + + /// Signature of the chunk producer. + pub signature: Signature, + + pub hash: ChunkHash, +} +``` + +```rust +pub struct ShardChunk { + pub chunk_hash: ChunkHash, + pub header: ShardChunkHeader, + pub transactions: Vec, + pub receipts: Vec, +} +``` + +where `ShardChunkHeaderInner` is + +```rust +pub struct ShardChunkHeaderInner { + /// Previous block hash. + pub prev_block_hash: CryptoHash, + pub prev_state_root: CryptoHash, + /// Root of the outcomes from execution transactions and results. + pub outcome_root: CryptoHash, + pub encoded_merkle_root: CryptoHash, + pub encoded_length: u64, + pub height_created: u64, + /// Shard index. + pub shard_id: u64, + /// Gas used in this chunk. + pub gas_used: u64, + /// Gas limit voted by validators. + pub gas_limit: u64, + /// Total balance burnt in previous chunk + pub balance_burnt: u128, + /// Outgoing receipts merkle root. + pub outgoing_receipts_root: CryptoHash, + /// Tx merkle root. + pub tx_root: CryptoHash, + /// Validator proposals. + pub validator_proposals: Vec, +} +``` + +## Chunk Hash + +Chunk hash is computed by +```rust +sha256( + concat( + sha256(borsh(inner)), + encoded_merkle_root + ) +) +```