Skip to content

Commit

Permalink
test: add finality, sync tests and enable several sanity cases. (#487)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mpaulucci
mpaulucci authored Nov 30, 2023
1 parent 914cb3b commit 9e6b9b4
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 106 deletions.
26 changes: 26 additions & 0 deletions lib/spec/runners/finality.ex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
defmodule FinalityTestRunner do
@moduledoc """
Runner for finality test cases. See: https://github.com/ethereum/consensus-specs/tree/dev/tests/formats/finality
"""

use ExUnit.CaseTemplate
use TestRunner

@disabled_cases [
# "finality_no_updates_at_genesis",
# "finality_rule_1",
# "finality_rule_2",
# "finality_rule_3",
# "finality_rule_4"
]

@impl TestRunner
def skip?(%SpecTestCase{fork: "capella", case: testcase}) do
Enum.member?(@disabled_cases, testcase)
end

@impl TestRunner
def run_test_case(testcase) do
Helpers.ProcessBlocks.process_blocks(testcase)
end
end
57 changes: 57 additions & 0 deletions lib/spec/runners/helpers/process_blocks.ex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
defmodule Helpers.ProcessBlocks do
@moduledoc """
Helper module for processing blocks.
"""

use ExUnit.CaseTemplate

alias LambdaEthereumConsensus.StateTransition
alias LambdaEthereumConsensus.Utils.Diff

def process_blocks(%SpecTestCase{} = testcase) do
case_dir = SpecTestCase.dir(testcase)

pre =
SpecTestUtils.read_ssz_from_file!(
case_dir <> "/pre.ssz_snappy",
SszTypes.BeaconState
)

post =
SpecTestUtils.read_ssz_from_optional_file!(
case_dir <> "/post.ssz_snappy",
SszTypes.BeaconState
)

meta =
YamlElixir.read_from_file!(case_dir <> "/meta.yaml") |> SpecTestUtils.sanitize_yaml()

%{blocks_count: blocks_count} = meta

blocks =
0..(blocks_count - 1)//1
|> Enum.map(fn index ->
SpecTestUtils.read_ssz_from_file!(
case_dir <> "/blocks_#{index}.ssz_snappy",
SszTypes.SignedBeaconBlock
)
end)

result =
blocks
|> Enum.reduce_while({:ok, pre}, fn block, {:ok, state} ->
case StateTransition.state_transition(state, block, true) do
{:ok, post_state} -> {:cont, {:ok, post_state}}
{:error, error} -> {:halt, {:error, error}}
end
end)

case result do
{:ok, state} ->
assert Diff.diff(state, post) == :unchanged

{:error, error} ->
assert post == nil, "Process block failed, error: #{error}"
end
end
end
169 changes: 63 additions & 106 deletions lib/spec/runners/sanity.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,82 +10,83 @@ defmodule SanityTestRunner do
alias LambdaEthereumConsensus.Utils.Diff

@disabled_block_cases [
"activate_and_partial_withdrawal_max_effective_balance",
"activate_and_partial_withdrawal_overdeposit",
"attestation",
"attester_slashing",
"balance_driven_status_transitions",
"bls_change",
"deposit_and_bls_change",
"deposit_in_block",
"deposit_top_up",
"duplicate_attestation_same_block",
"empty_block_transition",
"empty_block_transition_large_validator_set",
"empty_block_transition_no_tx",
"empty_block_transition_randomized_payload",
"empty_epoch_transition",
"empty_epoch_transition_large_validator_set",
"empty_epoch_transition_not_finalizing",
"eth1_data_votes_consensus",
"eth1_data_votes_no_consensus",
"exit_and_bls_change",
"full_random_operations_0",
"full_random_operations_1",
"full_random_operations_2",
"full_random_operations_3",
"full_withdrawal_in_epoch_transition",
"high_proposer_index",
"historical_batch",
"inactivity_scores_full_participation_leaking",
"inactivity_scores_leaking",
# "activate_and_partial_withdrawal_max_effective_balance",
# "activate_and_partial_withdrawal_overdeposit",
# "attestation",
# "attester_slashing",
# "balance_driven_status_transitions",
# "bls_change",
# "deposit_and_bls_change",
# "deposit_in_block",
# "deposit_top_up",
# "duplicate_attestation_same_block",
# "invalid_is_execution_enabled_false",
# "empty_block_transition",
# "empty_block_transition_large_validator_set",
# "empty_block_transition_no_tx",
# "empty_block_transition_randomized_payload",
# "empty_epoch_transition",
# "empty_epoch_transition_large_validator_set",
# "empty_epoch_transition_not_finalizing",
# "eth1_data_votes_consensus",
# "eth1_data_votes_no_consensus",
# "exit_and_bls_change",
# "full_random_operations_0",
# "full_random_operations_1",
# "full_random_operations_2",
# "full_random_operations_3",
# "full_withdrawal_in_epoch_transition",
# "high_proposer_index",
# "historical_batch",
# "inactivity_scores_full_participation_leaking",
# "inactivity_scores_leaking",
"invalid_all_zeroed_sig",
"invalid_duplicate_attester_slashing_same_block",
"invalid_duplicate_bls_changes_same_block",
"invalid_duplicate_deposit_same_block",
# "invalid_duplicate_deposit_same_block",
"invalid_duplicate_proposer_slashings_same_block",
"invalid_duplicate_validator_exit_same_block",
"invalid_incorrect_block_sig",
"invalid_incorrect_proposer_index_sig_from_expected_proposer",
"invalid_incorrect_proposer_index_sig_from_proposer_index",
# "invalid_incorrect_proposer_index_sig_from_expected_proposer",
# "invalid_incorrect_proposer_index_sig_from_proposer_index",
"invalid_incorrect_state_root",
"invalid_only_increase_deposit_count",
"invalid_parent_from_same_slot",
"invalid_prev_slot_block_transition",
"invalid_same_slot_block_transition",
# "invalid_only_increase_deposit_count",
# "invalid_parent_from_same_slot",
# "invalid_prev_slot_block_transition",
# "invalid_same_slot_block_transition",
"invalid_similar_proposer_slashings_same_block",
"invalid_two_bls_changes_of_different_addresses_same_validator_same_block",
"invalid_withdrawal_fail_second_block_payload_isnt_compatible",
"is_execution_enabled_false",
"many_partial_withdrawals_in_epoch_transition",
"multiple_attester_slashings_no_overlap",
"multiple_attester_slashings_partial_overlap",
"multiple_different_proposer_slashings_same_block",
"multiple_different_validator_exits_same_block",
"partial_withdrawal_in_epoch_transition",
"proposer_after_inactive_index",
"proposer_self_slashing",
"proposer_slashing",
"skipped_slots",
"slash_and_exit_diff_index",
"slash_and_exit_same_index",
"sync_committee_committee__empty",
"sync_committee_committee__full",
"sync_committee_committee__half",
"sync_committee_committee_genesis__empty",
"sync_committee_committee_genesis__full",
"sync_committee_committee_genesis__half",
"top_up_and_partial_withdrawable_validator",
"top_up_to_fully_withdrawn_validator",
"voluntary_exit",
"withdrawal_success_two_blocks"
# "invalid_withdrawal_fail_second_block_payload_isnt_compatible",
# "is_execution_enabled_false",
# "many_partial_withdrawals_in_epoch_transition",
# "multiple_attester_slashings_no_overlap",
# "multiple_attester_slashings_partial_overlap",
# "multiple_different_proposer_slashings_same_block",
# "multiple_different_validator_exits_same_block",
# "partial_withdrawal_in_epoch_transition",
# "proposer_after_inactive_index",
# "proposer_self_slashing",
# "proposer_slashing",
# "skipped_slots",
# "slash_and_exit_diff_index",
"slash_and_exit_same_index"
# "sync_committee_committee__empty",
# "sync_committee_committee__full",
# "sync_committee_committee__half",
# "sync_committee_committee_genesis__empty",
# "sync_committee_committee_genesis__full",
# "sync_committee_committee_genesis__half",
# "top_up_and_partial_withdrawable_validator",
# "top_up_to_fully_withdrawn_validator",
# "voluntary_exit",
# "withdrawal_success_two_blocks"
]

@disabled_slot_cases [
# "empty_epoch",
# "slots_1",
# "slots_2",
"over_epoch_boundary",
# "over_epoch_boundary",
"historical_accumulator",
"double_empty_epoch"
]
Expand Down Expand Up @@ -135,50 +136,6 @@ defmodule SanityTestRunner do
@impl TestRunner
def run_test_case(%SpecTestCase{handler: "blocks"} = testcase) do
# TODO process meta.yaml
case_dir = SpecTestCase.dir(testcase)

pre =
SpecTestUtils.read_ssz_from_file!(
case_dir <> "/pre.ssz_snappy",
SszTypes.BeaconState
)

post =
SpecTestUtils.read_ssz_from_optional_file!(
case_dir <> "/post.ssz_snappy",
SszTypes.BeaconState
)

meta =
YamlElixir.read_from_file!(case_dir <> "/meta.yaml") |> SpecTestUtils.sanitize_yaml()

%{blocks_count: blocks_count} = meta

blocks =
0..(blocks_count - 1)//1
|> Enum.map(fn index ->
SpecTestUtils.read_ssz_from_file!(
case_dir <> "/blocks_#{index}.ssz_snappy",
SszTypes.SignedBeaconBlock
)
end)
|> Enum.map(& &1.message)

result =
blocks
|> Enum.reduce_while({:ok, pre}, fn block, {:ok, state} ->
case StateTransition.process_block(state, block) do
{:ok, post_state} -> {:cont, {:ok, post_state}}
{:error, error} -> {:halt, {:error, error}}
end
end)

case result do
{:ok, state} ->
assert Diff.diff(state, post) == :unchanged

{:error, error} ->
assert post == nil, "Process block failed, error: #{error}"
end
Helpers.ProcessBlocks.process_blocks(testcase)
end
end
22 changes: 22 additions & 0 deletions lib/spec/runners/sync.ex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
defmodule SyncTestRunner do
@moduledoc """
Runner for Operations test cases. See: https://github.com/ethereum/consensus-specs/tree/dev/tests/formats/sync
"""

use ExUnit.CaseTemplate
use TestRunner

@disabled_cases [
"from_syncing_to_invalid"
]

@impl TestRunner
def skip?(%SpecTestCase{} = testcase) do
Enum.member?(@disabled_cases, testcase.case)
end

@impl TestRunner
def run_test_case(%SpecTestCase{} = testcase) do
ForkChoiceTestRunner.run_test_case(testcase)
end
end

0 comments on commit 9e6b9b4

Please sign in to comment.