From a1cd89d521e3c31dd6eeef74be369d0a2cfb475a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:11:48 -0300 Subject: [PATCH] test: un-skip passing spectests (#874) Co-authored-by: Martin Paulucci --- test/spec/runners/epoch_processing.ex | 4 +- test/spec/runners/finality.ex | 5 ++ test/spec/runners/fork_choice.ex | 76 +++------------------ test/spec/runners/helpers/process_blocks.ex | 3 +- test/spec/runners/light_client.ex | 5 ++ test/spec/runners/random.ex | 26 +------ test/spec/runners/rewards.ex | 7 +- test/spec/runners/sanity.ex | 8 +++ test/spec/runners/ssz_generic.ex | 10 +-- test/spec/runners/ssz_static.ex | 30 +++++++- test/spec/runners/sync.ex | 7 +- 11 files changed, 73 insertions(+), 108 deletions(-) diff --git a/test/spec/runners/epoch_processing.ex b/test/spec/runners/epoch_processing.ex index bf1daefc1..561310128 100644 --- a/test/spec/runners/epoch_processing.ex +++ b/test/spec/runners/epoch_processing.ex @@ -38,9 +38,7 @@ defmodule EpochProcessingTestRunner do end @impl TestRunner - def skip?(%SpecTestCase{fork: "deneb"}) do - false - end + def skip?(%SpecTestCase{fork: "deneb"}), do: false @impl TestRunner def skip?(_), do: true diff --git a/test/spec/runners/finality.ex b/test/spec/runners/finality.ex index c5e0d2644..99f7a3cc3 100644 --- a/test/spec/runners/finality.ex +++ b/test/spec/runners/finality.ex @@ -19,6 +19,11 @@ defmodule FinalityTestRunner do Enum.member?(@disabled_cases, testcase) end + @impl TestRunner + def skip?(%SpecTestCase{fork: "deneb", case: testcase}) do + Enum.member?(@disabled_cases, testcase) + end + @impl TestRunner def skip?(_), do: true diff --git a/test/spec/runners/fork_choice.ex b/test/spec/runners/fork_choice.ex index 8c7448fba..4b3925b01 100644 --- a/test/spec/runners/fork_choice.ex +++ b/test/spec/runners/fork_choice.ex @@ -16,75 +16,19 @@ defmodule ForkChoiceTestRunner do use HardForkAliasInjection - @disabled_on_block_cases [ - # "basic", - # "incompatible_justification_update_end_of_epoch", - # "incompatible_justification_update_start_of_epoch", - # "justification_update_beginning_of_epoch", - # "justification_update_end_of_epoch", - # "justification_withholding", - # "justification_withholding_reverse_order", - # "justified_update_always_if_better", - # "justified_update_monotonic", - # "justified_update_not_realized_finality", - # "new_finalized_slot_is_justified_checkpoint_ancestor", - # "not_pull_up_current_epoch_block", - # "on_block_bad_parent_root", - # "on_block_before_finalized", - # "on_block_checkpoints", - # "on_block_finalized_skip_slots", - # "on_block_finalized_skip_slots_not_in_skip_chain", - # "on_block_future_block", - # "proposer_boost", - # "proposer_boost_root_same_slot_untimely_block", - # "pull_up_on_tick", - # "pull_up_past_epoch_block" - ] - - @disabled_ex_ante_cases [ - # "ex_ante_attestations_is_greater_than_proposer_boost_with_boost", - # "ex_ante_sandwich_with_boost_not_sufficient", - # "ex_ante_sandwich_with_honest_attestation", - # "ex_ante_sandwich_without_attestations", - # "ex_ante_vanilla" - ] - - @disabled_get_head_cases [ - # "chain_no_attestations", - # "discard_equivocations_on_attester_slashing", - # "discard_equivocations_slashed_validator_censoring", - # "filtered_block_tree", - # "genesis", - # "proposer_boost_correct_head", - # "shorter_chain_but_heavier_weight", - # "split_tie_breaker_no_attestations", - # "voting_source_beyond_two_epoch", - # "voting_source_within_two_epoch" - ] - - @disabled_reorg_cases [ - # "delayed_justification_current_epoch", - # "delayed_justification_previous_epoch", - # "include_votes_another_empty_chain_with_enough_ffg_votes_current_epoch", - # "include_votes_another_empty_chain_with_enough_ffg_votes_previous_epoch", - # "include_votes_another_empty_chain_without_enough_ffg_votes_current_epoch", - # "simple_attempted_reorg_delayed_justification_current_epoch", - # "simple_attempted_reorg_delayed_justification_previous_epoch", - # "simple_attempted_reorg_without_enough_ffg_votes" - ] - - @disabled_withholding_cases [ - # "withholding_attack", - # "withholding_attack_unviable_honest_chain" + # TODO: implement blob checks + @disabled_deneb [ + "invalid_data_unavailable", + "invalid_wrong_proofs_length", + "invalid_incorrect_proof", + "invalid_wrong_blobs_length" ] @impl TestRunner - def skip?(%SpecTestCase{fork: "capella", case: testcase}) do - Enum.member?(@disabled_on_block_cases, testcase) or - Enum.member?(@disabled_ex_ante_cases, testcase) or - Enum.member?(@disabled_get_head_cases, testcase) or - Enum.member?(@disabled_reorg_cases, testcase) or - Enum.member?(@disabled_withholding_cases, testcase) + def skip?(%SpecTestCase{fork: "capella"}), do: false + + def skip?(%SpecTestCase{fork: "deneb", case: testcase}) do + Enum.member?(@disabled_deneb, testcase) end def skip?(_testcase), do: true diff --git a/test/spec/runners/helpers/process_blocks.ex b/test/spec/runners/helpers/process_blocks.ex index a59867d9d..8340f8d8f 100644 --- a/test/spec/runners/helpers/process_blocks.ex +++ b/test/spec/runners/helpers/process_blocks.ex @@ -8,6 +8,7 @@ defmodule Helpers.ProcessBlocks do alias LambdaEthereumConsensus.StateTransition alias LambdaEthereumConsensus.Utils.Diff alias Types.BeaconState + alias Types.SignedBeaconBlock use HardForkAliasInjection @@ -27,7 +28,7 @@ defmodule Helpers.ProcessBlocks do |> Enum.map(fn index -> SpecTestUtils.read_ssz_from_file!( case_dir <> "/blocks_#{index}.ssz_snappy", - Types.SignedBeaconBlock + SignedBeaconBlock ) end) diff --git a/test/spec/runners/light_client.ex b/test/spec/runners/light_client.ex index eae0e9bbb..01620af0d 100644 --- a/test/spec/runners/light_client.ex +++ b/test/spec/runners/light_client.ex @@ -19,6 +19,11 @@ defmodule LightClientTestRunner do Enum.member?(@disabled_handlers, testcase.handler) end + def skip?(%SpecTestCase{fork: "deneb"} = _testcase) do + # TODO: all of them fail + true + end + @impl TestRunner def skip?(_testcase) do true diff --git a/test/spec/runners/random.ex b/test/spec/runners/random.ex index a2ede505d..5fd9a957b 100644 --- a/test/spec/runners/random.ex +++ b/test/spec/runners/random.ex @@ -6,31 +6,9 @@ defmodule RandomTestRunner do use ExUnit.CaseTemplate use TestRunner - @disabled_cases [ - # "randomized_0", - # "randomized_1", - # "randomized_2", - # "randomized_3", - # "randomized_4", - # "randomized_5", - # "randomized_6", - # "randomized_7", - # "randomized_8", - # "randomized_9", - # "randomized_10", - # "randomized_11", - # "randomized_12", - # "randomized_13", - # "randomized_14", - # "randomized_15" - ] - - @impl TestRunner - def skip?(%SpecTestCase{fork: "capella", case: testcase}) do - Enum.member?(@disabled_cases, testcase) - end - @impl TestRunner + def skip?(%SpecTestCase{fork: "capella"}), do: false + def skip?(%SpecTestCase{fork: "deneb"}), do: false def skip?(_), do: true @impl TestRunner diff --git a/test/spec/runners/rewards.ex b/test/spec/runners/rewards.ex index adea7bb80..3795a8af3 100644 --- a/test/spec/runners/rewards.ex +++ b/test/spec/runners/rewards.ex @@ -15,10 +15,13 @@ defmodule RewardsTestRunner do ] @impl TestRunner - def skip?(%SpecTestCase{fork: fork, handler: handler}) do - fork != "capella" or Enum.member?(@disabled, handler) + def skip?(%SpecTestCase{fork: "capella", handler: handler}) do + Enum.member?(@disabled, handler) end + def skip?(%SpecTestCase{fork: "deneb"}), do: false + def skip?(_), do: true + @impl TestRunner def run_test_case(%SpecTestCase{} = testcase) do case_dir = SpecTestCase.dir(testcase) diff --git a/test/spec/runners/sanity.ex b/test/spec/runners/sanity.ex index 28402cd42..6e74859db 100644 --- a/test/spec/runners/sanity.ex +++ b/test/spec/runners/sanity.ex @@ -105,6 +105,14 @@ defmodule SanityTestRunner do Enum.member?(@disabled_slot_cases, testcase) end + def skip?(%SpecTestCase{fork: "deneb", handler: "blocks", case: testcase}) do + Enum.member?(@disabled_block_cases, testcase) + end + + def skip?(%SpecTestCase{fork: "deneb", handler: "slots", case: testcase}) do + Enum.member?(@disabled_slot_cases, testcase) + end + def skip?(_), do: true @impl TestRunner diff --git a/test/spec/runners/ssz_generic.ex b/test/spec/runners/ssz_generic.ex index 55c7b6007..03416f2c7 100644 --- a/test/spec/runners/ssz_generic.ex +++ b/test/spec/runners/ssz_generic.ex @@ -6,7 +6,7 @@ defmodule SszGenericTestRunner do use ExUnit.CaseTemplate use TestRunner - @disabled [ + @disabled_handlers [ # "basic_vector", # "bitlist", # "bitvector" @@ -26,12 +26,8 @@ defmodule SszGenericTestRunner do @impl TestRunner def skip?(%SpecTestCase{fork: fork, handler: handler, case: cse}) do - skip_container? = - @disabled_containers - |> Enum.map(fn container -> String.contains?(cse, container) end) - |> Enum.any?() - - fork != "phase0" or Enum.member?(@disabled, handler) or skip_container? + skip_container? = Enum.any?(@disabled_containers, &String.contains?(cse, &1)) + fork != "phase0" or Enum.member?(@disabled_handlers, handler) or skip_container? end @impl TestRunner diff --git a/test/spec/runners/ssz_static.ex b/test/spec/runners/ssz_static.ex index 3b0f073e5..ceff8ec75 100644 --- a/test/spec/runners/ssz_static.ex +++ b/test/spec/runners/ssz_static.ex @@ -5,6 +5,14 @@ defmodule SszStaticTestRunner do alias LambdaEthereumConsensus.SszEx alias LambdaEthereumConsensus.Utils.Diff alias Ssz + alias Types.BeaconBlock + alias Types.BeaconBlockBody + alias Types.BeaconState + alias Types.ExecutionPayload + alias Types.ExecutionPayloadHeader + alias Types.SignedBeaconBlock + + use HardForkAliasInjection use ExUnit.CaseTemplate use TestRunner @@ -60,11 +68,27 @@ defmodule SszStaticTestRunner do "SyncCommitteeMessage" ] + @type_map %{ + "BeaconBlock" => BeaconBlock, + "BeaconBlockBody" => BeaconBlockBody, + "BeaconState" => BeaconState, + "ExecutionPayload" => ExecutionPayload, + "ExecutionPayloadHeader" => ExecutionPayloadHeader, + "SignedBeaconBlock" => SignedBeaconBlock + } + @impl TestRunner - def skip?(%SpecTestCase{fork: fork, handler: handler}) do - fork != "capella" or Enum.member?(@disabled, handler) + def skip?(%SpecTestCase{fork: "capella", handler: handler}) do + Enum.member?(@disabled, handler) end + def skip?(%SpecTestCase{fork: "deneb", handler: handler}) do + # TODO: fix types + Enum.member?(@disabled, handler) + end + + def skip?(_), do: true + @impl TestRunner def run_test_case(%SpecTestCase{} = testcase) do case_dir = SpecTestCase.dir(testcase) @@ -107,6 +131,6 @@ defmodule SszStaticTestRunner do end defp parse_type(%SpecTestCase{handler: handler}) do - Module.concat(Types, handler) + Map.get(@type_map, handler, Module.concat(Types, handler)) end end diff --git a/test/spec/runners/sync.ex b/test/spec/runners/sync.ex index dab32d54e..f27370d74 100644 --- a/test/spec/runners/sync.ex +++ b/test/spec/runners/sync.ex @@ -18,11 +18,14 @@ defmodule SyncTestRunner do Enum.member?(@disabled_cases, testcase.case) end - @impl TestRunner - def skip?(_testcase) do + def skip?(%SpecTestCase{fork: "deneb"}) do + # TODO: update `EngineApiMock` to support the new `new_payload/3` function, + # and the runner to load the block's blobs if on deneb true end + def skip?(_testcase), do: true + @impl TestRunner def run_test_case(%SpecTestCase{} = testcase) do original_engine_api_config =