From 30f853857c56a903fd97543949152bf1fe0dffa5 Mon Sep 17 00:00:00 2001 From: Fernando Ledesma Date: Mon, 11 Dec 2023 12:32:45 +0000 Subject: [PATCH 1/3] feat: implement `ssz_generic` spec tests (#490) --- .../ssz_static_containers/bits_struct.ex | 36 ++++++ .../complex_test_struct.ex | 44 +++++++ .../fixed_test_struct.ex | 32 +++++ .../single_field_test_struct.ex | 24 ++++ .../small_test_struct.ex | 27 ++++ .../ssz_static_containers/var_test_struct.ex | 30 +++++ lib/spec/runners/ssz_generic.ex | 121 ++++++++++++++++++ lib/ssz_types/mod.ex | 1 + 8 files changed, 315 insertions(+) create mode 100644 lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex create mode 100644 lib/spec/runners/ssz_generic.ex diff --git a/lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex new file mode 100644 index 000000000..a23932c75 --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex @@ -0,0 +1,36 @@ +defmodule Helpers.SszStaticContainers.BitsStruct do + @moduledoc """ + Struct definition for `BitsStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C, + :D, + :E + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.bitlist(), + B: SszTypes.bitvector(), + C: SszTypes.bitvector(), + D: SszTypes.bitlist(), + E: SszTypes.bitvector() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:bitlist, 5}}, + {:B, {:bitvector, 2}}, + {:C, {:bitvector, 1}}, + {:D, {:bitlist, 6}}, + {:E, {:bitvector, 8}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex new file mode 100644 index 000000000..ff85ca3cc --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex @@ -0,0 +1,44 @@ +defmodule Helpers.SszStaticContainers.ComplexTestStruct do + @moduledoc """ + Struct definition for `ComplexTestStruct`. + """ + alias Helpers.SszStaticContainers.FixedTestStruct + alias Helpers.SszStaticContainers.VarTestStruct + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C, + :D, + :E, + :F, + :G + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint16(), + B: list(SszTypes.uint16()), + C: SszTypes.uint8(), + D: SszTypes.bitlist(), + E: VarTestStruct, + F: list(FixedTestStruct), + G: list(VarTestStruct) + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 16}}, + {:B, {:list, {:int, 16}, 1024}}, + {:C, {:int, 8}}, + {:D, {:bitlist, 256}}, + {:E, VarTestStruct}, + {:F, {:vector, FixedTestStruct, 4}}, + {:G, {:vector, VarTestStruct, 2}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex new file mode 100644 index 000000000..f8503e43d --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex @@ -0,0 +1,32 @@ +defmodule Helpers.SszStaticContainers.FixedTestStruct do + @moduledoc """ + Struct definition for `FixedTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C + ] + + @type uint32 :: 0..unquote(2 ** 32 - 1) + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint8(), + B: SszTypes.uint64(), + C: uint32() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 8}}, + {:B, {:int, 64}}, + {:C, {:int, 32}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex new file mode 100644 index 000000000..6d6655bfc --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex @@ -0,0 +1,24 @@ +defmodule Helpers.SszStaticContainers.SingleFieldTestStruct do + @moduledoc """ + Struct definition for `SingleFieldTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint8() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 8}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex new file mode 100644 index 000000000..b54f8e8d7 --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex @@ -0,0 +1,27 @@ +defmodule Helpers.SszStaticContainers.SmallTestStruct do + @moduledoc """ + Struct definition for `SmallTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint16(), + B: SszTypes.uint16() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 16}}, + {:B, {:int, 16}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex new file mode 100644 index 000000000..60d750e41 --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex @@ -0,0 +1,30 @@ +defmodule Helpers.SszStaticContainers.VarTestStruct do + @moduledoc """ + Struct definition for `VarTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint16(), + B: list(SszTypes.uint16()), + C: SszTypes.uint8() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 16}}, + {:B, {:list, {:int, 16}, 1024}}, + {:C, {:int, 8}} + ] + end +end diff --git a/lib/spec/runners/ssz_generic.ex b/lib/spec/runners/ssz_generic.ex new file mode 100644 index 000000000..75a99e13b --- /dev/null +++ b/lib/spec/runners/ssz_generic.ex @@ -0,0 +1,121 @@ +defmodule SszGenericTestRunner do + @moduledoc """ + Runner for SSZ general test cases. `run_test_case/1` is the main entrypoint. + """ + alias LambdaEthereumConsensus.SszEx + use ExUnit.CaseTemplate + use TestRunner + + @disabled [ + "basic_vector", + "bitlist", + "bitvector" + # "boolean", + # "containers" + # "uints" + ] + + @disabled_containers [ + # "SingleFieldTestStruct", + # "SmallTestStruct", + # "FixedTestStruct", + # "VarTestStruct", + "ComplexTestStruct", + "BitsStruct" + ] + + @impl TestRunner + def skip?(%SpecTestCase{fork: fork, handler: handler, case: cse}) do + skip_container? = + @disabled_containers + |> Enum.map(fn container -> String.contains?(cse, container) end) + |> Enum.any?() + + fork != "phase0" or Enum.member?(@disabled, handler) or skip_container? + end + + @impl TestRunner + def run_test_case(%SpecTestCase{} = testcase) do + case_dir = SpecTestCase.dir(testcase) + + schema = parse_type(testcase) + + compressed = File.read!(case_dir <> "/serialized.ssz_snappy") + assert {:ok, decompressed} = :snappyer.decompress(compressed) + + handle_case(testcase.suite, schema, decompressed, testcase) + end + + defp handle_case("valid", schema, real_serialized, testcase) do + case_dir = SpecTestCase.dir(testcase) + + expected = + YamlElixir.read_from_file!(case_dir <> "/value.yaml") + |> SpecTestUtils.sanitize_yaml() + + assert_ssz("valid", schema, real_serialized, expected) + end + + defp handle_case("invalid", schema, real_serialized, _testcase) do + assert_ssz("invalid", schema, real_serialized) + end + + defp assert_ssz("valid", {:container, module}, real_serialized, real_deserialized) do + real_struct = struct!(module, real_deserialized) + {:ok, deserialized} = SszEx.decode(real_serialized, module) + assert deserialized == real_struct + {:ok, serialized} = SszEx.encode(real_struct, module) + assert serialized == real_serialized + end + + defp assert_ssz("valid", schema, real_serialized, real_deserialized) do + {:ok, deserialized} = SszEx.decode(real_serialized, schema) + assert deserialized == real_deserialized + + {:ok, serialized} = SszEx.encode(real_deserialized, schema) + + assert serialized == real_serialized + end + + defp assert_ssz("invalid", schema, real_serialized) do + catch_error(SszEx.encode(real_serialized, schema)) + end + + defp parse_type(%SpecTestCase{handler: handler, case: cse}) do + case handler do + "boolean" -> + :bool + + "uints" -> + case cse do + "uint_" <> _rest -> + [_head, size] = Regex.run(~r/^.*?_(.*?)_.*$/, cse) + {:int, String.to_integer(size)} + end + + "containers" -> + [name] = Regex.run(~r/^[^_]+(?=_)/, cse) + {:container, Module.concat(Helpers.SszStaticContainers, name)} + + # TODO enable when basic_vector and bitlist tests are enable + # "basic_vector" -> + # case cse do + # "vec_" <> rest -> + # case String.split(rest, "_") do + # ["bool", max_size | _] -> + # {:vector, :bool, String.to_integer(max_size)} + # + # ["uint" <> size, max_size | _] -> + # {:vector, {:int, String.to_integer(size)}, String.to_integer(max_size)} + # end + # end + # + # "bitlist" -> + # case cse do + # "bitlist_" <> rest -> + # [size | _] = String.split(rest, "_") + # {:bitlist, String.to_integer(size)} + # end + end + end +end diff --git a/lib/ssz_types/mod.ex b/lib/ssz_types/mod.ex index b471ac074..9da84782e 100644 --- a/lib/ssz_types/mod.ex +++ b/lib/ssz_types/mod.ex @@ -6,6 +6,7 @@ defmodule SszTypes do # Primitive types ## Integer types @type uint8 :: 0..unquote(2 ** 8 - 1) + @type uint16 :: 0..unquote(2 ** 16 - 1) @type uint64 :: 0..unquote(2 ** 64 - 1) @type uint256 :: 0..unquote(2 ** 256 - 1) From 39f9753c3804d1288b10c3d66e2fd1873d070726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:15:07 -0300 Subject: [PATCH 2/3] fix: errors in `compute_proposer_index` (#503) --- lib/constants.ex | 111 ++++++++++++------ .../state_transition/accessors.ex | 21 ++-- .../state_transition/epoch_processing.ex | 4 +- .../state_transition/misc.ex | 91 ++++++-------- .../state_transition/operations.ex | 2 +- lib/spec/runners/random.ex | 22 ++-- lib/spec/runners/sync.ex | 1 + lib/ssz_types/mod.ex | 1 + 8 files changed, 138 insertions(+), 115 deletions(-) diff --git a/lib/constants.ex b/lib/constants.ex index a241dabbe..7430d13f6 100644 --- a/lib/constants.ex +++ b/lib/constants.ex @@ -1,81 +1,120 @@ defmodule Constants do @moduledoc """ Constants module with 0-arity functions. + The following values are (non-configurable) constants used throughout the specification. """ - @spec genesis_epoch() :: integer + ### Misc + + @spec genesis_epoch() :: SszTypes.slot() def genesis_epoch, do: 0 - @spec genesis_slot() :: integer + @spec genesis_slot() :: SszTypes.slot() def genesis_slot, do: 0 - @spec bls_withdrawal_prefix() :: <<_::8>> + @far_future_epoch 2 ** 64 - 1 + + @spec far_future_epoch() :: non_neg_integer() + def far_future_epoch, do: @far_future_epoch + + @spec base_rewards_per_epoch() :: non_neg_integer() + def base_rewards_per_epoch, do: 4 + + @spec deposit_contract_tree_depth() :: non_neg_integer() + def deposit_contract_tree_depth, do: 32 + + @spec justification_bits_length() :: non_neg_integer() + def justification_bits_length, do: 4 + + @spec endianness() :: atom() + def endianness, do: :little + + @spec participation_flag_weights() :: list(non_neg_integer()) + def participation_flag_weights, + do: [timely_source_weight(), timely_target_weight(), timely_head_weight()] + + ### Withdrawal prefixes + + @spec bls_withdrawal_prefix() :: SszTypes.bytes1() def bls_withdrawal_prefix, do: <<0>> - @spec eth1_address_withdrawal_prefix() :: <<_::8>> + @spec eth1_address_withdrawal_prefix() :: SszTypes.bytes1() def eth1_address_withdrawal_prefix, do: <<1>> - @spec domain_beacon_attester() :: SszTypes.domain_type() - def domain_beacon_attester, do: <<1, 0, 0, 0>> + ### Domain types @spec domain_beacon_proposer() :: SszTypes.domain_type() def domain_beacon_proposer, do: <<0, 0, 0, 0>> - @spec domain_deposit() :: SszTypes.domain_type() - def domain_deposit, do: <<3, 0, 0, 0>> + @spec domain_beacon_attester() :: SszTypes.domain_type() + def domain_beacon_attester, do: <<1, 0, 0, 0>> @spec domain_randao() :: SszTypes.domain_type() def domain_randao, do: <<2, 0, 0, 0>> - @spec domain_sync_committee() :: SszTypes.domain_type() - def domain_sync_committee, do: <<7, 0, 0, 0>> + @spec domain_deposit() :: SszTypes.domain_type() + def domain_deposit, do: <<3, 0, 0, 0>> @spec domain_voluntary_exit() :: SszTypes.domain_type() def domain_voluntary_exit, do: <<4, 0, 0, 0>> + @spec domain_selection_proof() :: SszTypes.domain_type() + def domain_selection_proof, do: <<5, 0, 0, 0>> + + @spec domain_aggregate_and_proof() :: SszTypes.domain_type() + def domain_aggregate_and_proof, do: <<6, 0, 0, 0>> + + @spec domain_application_mask() :: SszTypes.domain_type() + def domain_application_mask, do: <<0, 0, 0, 1>> + + @spec domain_sync_committee() :: SszTypes.domain_type() + def domain_sync_committee, do: <<7, 0, 0, 0>> + + @spec domain_sync_committee_selection_proof() :: SszTypes.domain_type() + def domain_sync_committee_selection_proof, do: <<8, 0, 0, 0>> + + @spec domain_contribution_and_proof() :: SszTypes.domain_type() + def domain_contribution_and_proof, do: <<9, 0, 0, 0>> + @spec domain_bls_to_execution_change() :: SszTypes.domain_type() def domain_bls_to_execution_change, do: <<10, 0, 0, 0>> - @spec timely_source_flag_index() :: integer + ### Participation flag indices + + @spec timely_source_flag_index() :: non_neg_integer() def timely_source_flag_index, do: 0 - @spec timely_target_flag_index() :: integer + @spec timely_target_flag_index() :: non_neg_integer() def timely_target_flag_index, do: 1 - @spec timely_head_flag_index() :: integer + @spec timely_head_flag_index() :: non_neg_integer() def timely_head_flag_index, do: 2 - @spec proposer_weight() :: integer - def proposer_weight, do: 8 - - @spec sync_reward_weight() :: integer - def sync_reward_weight, do: 2 - - @spec weight_denominator() :: integer - def weight_denominator, do: 64 + ### Incentivization weights - @spec participation_flag_weights() :: list(integer) - def participation_flag_weights, - do: [timely_source_weight(), timely_target_weight(), timely_head_weight()] - - @spec base_reward_factor() :: integer - def base_reward_factor, do: 64 - - @spec timely_source_weight() :: integer + @spec timely_source_weight() :: non_neg_integer() def timely_source_weight, do: 14 - @spec timely_target_weight() :: integer + @spec timely_target_weight() :: non_neg_integer() def timely_target_weight, do: 26 - @spec timely_head_weight() :: integer + @spec timely_head_weight() :: non_neg_integer() def timely_head_weight, do: 14 - @spec far_future_epoch() :: integer - def far_future_epoch, do: 2 ** 64 - 1 + @spec sync_reward_weight() :: non_neg_integer() + def sync_reward_weight, do: 2 - @spec deposit_contract_tree_depth() :: integer - def deposit_contract_tree_depth, do: 32 + @spec proposer_weight() :: non_neg_integer() + def proposer_weight, do: 8 + + @spec weight_denominator() :: non_neg_integer() + def weight_denominator, do: 64 - @spec intervals_per_slot() :: integer + ## Fork choice + + @spec intervals_per_slot() :: non_neg_integer() def intervals_per_slot, do: 3 + + @spec proposer_score_boost() :: non_neg_integer() + def proposer_score_boost, do: 3 end diff --git a/lib/lambda_ethereum_consensus/state_transition/accessors.ex b/lib/lambda_ethereum_consensus/state_transition/accessors.ex index 1399d7283..574bfb59b 100644 --- a/lib/lambda_ethereum_consensus/state_transition/accessors.ex +++ b/lib/lambda_ethereum_consensus/state_transition/accessors.ex @@ -8,6 +8,8 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do alias LambdaEthereumConsensus.Utils alias SszTypes.{Attestation, BeaconState, IndexedAttestation, SyncCommittee, Validator} + @max_random_byte 2 ** 8 - 1 + @doc """ Return the next sync committee, with possible pubkey duplicates. """ @@ -80,8 +82,6 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do validators, sync_committee_indices ) do - max_random_byte = 2 ** 8 - 1 - with {:ok, shuffled_index} <- rem(index, active_validator_count) |> Misc.compute_shuffled_index(active_validator_count, seed) do @@ -90,10 +90,10 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do <<_::binary-size(rem(index, 32)), random_byte, _::binary>> = SszEx.hash(seed <> Misc.uint64_to_bytes(div(index, 32))) + max_effective_balance = ChainSpec.get("MAX_EFFECTIVE_BALANCE") effective_balance = Enum.fetch!(validators, candidate_index).effective_balance - if effective_balance * max_random_byte >= - ChainSpec.get("MAX_EFFECTIVE_BALANCE") * random_byte do + if effective_balance * @max_random_byte >= max_effective_balance * random_byte do {:ok, sync_committee_indices |> List.insert_at(0, candidate_index)} else {:ok, sync_committee_indices} @@ -278,7 +278,7 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do @spec get_base_reward_per_increment(BeaconState.t()) :: SszTypes.gwei() def get_base_reward_per_increment(state) do - numerator = ChainSpec.get("EFFECTIVE_BALANCE_INCREMENT") * Constants.base_reward_factor() + numerator = ChainSpec.get("EFFECTIVE_BALANCE_INCREMENT") * ChainSpec.get("BASE_REWARD_FACTOR") denominator = Math.integer_squareroot(get_total_active_balance(state)) div(numerator, denominator) end @@ -397,12 +397,11 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do """ @spec get_seed(BeaconState.t(), SszTypes.epoch(), SszTypes.domain_type()) :: SszTypes.bytes32() def get_seed(state, epoch, domain_type) do - mix = - get_randao_mix( - state, - epoch + ChainSpec.get("EPOCHS_PER_HISTORICAL_VECTOR") - - ChainSpec.get("MIN_SEED_LOOKAHEAD") - 1 - ) + future_epoch = + epoch + ChainSpec.get("EPOCHS_PER_HISTORICAL_VECTOR") - + ChainSpec.get("MIN_SEED_LOOKAHEAD") - 1 + + mix = get_randao_mix(state, future_epoch) SszEx.hash(domain_type <> Misc.uint64_to_bytes(epoch) <> mix) end diff --git a/lib/lambda_ethereum_consensus/state_transition/epoch_processing.ex b/lib/lambda_ethereum_consensus/state_transition/epoch_processing.ex index fe31f6d41..4e9f49e53 100644 --- a/lib/lambda_ethereum_consensus/state_transition/epoch_processing.ex +++ b/lib/lambda_ethereum_consensus/state_transition/epoch_processing.ex @@ -95,9 +95,9 @@ defmodule LambdaEthereumConsensus.StateTransition.EpochProcessing do current_epoch = Accessors.get_current_epoch(state) next_epoch = current_epoch + 1 epochs_per_historical_vector = ChainSpec.get("EPOCHS_PER_HISTORICAL_VECTOR") - random_mix = Accessors.get_randao_mix(state, current_epoch) + randao_mix = Accessors.get_randao_mix(state, current_epoch) index = rem(next_epoch, epochs_per_historical_vector) - new_randao_mixes = List.replace_at(randao_mixes, index, random_mix) + new_randao_mixes = List.replace_at(randao_mixes, index, randao_mix) new_state = %BeaconState{state | randao_mixes: new_randao_mixes} {:ok, new_state} end diff --git a/lib/lambda_ethereum_consensus/state_transition/misc.ex b/lib/lambda_ethereum_consensus/state_transition/misc.ex index dd029d20e..074474086 100644 --- a/lib/lambda_ethereum_consensus/state_transition/misc.ex +++ b/lib/lambda_ethereum_consensus/state_transition/misc.ex @@ -3,13 +3,13 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do Misc functions """ - alias LambdaEthereumConsensus.SszEx - alias SszTypes.BeaconState import Bitwise - alias SszTypes.BeaconState + alias LambdaEthereumConsensus.SszEx alias SszTypes.BeaconState + @max_random_byte 2 ** 8 - 1 + @doc """ Returns the Unix timestamp at the start of the given slot """ @@ -52,38 +52,23 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do def compute_shuffled_index(index, index_count, seed) do shuffle_round_count = ChainSpec.get("SHUFFLE_ROUND_COUNT") - new_index = - Enum.reduce(0..(shuffle_round_count - 1), index, fn round, current_index -> - round_as_bytes = <> - - hash_of_seed_round = SszEx.hash(seed <> round_as_bytes) + 0..(shuffle_round_count - 1) + |> Enum.reduce(index, fn round, current_index -> + pivot = SszEx.hash(seed <> <>) |> bytes_to_uint64() |> rem(index_count) - pivot = rem(bytes_to_uint64(hash_of_seed_round), index_count) + flip = rem(pivot + index_count - current_index, index_count) + position = max(current_index, flip) - flip = rem(pivot + index_count - current_index, index_count) - position = max(current_index, flip) + position_div_256 = position |> div(256) |> uint_to_bytes(32) - position_div_256 = uint_to_bytes4(div(position, 256)) + source = SszEx.hash(seed <> <> <> position_div_256) - source = - SszEx.hash(seed <> round_as_bytes <> position_div_256) + bit_index = rem(position, 256) + 7 - 2 * rem(position, 8) + <<_::size(bit_index), bit::1, _::bits>> = source - byte_index = div(rem(position, 256), 8) - <<_::binary-size(byte_index), byte, _::binary>> = source - right_shift = byte >>> rem(position, 8) - bit = rem(right_shift, 2) - - current_index = - if bit == 1 do - flip - else - current_index - end - - current_index - end) - - {:ok, new_index} + if bit == 1, do: flip, else: current_index + end) + |> then(&{:ok, &1}) end @spec increase_inactivity_score(SszTypes.uint64(), integer, MapSet.t(), SszTypes.uint64()) :: @@ -132,31 +117,29 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do """ @spec compute_proposer_index(BeaconState.t(), [SszTypes.validator_index()], SszTypes.bytes32()) :: {:ok, SszTypes.validator_index()} | {:error, binary()} - def compute_proposer_index(state, indices, seed) do - if length(indices) <= 0 do - {:error, "Empty indices"} - else - {:ok, compute_proposer_index(state, indices, seed, 0)} - end - end + def compute_proposer_index(_state, [], _seed), do: {:error, "Empty indices"} - defp compute_proposer_index(state, indices, seed, i) when i < length(indices) do - max_random_byte = 2 ** 8 - 1 + def compute_proposer_index(state, indices, seed) do max_effective_balance = ChainSpec.get("MAX_EFFECTIVE_BALANCE") - total = length(indices) - {:ok, i} = compute_shuffled_index(rem(i, total), total, seed) - candidate_index = Enum.at(indices, i) - random_byte = SszEx.hash(seed <> uint_to_bytes4(div(i, 32))) - <<_::binary-size(rem(i, 32)), byte, _::binary>> = random_byte - effective_balance = Enum.at(state.validators, candidate_index).effective_balance + Stream.iterate(0, &(&1 + 1)) + |> Stream.map(fn i -> + {:ok, index} = compute_shuffled_index(rem(i, total), total, seed) + candidate_index = Enum.at(indices, index) - if effective_balance * max_random_byte >= max_effective_balance * byte do - candidate_index - else - compute_proposer_index(state, indices, seed, i + 1) - end + <<_::binary-size(rem(i, 32)), random_byte, _::binary>> = + SszEx.hash(seed <> uint_to_bytes(div(i, 32), 64)) + + effective_balance = Enum.at(state.validators, candidate_index).effective_balance + + {effective_balance, random_byte, candidate_index} + end) + |> Stream.filter(fn {effective_balance, random_byte, _} -> + effective_balance * @max_random_byte >= max_effective_balance * random_byte + end) + |> Enum.take(1) + |> then(fn [{_, _, candidate_index}] -> {:ok, candidate_index} end) end @doc """ @@ -180,10 +163,10 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do first_8_bytes end - @spec uint_to_bytes4(integer()) :: SszTypes.bytes4() - def uint_to_bytes4(value) do - # Converts an unsigned integer value to a bytes 4 value - <> + @spec uint_to_bytes(non_neg_integer(), 8 | 32 | 64) :: binary() + def uint_to_bytes(value, size) do + # Converts an unsigned integer value to a bytes value + <> end @spec uint64_to_bytes(SszTypes.uint64()) :: <<_::64>> diff --git a/lib/lambda_ethereum_consensus/state_transition/operations.ex b/lib/lambda_ethereum_consensus/state_transition/operations.ex index b5bf22ba5..82d9cd779 100644 --- a/lib/lambda_ethereum_consensus/state_transition/operations.ex +++ b/lib/lambda_ethereum_consensus/state_transition/operations.ex @@ -191,7 +191,7 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do effective_balance_increment = ChainSpec.get("EFFECTIVE_BALANCE_INCREMENT") total_active_increments = total_active_balance |> div(effective_balance_increment) - numerator = effective_balance_increment * Constants.base_reward_factor() + numerator = effective_balance_increment * ChainSpec.get("BASE_REWARD_FACTOR") denominator = Math.integer_squareroot(total_active_balance) base_reward_per_increment = div(numerator, denominator) total_base_rewards = base_reward_per_increment * total_active_increments diff --git a/lib/spec/runners/random.ex b/lib/spec/runners/random.ex index 7edcaa113..f7521caf3 100644 --- a/lib/spec/runners/random.ex +++ b/lib/spec/runners/random.ex @@ -7,22 +7,22 @@ defmodule RandomTestRunner do use TestRunner @disabled_cases [ - "randomized_0", + # "randomized_0", # "randomized_1", - "randomized_2", - "randomized_3", + # "randomized_2", + # "randomized_3", # "randomized_4", - "randomized_5", - "randomized_6", - "randomized_7", + # "randomized_5", + # "randomized_6", + # "randomized_7", # "randomized_8", # "randomized_9", - "randomized_10", + # "randomized_10", # "randomized_11", - "randomized_12", - "randomized_13", - "randomized_14", - "randomized_15" + # "randomized_12", + # "randomized_13", + # "randomized_14", + # "randomized_15" ] @impl TestRunner diff --git a/lib/spec/runners/sync.ex b/lib/spec/runners/sync.ex index e884f7401..2d51f88f1 100644 --- a/lib/spec/runners/sync.ex +++ b/lib/spec/runners/sync.ex @@ -7,6 +7,7 @@ defmodule SyncTestRunner do use TestRunner @disabled_cases [ + # TODO: we have to support https://github.com/ethereum/consensus-specs/blob/dev/tests/formats/fork_choice/README.md#on_payload_info-execution-step "from_syncing_to_invalid" ] diff --git a/lib/ssz_types/mod.ex b/lib/ssz_types/mod.ex index 9da84782e..839266c6d 100644 --- a/lib/ssz_types/mod.ex +++ b/lib/ssz_types/mod.ex @@ -11,6 +11,7 @@ defmodule SszTypes do @type uint256 :: 0..unquote(2 ** 256 - 1) ## Binary types + @type bytes1 :: <<_::8>> @type bytes4 :: <<_::32>> @type bytes20 :: <<_::160>> @type bytes32 :: <<_::256>> From e0ddb7eb35291d0dbc1676fa3d62f82752048f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:31:52 -0300 Subject: [PATCH 3/3] fix: errors "mismatched aggregation bits length" and "invalid signature" (#510) --- .../state_transition/accessors.ex | 60 ++++----- .../state_transition/misc.ex | 9 +- .../state_transition/operations.ex | 116 ++++++++++-------- .../state_transition/state_transition.ex | 1 - ...tion_mainnet.ex => indexed_attestation.ex} | 0 5 files changed, 88 insertions(+), 98 deletions(-) rename lib/ssz_types/beacon_chain/{indexed_attestation_mainnet.ex => indexed_attestation.ex} (100%) diff --git a/lib/lambda_ethereum_consensus/state_transition/accessors.ex b/lib/lambda_ethereum_consensus/state_transition/accessors.ex index 574bfb59b..588641ba9 100644 --- a/lib/lambda_ethereum_consensus/state_transition/accessors.ex +++ b/lib/lambda_ethereum_consensus/state_transition/accessors.ex @@ -433,21 +433,14 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do @spec get_indexed_attestation(BeaconState.t(), Attestation.t()) :: {:ok, IndexedAttestation.t()} | {:error, binary()} def get_indexed_attestation(%BeaconState{} = state, attestation) do - case get_attesting_indices(state, attestation.data, attestation.aggregation_bits) do - {:ok, indices} -> - attesting_indices = indices - sorted_attesting_indices = Enum.sort(attesting_indices) - - res = %IndexedAttestation{ - attesting_indices: sorted_attesting_indices, - data: attestation.data, - signature: attestation.signature - } - - {:ok, res} - - {:error, reason} -> - {:error, reason} + with {:ok, indices} <- + get_attesting_indices(state, attestation.data, attestation.aggregation_bits) do + %IndexedAttestation{ + attesting_indices: Enum.sort(indices), + data: attestation.data, + signature: attestation.signature + } + |> then(&{:ok, &1}) end end @@ -457,33 +450,22 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do @spec get_attesting_indices(BeaconState.t(), SszTypes.AttestationData.t(), SszTypes.bitlist()) :: {:ok, MapSet.t()} | {:error, binary()} def get_attesting_indices(%BeaconState{} = state, data, bits) do - case get_beacon_committee(state, data.slot, data.index) do - {:ok, committee} -> - bit_list = bitstring_to_list(bits) - - res = - committee - |> Stream.with_index() - |> Stream.filter(fn {_value, index} -> Enum.at(bit_list, index) == "1" end) - |> Stream.map(fn {value, _index} -> value end) - |> MapSet.new() - - {:ok, res} - - {:error, reason} -> - {:error, reason} + with {:ok, committee} <- get_beacon_committee(state, data.slot, data.index) do + committee + |> Stream.with_index() + |> Stream.filter(fn {_value, index} -> participated?(bits, index) end) + |> Stream.map(fn {value, _index} -> value end) + |> MapSet.new() + |> then(&{:ok, &1}) end end - def bitstring_to_list(binary) when is_binary(binary) do - binary - |> :binary.bin_to_list() - |> Enum.reduce("", fn byte, acc -> - acc <> Integer.to_string(byte, 2) - end) - # Exclude last bit - |> String.slice(0..-2) - |> String.graphemes() + defp participated?(bits, index) do + # The bit order inside the byte is reversed (e.g. bits[0] is the 8th bit). + # Here we keep the byte index the same, but reverse the bit index. + bit_index = index + 7 - 2 * rem(index, 8) + <<_::size(bit_index), flag::1, _::bits>> = bits + flag == 1 end @doc """ diff --git a/lib/lambda_ethereum_consensus/state_transition/misc.ex b/lib/lambda_ethereum_consensus/state_transition/misc.ex index 074474086..5e152cebb 100644 --- a/lib/lambda_ethereum_consensus/state_transition/misc.ex +++ b/lib/lambda_ethereum_consensus/state_transition/misc.ex @@ -230,11 +230,8 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do Return the signing root for the corresponding signing data. """ @spec compute_signing_root(SszTypes.bytes32(), SszTypes.domain()) :: SszTypes.root() - def compute_signing_root(<<_::256>> = data, domain) do - Ssz.hash_tree_root!(%SszTypes.SigningData{ - object_root: data, - domain: domain - }) + def compute_signing_root(<<_::256>> = root, domain) do + Ssz.hash_tree_root!(%SszTypes.SigningData{object_root: root, domain: domain}) end @spec compute_signing_root(any(), SszTypes.domain()) :: SszTypes.root() @@ -242,7 +239,7 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do ssz_object |> Ssz.hash_tree_root!() |> compute_signing_root(domain) end - @spec compute_signing_root(any(), module, SszTypes.domain()) :: SszTypes.root() + @spec compute_signing_root(any(), module(), SszTypes.domain()) :: SszTypes.root() def compute_signing_root(ssz_object, schema, domain) do ssz_object |> Ssz.hash_tree_root!(schema) |> compute_signing_root(domain) end diff --git a/lib/lambda_ethereum_consensus/state_transition/operations.ex b/lib/lambda_ethereum_consensus/state_transition/operations.ex index 82d9cd779..64bd832d7 100644 --- a/lib/lambda_ethereum_consensus/state_transition/operations.ex +++ b/lib/lambda_ethereum_consensus/state_transition/operations.ex @@ -644,11 +644,18 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do """ @spec process_attestation(BeaconState.t(), Attestation.t()) :: {:ok, BeaconState.t()} | {:error, binary()} - def process_attestation(state, attestation) do + def process_attestation(state, %Attestation{data: data} = attestation) do # TODO: optimize (takes ~3s) - with :ok <- verify_attestation_for_process(state, attestation) do + with :ok <- check_valid_target_epoch(data, state), + :ok <- check_epoch_matches(data), + :ok <- check_valid_slot_range(data, state), + :ok <- check_committee_count(data, state), + {:ok, beacon_committee} <- Accessors.get_beacon_committee(state, data.slot, data.index), + :ok <- check_matching_aggregation_bits_length(attestation, beacon_committee), + {:ok, indexed_attestation} <- Accessors.get_indexed_attestation(state, attestation), + :ok <- check_valid_signature(state, indexed_attestation) do # TODO: optimize (takes ~1s) - process_attestation(state, attestation.data, attestation.aggregation_bits) + process_attestation(state, data, attestation.aggregation_bits) end end @@ -740,34 +747,6 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do defp update_state(state, false, updated_epoch_participation), do: %{state | previous_epoch_participation: updated_epoch_participation} - def verify_attestation_for_process(state, %Attestation{data: data} = attestation) do - with {:ok, beacon_committee} <- Accessors.get_beacon_committee(state, data.slot, data.index), - {:ok, indexed_attestation} <- Accessors.get_indexed_attestation(state, attestation) do - cond do - invalid_target_epoch?(data, state) -> - {:error, "Invalid target epoch"} - - epoch_mismatch?(data) -> - {:error, "Epoch mismatch"} - - invalid_slot_range?(data, state) -> - {:error, "Invalid slot range"} - - exceeds_committee_count?(data, state) -> - {:error, "Index exceeds committee count"} - - mismatched_aggregation_bits_length?(attestation, beacon_committee) -> - {:error, "Mismatched aggregation bits length"} - - not valid_signature?(state, indexed_attestation) -> - {:error, "Invalid signature"} - - true -> - :ok - end - end - end - @doc """ Provide randomness to the operation of the beacon chain. """ @@ -835,41 +814,74 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do end end - defp invalid_target_epoch?(data, state) do - data.target.epoch < Accessors.get_previous_epoch(state) || - data.target.epoch > Accessors.get_current_epoch(state) + defp check_valid_target_epoch(data, state) do + if data.target.epoch in [ + Accessors.get_previous_epoch(state), + Accessors.get_current_epoch(state) + ] do + :ok + else + {:error, "Invalid target epoch"} + end end - defp epoch_mismatch?(data) do - data.target.epoch != Misc.compute_epoch_at_slot(data.slot) + defp check_epoch_matches(data) do + if data.target.epoch == Misc.compute_epoch_at_slot(data.slot) do + :ok + else + {:error, "Epoch mismatch"} + end end - defp invalid_slot_range?(data, state) do - state.slot < data.slot + ChainSpec.get("MIN_ATTESTATION_INCLUSION_DELAY") || - state.slot > data.slot + ChainSpec.get("SLOTS_PER_EPOCH") + defp check_valid_slot_range(data, state) do + if data.slot + ChainSpec.get("MIN_ATTESTATION_INCLUSION_DELAY") <= state.slot and + state.slot <= data.slot + ChainSpec.get("SLOTS_PER_EPOCH") do + :ok + else + {:error, "Invalid slot range"} + end end - defp exceeds_committee_count?(data, state) do - data.index >= Accessors.get_committee_count_per_slot(state, data.target.epoch) + defp check_committee_count(data, state) do + if data.index >= Accessors.get_committee_count_per_slot(state, data.target.epoch) do + {:error, "Index exceeds committee count"} + else + :ok + end end - defp mismatched_aggregation_bits_length?(attestation, beacon_committee) do - length_of_bitstring(attestation.aggregation_bits) - 1 != length(beacon_committee) + defp check_matching_aggregation_bits_length(attestation, beacon_committee) do + if length_of_bitlist(attestation.aggregation_bits) == length(beacon_committee) do + :ok + else + {:error, "Mismatched aggregation bits length"} + end end - defp valid_signature?(state, indexed_attestation) do - Predicates.is_valid_indexed_attestation(state, indexed_attestation) + defp check_valid_signature(state, indexed_attestation) do + if Predicates.is_valid_indexed_attestation(state, indexed_attestation) do + :ok + else + {:error, "Invalid signature"} + end end - defp length_of_bitstring(binary) when is_binary(binary) do - binary - |> :binary.bin_to_list() - |> Enum.reduce("", fn byte, acc -> - acc <> Integer.to_string(byte, 2) - end) - |> String.length() + defp length_of_bitlist(bitlist) when is_binary(bitlist) do + bit_size = bit_size(bitlist) + <<_::size(bit_size - 8), last_byte>> = bitlist + bit_size - leading_zeros(<>) - 1 end + defp leading_zeros(<<1::1, _::7>>), do: 0 + defp leading_zeros(<<0::1, 1::1, _::6>>), do: 1 + defp leading_zeros(<<0::2, 1::1, _::5>>), do: 2 + defp leading_zeros(<<0::3, 1::1, _::4>>), do: 3 + defp leading_zeros(<<0::4, 1::1, _::3>>), do: 4 + defp leading_zeros(<<0::5, 1::1, _::2>>), do: 5 + defp leading_zeros(<<0::6, 1::1, _::1>>), do: 6 + defp leading_zeros(<<0::7, 1::1>>), do: 7 + defp leading_zeros(<<0::8>>), do: 8 + def process_bls_to_execution_change(state, signed_address_change) do address_change = signed_address_change.message diff --git a/lib/lambda_ethereum_consensus/state_transition/state_transition.ex b/lib/lambda_ethereum_consensus/state_transition/state_transition.ex index fcce6062d..16c9446d8 100644 --- a/lib/lambda_ethereum_consensus/state_transition/state_transition.ex +++ b/lib/lambda_ethereum_consensus/state_transition/state_transition.ex @@ -113,7 +113,6 @@ defmodule LambdaEthereumConsensus.StateTransition do Bls.valid?(proposer.pubkey, signing_root, signed_block.signature) end - # TODO: uncomment when implemented def process_block(state, block) do verify_and_notify_new_payload = &Execution.verify_and_notify_new_payload/1 diff --git a/lib/ssz_types/beacon_chain/indexed_attestation_mainnet.ex b/lib/ssz_types/beacon_chain/indexed_attestation.ex similarity index 100% rename from lib/ssz_types/beacon_chain/indexed_attestation_mainnet.ex rename to lib/ssz_types/beacon_chain/indexed_attestation.ex