Merge remote-tracking branch 'origin/main' into ssz_write_schemas_for…

…_all_containers

README.md

@@ -127,9 +127,9 @@ Or by a single runner in all configs, with:
 make spec-test-runner-`runner`
 
 # Some examples
-make spec-test-config-ssz_static
-make spec-test-config-bls
-make spec-test-config-operations
+make spec-test-runner-ssz_static
+make spec-test-runner-bls
+make spec-test-runner-operations
 ```
 
 The complete list of test runners can be found [here](https://github.com/ethereum/consensus-specs/tree/dev/tests/formats).

lib/lambda_ethereum_consensus/fork_choice/helpers.ex

@@ -8,6 +8,23 @@ defmodule LambdaEthereumConsensus.ForkChoice.Helpers do
   alias SszTypes.Checkpoint
   alias SszTypes.Store
 
+  @spec current_status_message(Store.t()) ::
+          {:ok, SszTypes.StatusMessage.t()} | {:error, any}
+  def current_status_message(store) do
+    with {:ok, head_root} <- get_head(store),
+         {:ok, state} <- Map.fetch(store.block_states, head_root) do
+      {:ok,
+       %SszTypes.StatusMessage{
+         fork_digest:
+           Misc.compute_fork_digest(state.fork.current_version, state.genesis_validators_root),
+         finalized_root: state.finalized_checkpoint.root,
+         finalized_epoch: state.finalized_checkpoint.epoch,
+         head_root: head_root,
+         head_slot: state.slot
+       }}
+    end
+  end
+
   @spec get_forkchoice_store(BeaconState.t(), BeaconBlock.t()) :: {:ok, Store.t()} | {:error, any}
   def get_forkchoice_store(%BeaconState{} = anchor_state, %BeaconBlock{} = anchor_block) do
     anchor_state_root = Ssz.hash_tree_root!(anchor_state)

lib/lambda_ethereum_consensus/fork_choice/store.ex

@@ -36,6 +36,11 @@ defmodule LambdaEthereumConsensus.ForkChoice.Store do
     div(time - genesis_time, ChainSpec.get("SECONDS_PER_SLOT"))
   end
 
+  @spec get_current_status_message() :: {:ok, SszTypes.StatusMessage.t()} | {:error, any}
+  def get_current_status_message do
+    GenServer.call(__MODULE__, :get_current_status_message, @default_timeout)
+  end
+
   @spec has_block?(SszTypes.root()) :: boolean()
   def has_block?(block_root) do
     block = get_block(block_root)
@@ -89,6 +94,11 @@ defmodule LambdaEthereumConsensus.ForkChoice.Store do
     {:reply, values, state}
   end
 
+  @impl GenServer
+  def handle_call(:get_current_status_message, _from, state) do
+    {:reply, Helpers.current_status_message(state)}
+  end
+
   def handle_call({:get_block, block_root}, _from, state) do
     {:reply, Map.get(state.blocks, block_root), state}
   end

lib/lambda_ethereum_consensus/p2p/incoming_requests/handler.ex

@@ -2,9 +2,11 @@ defmodule LambdaEthereumConsensus.P2P.IncomingRequests.Handler do
   @moduledoc """
   This module handles Req/Resp domain requests.
   """
-  require Logger
-  alias LambdaEthereumConsensus.{Libp2pPort, P2P}
+
+  alias LambdaEthereumConsensus.ForkChoice
   alias LambdaEthereumConsensus.Store.BlockStore
+  alias LambdaEthereumConsensus.{Libp2pPort, P2P}
+  require Logger
 
   # This is the `ForkDigest` for mainnet in the capella fork
   # TODO: compute this at runtime
@@ -28,18 +30,8 @@ defmodule LambdaEthereumConsensus.P2P.IncomingRequests.Handler do
   @spec handle_req(String.t(), String.t(), binary()) ::
           :ok | :not_implemented | {:error, binary()}
   defp handle_req("status/1/ssz_snappy", message_id, message) do
-    # hardcoded response from random peer
-    current_status = %SszTypes.StatusMessage{
-      fork_digest: Base.decode16!("BBA4DA96"),
-      finalized_root:
-        Base.decode16!("7715794499C07D9954DD223EC2C6B846D3BAB27956D093000FADC1B8219F74D4"),
-      finalized_epoch: 228_168,
-      head_root:
-        Base.decode16!("D62A74AE0F933224133C5E6E1827A2835A1E705F0CDFEE3AD25808DDEA5572DB"),
-      head_slot: 7_301_450
-    }
-
     with <<84, snappy_status::binary>> <- message,
+         {:ok, current_status} <- ForkChoice.Store.get_current_status_message(),
          {:ok, ssz_status} <- Snappy.decompress(snappy_status),
          {:ok, status} <- Ssz.from_ssz(ssz_status, SszTypes.StatusMessage),
          status

lib/lambda_ethereum_consensus/state_transition/accessors.ex

@@ -251,8 +251,9 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do
   """
   @spec get_committee_count_per_slot(BeaconState.t(), SszTypes.epoch()) :: SszTypes.uint64()
   def get_committee_count_per_slot(%BeaconState{} = state, epoch) do
-    get_active_validator_indices(state, epoch)
-    |> length()
+    state.validators
+    |> Stream.filter(&Predicates.is_active_validator(&1, epoch))
+    |> Enum.count()
     |> div(ChainSpec.get("SLOTS_PER_EPOCH"))
     |> div(ChainSpec.get("TARGET_COMMITTEE_SIZE"))
     |> min(ChainSpec.get("MAX_COMMITTEES_PER_SLOT"))
@@ -444,6 +445,20 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do
     end
   end
 
+  @spec get_committee_indexed_attestation([SszTypes.validator_index()], Attestation.t()) ::
+          IndexedAttestation.t()
+  def get_committee_indexed_attestation(beacon_committee, attestation) do
+    get_committee_attesting_indices(beacon_committee, attestation.aggregation_bits)
+    |> Enum.sort()
+    |> then(
+      &%IndexedAttestation{
+        attesting_indices: &1,
+        data: attestation.data,
+        signature: attestation.signature
+      }
+    )
+  end
+
   @doc """
   Return the set of attesting indices corresponding to ``data`` and ``bits``.
   """
@@ -460,6 +475,16 @@ defmodule LambdaEthereumConsensus.StateTransition.Accessors do
     end
   end
 
+  @spec get_committee_attesting_indices([SszTypes.validator_index()], SszTypes.bitlist()) ::
+          MapSet.t()
+  def get_committee_attesting_indices(committee, bits) do
+    committee
+    |> Stream.with_index()
+    |> Stream.filter(fn {_value, index} -> participated?(bits, index) end)
+    |> Stream.map(fn {value, _index} -> value end)
+    |> MapSet.new()
+  end
+
   defp participated?(bits, index) do
     # The bit order inside the byte is reversed (e.g. bits[0] is the 8th bit).
     # Here we keep the byte index the same, but reverse the bit index.

lib/lambda_ethereum_consensus/state_transition/misc.ex

@@ -174,35 +174,52 @@ defmodule LambdaEthereumConsensus.StateTransition.Misc do
     <<value::unsigned-integer-little-size(64)>>
   end
 
+  @doc """
+  Computes the validator indices of the ``committee_index``-th committee at some epoch
+  with ``committee_count`` committees, and for some given ``indices`` and ``seed``.
+  """
   @spec compute_committee(
           list(SszTypes.validator_index()),
           SszTypes.bytes32(),
           SszTypes.uint64(),
           SszTypes.uint64()
         ) ::
           {:ok, list(SszTypes.validator_index())} | {:error, binary()}
-  def compute_committee(indices, seed, index, count) do
-    start_ = div(length(indices) * index, count)
-    end_ = div(length(indices) * (index + 1), count) - 1
-
-    case compute_committee_indices(start_, end_, indices, seed) do
-      {:ok, result_list} -> {:ok, Enum.reverse(result_list)}
-      _ -> {:error, "invalid index_count"}
+  def compute_committee(indices, seed, committee_index, committee_count) do
+    index_count = length(indices)
+    committee_start = div(index_count * committee_index, committee_count)
+    committee_end = div(index_count * (committee_index + 1), committee_count) - 1
+
+    result =
+      committee_start..committee_end//1
+      |> Stream.map(&compute_shuffled_index(&1, index_count, seed))
+      |> Stream.with_index()
+      |> Enum.reduce_while({:ok, []}, fn
+        {{:ok, shuffled_index}, i}, {:ok, acc} ->
+          {:cont, {:ok, [{shuffled_index, i} | acc]}}
+
+        {{:error, _} = err, _}, _ ->
+          {:halt, err}
+      end)
+
+    with {:ok, to_swap_indices} <- result do
+      to_swap_indices = Enum.sort(to_swap_indices, fn {a, _}, {b, _} -> a <= b end)
+
+      {swapped_indices, []} =
+        indices
+        |> Stream.with_index()
+        |> Enum.flat_map_reduce(to_swap_indices, fn
+          {v, i}, [{i, j} | tail] -> {[{v, j}], tail}
+          _, acc -> {[], acc}
+        end)
+
+      swapped_indices
+      |> Enum.sort(fn {_, a}, {_, b} -> a <= b end)
+      |> Enum.map(fn {v, _} -> v end)
+      |> then(&{:ok, &1})
     end
   end
 
-  defp compute_committee_indices(start_, end_, indices, seed) do
-    Enum.reduce_while(start_..end_, {:ok, []}, fn i, {:ok, acc_list} ->
-      case compute_shuffled_index(i, length(indices), seed) do
-        {:ok, shuffled_index} ->
-          {:cont, {:ok, [Enum.at(indices, shuffled_index) | acc_list]}}
-
-        {:error, _} = error ->
-          {:halt, error}
-      end
-    end)
-  end
-
   @doc """
   Return the 32-byte fork data root for the ``current_version`` and ``genesis_validators_root``.
   This is used primarily in signature domains to avoid collisions across forks/chains.

lib/lambda_ethereum_consensus/state_transition/operations.ex

@@ -645,29 +645,27 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do
   @spec process_attestation(BeaconState.t(), Attestation.t()) ::
           {:ok, BeaconState.t()} | {:error, binary()}
   def process_attestation(state, %Attestation{data: data} = attestation) do
-    # TODO: optimize (takes ~3s)
     with :ok <- check_valid_target_epoch(data, state),
          :ok <- check_epoch_matches(data),
          :ok <- check_valid_slot_range(data, state),
          :ok <- check_committee_count(data, state),
          {:ok, beacon_committee} <- Accessors.get_beacon_committee(state, data.slot, data.index),
          :ok <- check_matching_aggregation_bits_length(attestation, beacon_committee),
-         {:ok, indexed_attestation} <- Accessors.get_indexed_attestation(state, attestation),
+         indexed_attestation =
+           Accessors.get_committee_indexed_attestation(beacon_committee, attestation),
          :ok <- check_valid_signature(state, indexed_attestation) do
-      # TODO: optimize (takes ~1s)
-      process_attestation(state, data, attestation.aggregation_bits)
+      inner_process_attestation(state, data, attestation.aggregation_bits, beacon_committee)
     end
   end
 
-  defp process_attestation(state, data, aggregation_bits) do
+  defp inner_process_attestation(state, data, aggregation_bits, committee) do
+    slot = state.slot - data.slot
+
     with {:ok, participation_flag_indices} <-
-           Accessors.get_attestation_participation_flag_indices(
-             state,
-             data,
-             state.slot - data.slot
-           ),
-         {:ok, attesting_indices} <-
-           Accessors.get_attesting_indices(state, data, aggregation_bits) do
+           Accessors.get_attestation_participation_flag_indices(state, data, slot) do
+      attesting_indices =
+        Accessors.get_committee_attesting_indices(committee, aggregation_bits)
+
       is_current_epoch = data.target.epoch == Accessors.get_current_epoch(state)
       initial_epoch_participation = get_initial_epoch_participation(state, is_current_epoch)
 
@@ -687,8 +685,6 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do
       |> Mutators.increase_balance(proposer_index, proposer_reward)
       |> update_state(is_current_epoch, updated_epoch_participation)
       |> then(&{:ok, &1})
-    else
-      {:error, reason} -> {:error, reason}
     end
   end
 

lib/spec/runners/ssz_generic.ex

@@ -49,32 +49,46 @@ defmodule SszGenericTestRunner do
   defp handle_case("valid", schema, real_serialized, testcase) do
     case_dir = SpecTestCase.dir(testcase)
 
-    expected =
+    expected_value =
       YamlElixir.read_from_file!(case_dir <> "/value.yaml")
       |> SpecTestUtils.sanitize_yaml()
 
-    assert_ssz("valid", schema, real_serialized, expected)
+    %{root: expected_root} =
+      YamlElixir.read_from_file!(case_dir <> "/meta.yaml")
+      |> SpecTestUtils.sanitize_yaml()
+
+    assert_ssz("valid", schema, real_serialized, expected_value, expected_root)
   end
 
   defp handle_case("invalid", schema, real_serialized, _testcase) do
     assert_ssz("invalid", schema, real_serialized)
   end
 
-  defp assert_ssz("valid", {:container, module}, real_serialized, real_deserialized) do
+  defp assert_ssz(
+         "valid",
+         {:container, module},
+         real_serialized,
+         real_deserialized,
+         _hash_tree_root
+       ) do
     real_struct = struct!(module, real_deserialized)
     {:ok, deserialized} = SszEx.decode(real_serialized, module)
     assert deserialized == real_struct
     {:ok, serialized} = SszEx.encode(real_struct, module)
     assert serialized == real_serialized
   end
 
-  defp assert_ssz("valid", schema, real_serialized, real_deserialized) do
+  defp assert_ssz("valid", schema, real_serialized, real_deserialized, expected_hash_tree_root) do
     {:ok, deserialized} = SszEx.decode(real_serialized, schema)
     assert deserialized == real_deserialized
 
     {:ok, serialized} = SszEx.encode(real_deserialized, schema)
 
     assert serialized == real_serialized
+
+    actual_hash_tree_root = SszEx.hash_tree_root!(real_deserialized, schema)
+
+    assert actual_hash_tree_root == expected_hash_tree_root
   end
 
   defp assert_ssz("invalid", schema, real_serialized) do

lib/ssz.ex

@@ -1,6 +1,6 @@
 defmodule Ssz do
   @moduledoc """
-  SimpleSerialize (SSZ) serialization and deserialization.
+  SimpleSerialize (SSZ) serialization, deserialization and merkleization.
   """
   use Rustler, otp_app: :lambda_ethereum_consensus, crate: "ssz_nif"
 

lib/ssz_ex.ex

@@ -7,6 +7,8 @@ defmodule LambdaEthereumConsensus.SszEx do
   ### Public API
   #################
 
+  @bits_per_chunk 256
+
   @spec hash(iodata()) :: binary()
   def hash(data), do: :crypto.hash(:sha256, data)
 
@@ -36,6 +38,12 @@ defmodule LambdaEthereumConsensus.SszEx do
 
   def decode(binary, module) when is_atom(module), do: decode_container(binary, module)
 
+  @spec hash_tree_root!(boolean, atom) :: SszTypes.root()
+  def hash_tree_root!(value, :bool), do: pack(value)
+
+  @spec hash_tree_root!(non_neg_integer, {:int, non_neg_integer}) :: SszTypes.root()
+  def hash_tree_root!(value, {:int, size}), do: pack(value, size)
+
   #################
   ### Private functions
   #################
@@ -398,4 +406,16 @@ defmodule LambdaEthereumConsensus.SszEx do
     |> Enum.map(fn {_, schema} -> variable_size?(schema) end)
     |> Enum.any?()
   end
+
+  defp pack(value, size) when is_integer(value) and value >= 0 do
+    pad = @bits_per_chunk - size
+    <<value::size(size)-little, 0::size(pad)>>
+  end
+
+  defp pack(value) when is_boolean(value) do
+    case value do
+      true -> <<1::@bits_per_chunk-little>>
+      false -> <<0::@bits_per_chunk>>
+    end
+  end
 end