Merge remote-tracking branch 'origin/main' into implement_bit_list_utils

.github/workflows/ci.yml

@@ -51,9 +51,25 @@ jobs:
         if: steps.output-cache.outputs.cache-hit != 'true'
         run: make compile-port compile-native
 
+  download-beacon-node-oapi:
+    name: Download Beacon Node OAPI
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache Beacon Node OAPI
+        id: output-cache
+        uses: actions/cache@v3
+        with:
+          path: ./beacon-node-oapi.json
+          key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
+          lookup-only: true
+      - name: Download Beacon Node OAPI
+        if: steps.output-cache.outputs.cache-hit != 'true'
+        run: make download-beacon-node-oapi
+
   build:
     name: Build project
-    needs: compile-native
+    needs: [compile-native, download-beacon-node-oapi]
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
@@ -75,6 +91,12 @@ jobs:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
+    - name: Fetch beacon node oapi file
+      uses: actions/cache/restore@v3
+      with:
+        path: ./beacon-node-oapi.json
+        key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
+        fail-on-cache-miss: true
     - name: Install dependencies
       run: |
         sudo apt-get install -y protobuf-compiler
@@ -104,7 +126,7 @@ jobs:
 
   smoke:
     name: Start and stop the node
-    needs: compile-native
+    needs: [compile-native, download-beacon-node-oapi]
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
@@ -126,6 +148,12 @@ jobs:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
+    - name: Fetch beacon node oapi file
+      uses: actions/cache/restore@v3
+      with:
+        path: ./beacon-node-oapi.json
+        key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
+        fail-on-cache-miss: true
     - name: Install dependencies
       run: |
         sudo apt-get install -y protobuf-compiler
@@ -145,7 +173,7 @@ jobs:
 
   test:
     name: Test
-    needs: compile-native
+    needs: [compile-native, download-beacon-node-oapi]
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
@@ -167,6 +195,12 @@ jobs:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
+    - name: Fetch beacon node oapi file
+      uses: actions/cache/restore@v3
+      with:
+        path: ./beacon-node-oapi.json
+        key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
+        fail-on-cache-miss: true
     - name: Set up cargo cache
       uses: Swatinem/rust-cache@v2
       with:
@@ -223,7 +257,7 @@ jobs:
 
   spectests:
     name: Run spec-tests
-    needs: [compile-native, download-spectests]
+    needs: [compile-native, download-spectests, download-beacon-node-oapi]
     strategy:
       matrix:
         config: ["minimal", "general", "mainnet"]
@@ -253,6 +287,12 @@ jobs:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
           restore-keys: ${{ runner.os }}-mix-
+      - name: Fetch beacon node oapi file
+        uses: actions/cache/restore@v3
+        with:
+          path: ./beacon-node-oapi.json
+          key: ${{ runner.os }}-beacon-node-oapi-${{ hashFiles('.oapi_version') }}
+          fail-on-cache-miss: true
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:

.gitignore

@@ -44,20 +44,24 @@ priv
 .vscode/
 
 # spec-test vectors
-test/spec/vectors
+/test/spec/vectors
 
-native/libp2p_port/libp2p_port
+/native/libp2p_port/libp2p_port
 
 # Proto generated code.
 *.pb.ex
 *.pb.go
 
 # local db.
-level_db
+/level_db
+/logs
 
 # Generated tests
-test/generated
+/test/generated
 
 # profiling artifacts
 callgrind.out.*
 *-eflambe-output.bggg
+
+# beacon node oapi json file
+beacon-node-oapi.json

.oapi_version

@@ -0,0 +1 @@
+v2.4.2

Makefile

@@ -2,7 +2,7 @@
 		clean-vectors download-vectors uncompress-vectors proto \
 		spec-test-% spec-test spec-test-config-% spec-test-runner-% \
 		spec-test-mainnet-% spec-test-minimal-% spec-test-general-% \
-		clean-tests gen-spec compile-all
+		clean-tests gen-spec compile-all download-beacon-node-oapi
 
 # Delete current file when command fails
 .DELETE_ON_ERROR:
@@ -86,7 +86,7 @@ proto: $(PROTOBUF_EX_FILES) $(PROTOBUF_GO_FILES)
 compile-native: $(OUTPUT_DIR)/libp2p_nif.so $(OUTPUT_DIR)/libp2p_port
 
 #🔨 compile-all: @ Compile the elixir project and its dependencies.
-compile-all: compile-native $(PROTOBUF_EX_FILES)
+compile-all: compile-native $(PROTOBUF_EX_FILES) download-beacon-node-oapi
 	mix compile
 
 #🗑️ clean: @ Remove the build files.
@@ -126,6 +126,17 @@ sepolia: compile-all
 test: compile-all
 	mix test --no-start --exclude spectest
 
+#### BEACON NODE OAPI ####
+OAPI_NAME = beacon-node-oapi
+OAPI_VERSION := $(shell cat .oapi_version)
+$(OAPI_NAME).json: .oapi_version
+	curl -L -o "$@" \
+		"https://ethereum.github.io/beacon-APIs/releases/${OAPI_VERSION}/beacon-node-oapi.json"
+
+OPENAPI_JSON := $(OAPI_NAME).json 
+
+download-beacon-node-oapi: ${OPENAPI_JSON}
+
 ##### SPEC TEST VECTORS #####
 
 SPECTEST_VERSION := $(shell cat .spectest_version)
@@ -206,3 +217,7 @@ fmt:
 	cd native/snappy_nif; cargo fmt
 	cd native/ssz_nif; cargo fmt
 	cd native/bls_nif; cargo fmt
+
+#✅ dialyzer: @ Run dialyzer (static analysis tool).
+dialyzer:
+	mix dialyzer

README.md

@@ -96,7 +96,7 @@ To run these checks locally:
 make test      # Runs tests
 make spec-test # Runs all spec-tests
 make lint      # Runs linter and format-checker
-mix dialyzer   # Runs type-checker
+make dialyzer  # Runs type-checker
 ```
 
 Source code can be formatted using `make fmt`.

bench/block_processing.exs

@@ -1,6 +1,5 @@
 alias LambdaEthereumConsensus.ForkChoice
 alias LambdaEthereumConsensus.ForkChoice.Handlers
-alias LambdaEthereumConsensus.ForkChoice.Helpers
 alias LambdaEthereumConsensus.StateTransition.Cache
 alias LambdaEthereumConsensus.Store
 alias LambdaEthereumConsensus.Store.BlockStore
@@ -20,12 +19,15 @@ IO.puts("fetching blocks...")
 {:ok, %SignedBeaconBlock{} = new_block} = BlockStore.get_block_by_slot(slot + 1)
 
 IO.puts("initializing store...")
-{:ok, store} = Helpers.get_forkchoice_store(state, block, true)
+{:ok, store} = Types.Store.get_forkchoice_store(state, block, true)
 store = Handlers.on_tick(store, store.time + 30)
 
 attestations = new_block.message.body.attestations
 attester_slashings = new_block.message.body.attester_slashings
 
+{:ok, root} = BlockStore.get_block_root_by_slot(slot)
+
+IO.puts("about to process block: #{slot + 1}, with root: #{Base.encode16(root)}...")
 IO.puts("#{length(attestations)} attestations ; #{length(attester_slashings)} attester slashings")
 IO.puts("")
 

config/config.exs

@@ -4,6 +4,31 @@ import Config
 # Configure logging
 config :logger, level: :info, truncate: :infinity
 
+# # Uncomment to log to a file
+# # TODO: we might want to enable this with a CLI flag
+# config :logger, :default_handler,
+#   config: [
+#     file: ~c"logs/system.log",
+#     filesync_repeat_interval: 5000,
+#     file_check: 5000,
+#     max_no_bytes: 10_000_000,
+#     max_no_files: 5,
+#     compress_on_rotate: true
+#   ]
+
+# # NOTE: We want to log UTC timestamps, for convenience
+# config :logger, utc_log: true
+
+# config :logger, :default_formatter,
+#   format: {LogfmtEx, :format},
+#   colors: [enabled: false],
+#   metadata: [:mfa]
+
+# config :logfmt_ex, :opts,
+#   message_key: "msg",
+#   timestamp_key: "ts",
+#   timestamp_format: :iso8601
+
 # Configures the phoenix endpoint
 config :lambda_ethereum_consensus, BeaconApi.Endpoint,
   http: [port: 4000],

config/runtime.exs

@@ -1,4 +1,5 @@
 import Config
+require Logger
 
 {args, _remaining_args, _errors} =
   OptionParser.parse(System.argv(),
@@ -15,7 +16,7 @@ network = Keyword.get(args, :network, "mainnet")
 checkpoint_sync = Keyword.get(args, :checkpoint_sync)
 execution_endpoint = Keyword.get(args, :execution_endpoint, "http://localhost:8551")
 jwt_path = Keyword.get(args, :execution_jwt)
-mock_execution = Keyword.get(args, :mock_execution, false)
+mock_execution = Keyword.get(args, :mock_execution, config_env() == :test or is_nil(jwt_path))
 
 config :lambda_ethereum_consensus, LambdaEthereumConsensus.ForkChoice,
   checkpoint_sync: checkpoint_sync
@@ -61,3 +62,10 @@ block_time_ms =
 
 config :lambda_ethereum_consensus, LambdaEthereumConsensus.Telemetry,
   block_processing_buckets: [0.5, 1.0, 1.5, 2, 4, 6, 8] |> Enum.map(&(&1 * block_time_ms))
+
+if is_nil(jwt_secret) do
+  Logger.warning(
+    "[EngineAPI] A JWT secret is needed for communication with the execution engine. " <>
+      "Please specify the file to load it from with the --execution-jwt flag."
+  )
+end

lib/beacon_api/api_spec.ex

@@ -0,0 +1,15 @@
+defmodule BeaconApi.ApiSpec do
+  @moduledoc false
+  alias OpenApiSpex.OpenApi
+  @behaviour OpenApi
+
+  file = "beacon-node-oapi.json"
+  @external_resource file
+  @ethspec file
+           |> File.read!()
+           |> Jason.decode!()
+           |> OpenApiSpex.OpenApi.Decode.decode()
+
+  @impl OpenApi
+  def spec, do: @ethspec
+end

lib/beacon_api/controllers/v1/beacon_controller.ex

@@ -1,11 +1,25 @@
 defmodule BeaconApi.V1.BeaconController do
+  alias BeaconApi.ApiSpec
   alias BeaconApi.ErrorController
+
   alias LambdaEthereumConsensus.ForkChoice
   alias LambdaEthereumConsensus.Store.BlockStore
   use BeaconApi, :controller
 
+  plug(OpenApiSpex.Plug.CastAndValidate, json_render_error_v2: true)
+
+  @doc """
+  action is an atom that correspond to the controller action's function atoms declared on `BeaconApi.Router` 
+  """
+  def open_api_operation(action) when is_atom(action) do
+    apply(__MODULE__, :"#{action}_operation", [])
+  end
+
+  def get_state_root_operation,
+    do: ApiSpec.spec().paths["/eth/v1/beacon/states/{state_id}/root"].get
+
   @spec get_state_root(Plug.Conn.t(), any) :: Plug.Conn.t()
-  def get_state_root(conn, %{"state_id" => state_id}) do
+  def get_state_root(conn, %{state_id: state_id}) do
     with {:ok, {root, execution_optimistic, finalized}} <-
            BeaconApi.Utils.parse_id(state_id) |> ForkChoice.Helpers.root_by_id(),
          {:ok, state_root} <- ForkChoice.Helpers.get_state_root(root) do
@@ -25,28 +39,31 @@ defmodule BeaconApi.V1.BeaconController do
     end
   end
 
+  def get_block_root_operation,
+    do: ApiSpec.spec().paths["/eth/v1/beacon/blocks/{block_id}/root"].get
+
   @spec get_block_root(Plug.Conn.t(), any) :: Plug.Conn.t()
-  def get_block_root(conn, %{"block_id" => "head"}) do
+  def get_block_root(conn, %{block_id: "head"}) do
     # TODO: determine head and return it
     conn |> block_not_found()
   end
 
-  def get_block_root(conn, %{"block_id" => "finalized"}) do
+  def get_block_root(conn, %{block_id: "finalized"}) do
     # TODO
     conn |> block_not_found()
   end
 
-  def get_block_root(conn, %{"block_id" => "justified"}) do
+  def get_block_root(conn, %{block_id: "justified"}) do
     # TODO
     conn |> block_not_found()
   end
 
-  def get_block_root(conn, %{"block_id" => "genesis"}) do
+  def get_block_root(conn, %{block_id: "genesis"}) do
     # TODO
     conn |> block_not_found()
   end
 
-  def get_block_root(conn, %{"block_id" => "0x" <> hex_block_id}) do
+  def get_block_root(conn, %{block_id: "0x" <> hex_block_id}) do
     with {:ok, block_root} <- Base.decode16(hex_block_id, case: :mixed),
          {:ok, _signed_block} <- BlockStore.get_block(block_root) do
       conn |> root_response(block_root, true, false)
@@ -56,7 +73,7 @@ defmodule BeaconApi.V1.BeaconController do
     end
   end
 
-  def get_block_root(conn, %{"block_id" => block_id}) do
+  def get_block_root(conn, %{block_id: block_id}) do
     with {slot, ""} when slot >= 0 <- Integer.parse(block_id),
          {:ok, block_root} <- BlockStore.get_block_root_by_slot(slot) do
       conn |> root_response(block_root, true, false)