From df9bf51953694e81c05a3706cc5d92c79e0918ac Mon Sep 17 00:00:00 2001 From: f3r10 Date: Thu, 30 Nov 2023 13:00:02 -0500 Subject: [PATCH 1/6] handle invalid error for boolean and uints --- lib/ssz_ex.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index 85acb55eb..6081694c0 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -39,8 +39,10 @@ defmodule LambdaEthereumConsensus.SszEx do @offset_bits 32 defp encode_int(value, size) when is_integer(value), do: {:ok, <>} + defp encode_int(value, _size), do: {:error, "#{inspect(value)} is not a integer"} defp encode_bool(true), do: {:ok, "\x01"} defp encode_bool(false), do: {:ok, "\x00"} + defp encode_bool(not_valid), do: {:error, "#{inspect(not_valid)} not boolean"} defp decode_uint(binary, size) do <> = binary From e4c02fe6f50c9090b3992210fe2b4a92c25a9097 Mon Sep 17 00:00:00 2001 From: f3r10 Date: Thu, 30 Nov 2023 13:00:24 -0500 Subject: [PATCH 2/6] add ssz_generic spec test for boolean and uints --- lib/spec/runners/ssz_generic.ex | 81 +++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 lib/spec/runners/ssz_generic.ex diff --git a/lib/spec/runners/ssz_generic.ex b/lib/spec/runners/ssz_generic.ex new file mode 100644 index 000000000..a6711f333 --- /dev/null +++ b/lib/spec/runners/ssz_generic.ex @@ -0,0 +1,81 @@ +defmodule SszGenericTestRunner do + @moduledoc """ + Runner for SSZ general test cases. `run_test_case/1` is the main entrypoint. + """ + alias LambdaEthereumConsensus.SszEx + use ExUnit.CaseTemplate + use TestRunner + + @disabled [ + "basic_vector", + "bitlist", + "bitvector", + # "boolean", + "containers" + # "uints" + ] + + @impl TestRunner + def skip?(%SpecTestCase{fork: fork, handler: handler}) do + fork != "phase0" or Enum.member?(@disabled, handler) + end + + @impl TestRunner + def run_test_case(%SpecTestCase{} = testcase) do + case_dir = SpecTestCase.dir(testcase) + + schema = parse_type(testcase) + + compressed = File.read!(case_dir <> "/serialized.ssz_snappy") + assert {:ok, decompressed} = :snappyer.decompress(compressed) + + handle_case(testcase.suite, schema, decompressed, testcase) + end + + defp handle_case("valid", schema, real_deserialized, testcase) do + case_dir = SpecTestCase.dir(testcase) + + expected = + YamlElixir.read_from_file!(case_dir <> "/value.yaml") + |> SpecTestUtils.sanitize_yaml() + + assert_ssz("valid", schema, real_deserialized, expected) + end + + defp handle_case("invalid", schema, real_deserialized, _testcase) do + assert_ssz("invalid", schema, real_deserialized) + end + + defp assert_ssz("valid", schema, real_serialized, real_deserialized) do + {:ok, deserialized} = SszEx.decode(real_serialized, schema) + assert deserialized == real_deserialized + + {:ok, serialized} = SszEx.encode(real_deserialized, schema) + + assert serialized == real_serialized + end + + defp assert_ssz("invalid", schema, real_serialized) do + assert {:error, _error} = SszEx.encode(real_serialized, schema) + end + + defp parse_type(%SpecTestCase{handler: handler, case: cse}) do + case handler do + "boolean" -> + :bool + + "uints" -> + case cse do + "uint_" <> _rest -> + [_head, size] = Regex.run(~r/^.*?_(.*?)_.*$/, cse) + {:int, String.to_integer(size)} + + unknown -> + :error + end + + unknown -> + :error + end + end +end From 403085d079675b0c39a5edead11495b6c1e78418 Mon Sep 17 00:00:00 2001 From: f3r10 Date: Tue, 5 Dec 2023 18:21:46 -0500 Subject: [PATCH 3/6] add ssz_static containers modules --- .../fixed_test_struct.ex | 32 ++++++++++ .../single_field_test_struct.ex | 24 +++++++ .../small_test_struct.ex | 29 +++++++++ .../ssz_static_containers/var_test_struct.ex | 32 ++++++++++ lib/spec/runners/ssz_generic.ex | 63 ++++++++++++++----- lib/ssz_ex.ex | 2 - 6 files changed, 166 insertions(+), 16 deletions(-) create mode 100644 lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex diff --git a/lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex new file mode 100644 index 000000000..f8503e43d --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/fixed_test_struct.ex @@ -0,0 +1,32 @@ +defmodule Helpers.SszStaticContainers.FixedTestStruct do + @moduledoc """ + Struct definition for `FixedTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C + ] + + @type uint32 :: 0..unquote(2 ** 32 - 1) + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint8(), + B: SszTypes.uint64(), + C: uint32() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 8}}, + {:B, {:int, 64}}, + {:C, {:int, 32}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex new file mode 100644 index 000000000..6d6655bfc --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/single_field_test_struct.ex @@ -0,0 +1,24 @@ +defmodule Helpers.SszStaticContainers.SingleFieldTestStruct do + @moduledoc """ + Struct definition for `SingleFieldTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.uint8() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 8}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex new file mode 100644 index 000000000..c2f83c00b --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex @@ -0,0 +1,29 @@ +defmodule Helpers.SszStaticContainers.SmallTestStruct do + @moduledoc """ + Struct definition for `SmallTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B + ] + + @type uint16 :: 0..unquote(2 ** 16 - 1) + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: uint16(), + B: uint16() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 16}}, + {:B, {:int, 16}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex new file mode 100644 index 000000000..f9ddc0816 --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex @@ -0,0 +1,32 @@ +defmodule Helpers.SszStaticContainers.VarTestStruct do + @moduledoc """ + Struct definition for `VarTestStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C + ] + + @type uint16 :: 0..unquote(2 ** 16 - 1) + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: uint16(), + B: list(uint16()), + C: SszTypes.uint8() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 16}}, + {:B, {:list, {:int, 16}, 1024}}, + {:C, {:int, 8}} + ] + end +end diff --git a/lib/spec/runners/ssz_generic.ex b/lib/spec/runners/ssz_generic.ex index a6711f333..b53c49537 100644 --- a/lib/spec/runners/ssz_generic.ex +++ b/lib/spec/runners/ssz_generic.ex @@ -9,15 +9,29 @@ defmodule SszGenericTestRunner do @disabled [ "basic_vector", "bitlist", - "bitvector", + "bitvector" # "boolean", - "containers" + # "containers" # "uints" ] + @disabled_containers [ + # "SingleFieldTestStruct", + # "SmallTestStruct", + # "FixedTestStruct", + # "VarTestStruct", + "ComplexTestStruct", + "BitsStruct" + ] + @impl TestRunner - def skip?(%SpecTestCase{fork: fork, handler: handler}) do - fork != "phase0" or Enum.member?(@disabled, handler) + def skip?(%SpecTestCase{fork: fork, handler: handler, case: cse}) do + skip_container? = + @disabled_containers + |> Enum.map(fn container -> String.contains?(cse, container) end) + |> Enum.any?() + + fork != "phase0" or Enum.member?(@disabled, handler) or skip_container? end @impl TestRunner @@ -32,18 +46,26 @@ defmodule SszGenericTestRunner do handle_case(testcase.suite, schema, decompressed, testcase) end - defp handle_case("valid", schema, real_deserialized, testcase) do + defp handle_case("valid", schema, real_serialized, testcase) do case_dir = SpecTestCase.dir(testcase) expected = YamlElixir.read_from_file!(case_dir <> "/value.yaml") |> SpecTestUtils.sanitize_yaml() - assert_ssz("valid", schema, real_deserialized, expected) + assert_ssz("valid", schema, real_serialized, expected) end - defp handle_case("invalid", schema, real_deserialized, _testcase) do - assert_ssz("invalid", schema, real_deserialized) + defp handle_case("invalid", schema, real_serialized, _testcase) do + assert_ssz("invalid", schema, real_serialized) + end + + defp assert_ssz("valid", {:container, module}, real_serialized, real_deserialized) do + real_struct = struct!(module, real_deserialized) + {:ok, deserialized} = SszEx.decode(real_serialized, module) + assert deserialized == real_struct + {:ok, serialized} = SszEx.encode(real_struct, module) + assert serialized == real_serialized end defp assert_ssz("valid", schema, real_serialized, real_deserialized) do @@ -56,7 +78,7 @@ defmodule SszGenericTestRunner do end defp assert_ssz("invalid", schema, real_serialized) do - assert {:error, _error} = SszEx.encode(real_serialized, schema) + catch_error(SszEx.encode(real_serialized, schema)) end defp parse_type(%SpecTestCase{handler: handler, case: cse}) do @@ -69,13 +91,26 @@ defmodule SszGenericTestRunner do "uint_" <> _rest -> [_head, size] = Regex.run(~r/^.*?_(.*?)_.*$/, cse) {:int, String.to_integer(size)} - - unknown -> - :error end - unknown -> - :error + "containers" -> + [name] = Regex.run(~r/^[^_]+(?=_)/, cse) + {:container, Module.concat(Helpers.SszStaticContainers, name)} + # "basic_vector" -> + # case cse do + # "vec_" <> rest -> + # case String.split(rest, "_") do + # ["bool", max_size | _] -> {:vector, :bool, String.to_integer(max_size)} + # ["uint" <> size, max_size | _] -> + # {:vector, {:int, String.to_integer(size)}, String.to_integer(max_size)} + # end + # end + # "bitlist" -> + # case cse do + # "bitlist_" <> rest -> + # [size | _] = String.split(rest, "_") + # {:bitlist, :bool, String.to_integer(size)} + # end end end end diff --git a/lib/ssz_ex.ex b/lib/ssz_ex.ex index d5bc648b3..e46db263c 100644 --- a/lib/ssz_ex.ex +++ b/lib/ssz_ex.ex @@ -44,10 +44,8 @@ defmodule LambdaEthereumConsensus.SszEx do @offset_bits 32 defp encode_int(value, size) when is_integer(value), do: {:ok, <>} - defp encode_int(value, _size), do: {:error, "#{inspect(value)} is not a integer"} defp encode_bool(true), do: {:ok, "\x01"} defp encode_bool(false), do: {:ok, "\x00"} - defp encode_bool(not_valid), do: {:error, "#{inspect(not_valid)} not boolean"} defp decode_uint(binary, size) do <> = binary From e7b88b63258352abd587672940ededf921ad1245 Mon Sep 17 00:00:00 2001 From: f3r10 Date: Wed, 6 Dec 2023 17:31:42 -0500 Subject: [PATCH 4/6] add generic containers --- .../ssz_static_containers/bits_struct.ex | 36 +++++++++++++++ .../complex_test_struct.ex | 46 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex create mode 100644 lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex diff --git a/lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex new file mode 100644 index 000000000..a23932c75 --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/bits_struct.ex @@ -0,0 +1,36 @@ +defmodule Helpers.SszStaticContainers.BitsStruct do + @moduledoc """ + Struct definition for `BitsStruct`. + """ + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C, + :D, + :E + ] + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: SszTypes.bitlist(), + B: SszTypes.bitvector(), + C: SszTypes.bitvector(), + D: SszTypes.bitlist(), + E: SszTypes.bitvector() + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:bitlist, 5}}, + {:B, {:bitvector, 2}}, + {:C, {:bitvector, 1}}, + {:D, {:bitlist, 6}}, + {:E, {:bitvector, 8}} + ] + end +end diff --git a/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex new file mode 100644 index 000000000..449141a72 --- /dev/null +++ b/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex @@ -0,0 +1,46 @@ +defmodule Helpers.SszStaticContainers.ComplexTestStruct do + @moduledoc """ + Struct definition for `ComplexTestStruct`. + """ + alias Helpers.SszStaticContainers.FixedTestStruct + alias Helpers.SszStaticContainers.VarTestStruct + @behaviour LambdaEthereumConsensus.Container + + fields = [ + :A, + :B, + :C, + :D, + :E, + :F, + :G + ] + + @type uint16 :: 0..unquote(2 ** 16 - 1) + + @enforce_keys fields + defstruct fields + + @type t :: %__MODULE__{ + A: uint16(), + B: list(uint16()), + C: SszTypes.uint8(), + D: SszTypes.bitlist(), + E: VarTestStruct, + F: list(FixedTestStruct), + G: list(VarTestStruct) + } + + @impl LambdaEthereumConsensus.Container + def schema do + [ + {:A, {:int, 16}}, + {:B, {:list, {:int, 16}, 1024}}, + {:C, {:int, 8}}, + {:D, {:bitlist, 256}}, + {:E, VarTestStruct}, + {:F, {:vector, FixedTestStruct, 4}}, + {:G, {:vector, VarTestStruct, 2}} + ] + end +end From c57aafc6fe41cd7fb2f307b27c578902a827ada5 Mon Sep 17 00:00:00 2001 From: f3r10 Date: Mon, 11 Dec 2023 07:04:11 -0500 Subject: [PATCH 5/6] fix small tweaks --- .../ssz_static_containers/small_test_struct.ex | 6 ++---- .../helpers/ssz_static_containers/var_test_struct.ex | 6 ++---- lib/spec/runners/ssz_generic.ex | 11 ++++++++--- lib/ssz_types/mod.ex | 1 + 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex index c2f83c00b..b54f8e8d7 100644 --- a/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex +++ b/lib/spec/runners/helpers/ssz_static_containers/small_test_struct.ex @@ -9,14 +9,12 @@ defmodule Helpers.SszStaticContainers.SmallTestStruct do :B ] - @type uint16 :: 0..unquote(2 ** 16 - 1) - @enforce_keys fields defstruct fields @type t :: %__MODULE__{ - A: uint16(), - B: uint16() + A: SszTypes.uint16(), + B: SszTypes.uint16() } @impl LambdaEthereumConsensus.Container diff --git a/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex index f9ddc0816..60d750e41 100644 --- a/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex +++ b/lib/spec/runners/helpers/ssz_static_containers/var_test_struct.ex @@ -10,14 +10,12 @@ defmodule Helpers.SszStaticContainers.VarTestStruct do :C ] - @type uint16 :: 0..unquote(2 ** 16 - 1) - @enforce_keys fields defstruct fields @type t :: %__MODULE__{ - A: uint16(), - B: list(uint16()), + A: SszTypes.uint16(), + B: list(SszTypes.uint16()), C: SszTypes.uint8() } diff --git a/lib/spec/runners/ssz_generic.ex b/lib/spec/runners/ssz_generic.ex index b53c49537..75a99e13b 100644 --- a/lib/spec/runners/ssz_generic.ex +++ b/lib/spec/runners/ssz_generic.ex @@ -96,20 +96,25 @@ defmodule SszGenericTestRunner do "containers" -> [name] = Regex.run(~r/^[^_]+(?=_)/, cse) {:container, Module.concat(Helpers.SszStaticContainers, name)} + + # TODO enable when basic_vector and bitlist tests are enable # "basic_vector" -> # case cse do # "vec_" <> rest -> # case String.split(rest, "_") do - # ["bool", max_size | _] -> {:vector, :bool, String.to_integer(max_size)} + # ["bool", max_size | _] -> + # {:vector, :bool, String.to_integer(max_size)} + # # ["uint" <> size, max_size | _] -> - # {:vector, {:int, String.to_integer(size)}, String.to_integer(max_size)} + # {:vector, {:int, String.to_integer(size)}, String.to_integer(max_size)} # end # end + # # "bitlist" -> # case cse do # "bitlist_" <> rest -> # [size | _] = String.split(rest, "_") - # {:bitlist, :bool, String.to_integer(size)} + # {:bitlist, String.to_integer(size)} # end end end diff --git a/lib/ssz_types/mod.ex b/lib/ssz_types/mod.ex index b471ac074..9da84782e 100644 --- a/lib/ssz_types/mod.ex +++ b/lib/ssz_types/mod.ex @@ -6,6 +6,7 @@ defmodule SszTypes do # Primitive types ## Integer types @type uint8 :: 0..unquote(2 ** 8 - 1) + @type uint16 :: 0..unquote(2 ** 16 - 1) @type uint64 :: 0..unquote(2 ** 64 - 1) @type uint256 :: 0..unquote(2 ** 256 - 1) From 01b39715a8d19ed893988579cf15f0a00f14ffc3 Mon Sep 17 00:00:00 2001 From: f3r10 Date: Mon, 11 Dec 2023 07:08:49 -0500 Subject: [PATCH 6/6] fix small tweak --- .../helpers/ssz_static_containers/complex_test_struct.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex b/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex index 449141a72..ff85ca3cc 100644 --- a/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex +++ b/lib/spec/runners/helpers/ssz_static_containers/complex_test_struct.ex @@ -16,14 +16,12 @@ defmodule Helpers.SszStaticContainers.ComplexTestStruct do :G ] - @type uint16 :: 0..unquote(2 ** 16 - 1) - @enforce_keys fields defstruct fields @type t :: %__MODULE__{ - A: uint16(), - B: list(uint16()), + A: SszTypes.uint16(), + B: list(SszTypes.uint16()), C: SszTypes.uint8(), D: SszTypes.bitlist(), E: VarTestStruct,