Merge latest master to electra branch & add dynssz annotations to new electra types

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/holiman/uint256 v1.2.4
 	github.com/huandu/go-clone v1.6.0
 	github.com/huandu/go-clone/generic v1.6.0
+	github.com/pk910/dynamic-ssz v0.0.3
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.16.0
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20240328144219-a1caa50c3a1e
@@ -46,6 +47,7 @@ require (
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/Knetic/govaluate.v3 v3.0.0 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -65,6 +65,8 @@ github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dz
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/pk910/dynamic-ssz v0.0.3 h1:fCWzFowq9P6SYCc7NtJMkZcIHk+r5hSVD+32zVi6Aio=
+github.com/pk910/dynamic-ssz v0.0.3/go.mod h1:b6CrLaB2X7pYA+OSEEbkgXDEcRnjLOZIxZTsMuO/Y9c=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -130,6 +132,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/Knetic/govaluate.v3 v3.0.0 h1:18mUyIt4ZlRlFZAAfVetz4/rzlJs9yhN+U02F4u1AOc=
+gopkg.in/Knetic/govaluate.v3 v3.0.0/go.mod h1:csKLBORsPbafmSCGTEh3U7Ozmsuq8ZSIlKk1bcqph0E=
 gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
 gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

http/beaconstate.go

@@ -29,6 +29,7 @@ import (
 	"github.com/attestantio/go-eth2-client/spec/capella"
 	"github.com/attestantio/go-eth2-client/spec/deneb"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
+	dynssz "github.com/pk910/dynamic-ssz"
 )
 
 // BeaconState fetches a beacon state.
@@ -56,46 +57,82 @@ func (s *Service) BeaconState(ctx context.Context,
 
 	switch httpResponse.contentType {
 	case ContentTypeSSZ:
-		return s.beaconStateFromSSZ(httpResponse)
+		return s.beaconStateFromSSZ(ctx, httpResponse)
 	case ContentTypeJSON:
 		return s.beaconStateFromJSON(httpResponse)
 	default:
 		return nil, fmt.Errorf("unhandled content type %v", httpResponse.contentType)
 	}
 }
 
-func (*Service) beaconStateFromSSZ(res *httpResponse) (*api.Response[*spec.VersionedBeaconState], error) {
+func (s *Service) beaconStateFromSSZ(ctx context.Context, res *httpResponse) (*api.Response[*spec.VersionedBeaconState], error) {
 	response := &api.Response[*spec.VersionedBeaconState]{
 		Data: &spec.VersionedBeaconState{
 			Version: res.consensusVersion,
 		},
 		Metadata: metadataFromHeaders(res.headers),
 	}
 
+	var dynSSZ *dynssz.DynSsz
+	if s.customSpecSupport {
+		specs, err := s.Spec(ctx, &api.SpecOpts{})
+		if err != nil {
+			return nil, errors.Join(errors.New("failed to request specs"), err)
+		}
+
+		dynSSZ = dynssz.NewDynSsz(specs.Data)
+	}
+
+	var err error
 	switch res.consensusVersion {
 	case spec.DataVersionPhase0:
 		response.Data.Phase0 = &phase0.BeaconState{}
-		if err := response.Data.Phase0.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Phase0, res.body)
+		} else {
+			err = response.Data.Phase0.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode phase0 beacon state"), err)
 		}
 	case spec.DataVersionAltair:
 		response.Data.Altair = &altair.BeaconState{}
-		if err := response.Data.Altair.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Altair, res.body)
+		} else {
+			err = response.Data.Altair.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode altair beacon state"), err)
 		}
 	case spec.DataVersionBellatrix:
 		response.Data.Bellatrix = &bellatrix.BeaconState{}
-		if err := response.Data.Bellatrix.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Bellatrix, res.body)
+		} else {
+			err = response.Data.Bellatrix.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode bellatrix beacon state"), err)
 		}
 	case spec.DataVersionCapella:
 		response.Data.Capella = &capella.BeaconState{}
-		if err := response.Data.Capella.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Capella, res.body)
+		} else {
+			err = response.Data.Capella.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode capella beacon state"), err)
 		}
 	case spec.DataVersionDeneb:
 		response.Data.Deneb = &deneb.BeaconState{}
-		if err := response.Data.Deneb.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Deneb, res.body)
+		} else {
+			err = response.Data.Deneb.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode deneb beacon state"), err)
 		}
 	case spec.DataVersionElectra:

http/parameters.go

@@ -33,6 +33,7 @@ type parameters struct {
 	allowDelayedStart  bool
 	hooks              *Hooks
 	reducedMemoryUsage bool
+	customSpecSupport  bool
 }
 
 // Parameter is the interface for service parameters.
@@ -124,6 +125,14 @@ func WithReducedMemoryUsage(reducedMemoryUsage bool) Parameter {
 	})
 }
 
+// WithCustomSpecSupport switches from the built in static SSZ library to a new dynamic SSZ library, which is able to handle non-mainnet presets.
+// Dynamic SSZ en-/decoding is much slower than the static one, so this should only be used if required.
+func WithCustomSpecSupport(customSpecSupport bool) Parameter {
+	return parameterFunc(func(p *parameters) {
+		p.customSpecSupport = customSpecSupport
+	})
+}
+
 // parseAndCheckParameters parses and checks parameters to ensure that mandatory parameters are present and correct.
 func parseAndCheckParameters(params ...Parameter) (*parameters, error) {
 	parameters := parameters{

http/proposal.go

@@ -33,6 +33,7 @@ import (
 	"github.com/attestantio/go-eth2-client/spec/bellatrix"
 	"github.com/attestantio/go-eth2-client/spec/capella"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
+	dynssz "github.com/pk910/dynamic-ssz"
 	"go.opentelemetry.io/otel"
 )
 
@@ -80,7 +81,7 @@ func (s *Service) Proposal(ctx context.Context,
 	var response *api.Response[*api.VersionedProposal]
 	switch httpResponse.contentType {
 	case ContentTypeSSZ:
-		response, err = s.beaconBlockProposalFromSSZ(httpResponse)
+		response, err = s.beaconBlockProposalFromSSZ(ctx, httpResponse)
 	case ContentTypeJSON:
 		response, err = s.beaconBlockProposalFromJSON(httpResponse)
 	default:
@@ -114,7 +115,8 @@ func (s *Service) Proposal(ctx context.Context,
 	return response, nil
 }
 
-func (s *Service) beaconBlockProposalFromSSZ(res *httpResponse) (*api.Response[*api.VersionedProposal], error) {
+//nolint:nestif
+func (s *Service) beaconBlockProposalFromSSZ(ctx context.Context, res *httpResponse) (*api.Response[*api.VersionedProposal], error) {
 	response := &api.Response[*api.VersionedProposal]{
 		Data: &api.VersionedProposal{
 			Version:        res.consensusVersion,
@@ -128,37 +130,79 @@ func (s *Service) beaconBlockProposalFromSSZ(res *httpResponse) (*api.Response[*
 		return nil, err
 	}
 
+	var dynSSZ *dynssz.DynSsz
+	if s.customSpecSupport {
+		specs, err := s.Spec(ctx, &api.SpecOpts{})
+		if err != nil {
+			return nil, errors.Join(errors.New("failed to request specs"), err)
+		}
+
+		dynSSZ = dynssz.NewDynSsz(specs.Data)
+	}
+
 	var err error
 	switch res.consensusVersion {
 	case spec.DataVersionPhase0:
 		response.Data.Phase0 = &phase0.BeaconBlock{}
-		err = response.Data.Phase0.UnmarshalSSZ(res.body)
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Phase0, res.body)
+		} else {
+			err = response.Data.Phase0.UnmarshalSSZ(res.body)
+		}
 	case spec.DataVersionAltair:
 		response.Data.Altair = &altair.BeaconBlock{}
-		err = response.Data.Altair.UnmarshalSSZ(res.body)
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Altair, res.body)
+		} else {
+			err = response.Data.Altair.UnmarshalSSZ(res.body)
+		}
 	case spec.DataVersionBellatrix:
 		if response.Data.Blinded {
 			response.Data.BellatrixBlinded = &apiv1bellatrix.BlindedBeaconBlock{}
-			err = response.Data.BellatrixBlinded.UnmarshalSSZ(res.body)
+			if s.customSpecSupport {
+				err = dynSSZ.UnmarshalSSZ(response.Data.BellatrixBlinded, res.body)
+			} else {
+				err = response.Data.BellatrixBlinded.UnmarshalSSZ(res.body)
+			}
 		} else {
 			response.Data.Bellatrix = &bellatrix.BeaconBlock{}
-			err = response.Data.Bellatrix.UnmarshalSSZ(res.body)
+			if s.customSpecSupport {
+				err = dynSSZ.UnmarshalSSZ(response.Data.Bellatrix, res.body)
+			} else {
+				err = response.Data.Bellatrix.UnmarshalSSZ(res.body)
+			}
 		}
 	case spec.DataVersionCapella:
 		if response.Data.Blinded {
 			response.Data.CapellaBlinded = &apiv1capella.BlindedBeaconBlock{}
-			err = response.Data.CapellaBlinded.UnmarshalSSZ(res.body)
+			if s.customSpecSupport {
+				err = dynSSZ.UnmarshalSSZ(response.Data.CapellaBlinded, res.body)
+			} else {
+				err = response.Data.CapellaBlinded.UnmarshalSSZ(res.body)
+			}
 		} else {
 			response.Data.Capella = &capella.BeaconBlock{}
-			err = response.Data.Capella.UnmarshalSSZ(res.body)
+			if s.customSpecSupport {
+				err = dynSSZ.UnmarshalSSZ(response.Data.Capella, res.body)
+			} else {
+				err = response.Data.Capella.UnmarshalSSZ(res.body)
+			}
 		}
 	case spec.DataVersionDeneb:
 		if response.Data.Blinded {
 			response.Data.DenebBlinded = &apiv1deneb.BlindedBeaconBlock{}
-			err = response.Data.DenebBlinded.UnmarshalSSZ(res.body)
+			if s.customSpecSupport {
+				err = dynSSZ.UnmarshalSSZ(response.Data.DenebBlinded, res.body)
+			} else {
+				err = response.Data.DenebBlinded.UnmarshalSSZ(res.body)
+			}
 		} else {
 			response.Data.Deneb = &apiv1deneb.BlockContents{}
-			err = response.Data.Deneb.UnmarshalSSZ(res.body)
+			if s.customSpecSupport {
+				err = dynSSZ.UnmarshalSSZ(response.Data.Deneb, res.body)
+			} else {
+				err = response.Data.Deneb.UnmarshalSSZ(res.body)
+			}
 		}
 	case spec.DataVersionElectra:
 		if response.Data.Blinded {

http/service.go

@@ -73,6 +73,7 @@ type Service struct {
 	enforceJSON              bool
 	connectedToDVTMiddleware bool
 	reducedMemoryUsage       bool
+	customSpecSupport        bool
 }
 
 // New creates a new Ethereum 2 client service, connecting with a standard HTTP.
@@ -126,6 +127,7 @@ func New(ctx context.Context, params ...Parameter) (client.Service, error) {
 		pingSem:             semaphore.NewWeighted(1),
 		hooks:               parameters.hooks,
 		reducedMemoryUsage:  parameters.reducedMemoryUsage,
+		customSpecSupport:   parameters.customSpecSupport,
 	}
 
 	// Ping the client to see if it is ready to serve requests.

http/signedbeaconblock.go

@@ -29,6 +29,7 @@ import (
 	"github.com/attestantio/go-eth2-client/spec/capella"
 	"github.com/attestantio/go-eth2-client/spec/deneb"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
+	dynssz "github.com/pk910/dynamic-ssz"
 )
 
 // SignedBeaconBlock fetches a signed beacon block given a block ID.
@@ -54,7 +55,7 @@ func (s *Service) SignedBeaconBlock(ctx context.Context,
 	var response *api.Response[*spec.VersionedSignedBeaconBlock]
 	switch httpResponse.contentType {
 	case ContentTypeSSZ:
-		response, err = s.signedBeaconBlockFromSSZ(httpResponse)
+		response, err = s.signedBeaconBlockFromSSZ(ctx, httpResponse)
 	case ContentTypeJSON:
 		response, err = s.signedBeaconBlockFromJSON(httpResponse)
 	default:
@@ -67,38 +68,74 @@ func (s *Service) SignedBeaconBlock(ctx context.Context,
 	return response, nil
 }
 
-func (*Service) signedBeaconBlockFromSSZ(res *httpResponse) (*api.Response[*spec.VersionedSignedBeaconBlock], error) {
+func (s *Service) signedBeaconBlockFromSSZ(ctx context.Context, res *httpResponse) (*api.Response[*spec.VersionedSignedBeaconBlock], error) {
 	response := &api.Response[*spec.VersionedSignedBeaconBlock]{
 		Data: &spec.VersionedSignedBeaconBlock{
 			Version: res.consensusVersion,
 		},
 		Metadata: metadataFromHeaders(res.headers),
 	}
 
+	var dynSSZ *dynssz.DynSsz
+	if s.customSpecSupport {
+		specs, err := s.Spec(ctx, &api.SpecOpts{})
+		if err != nil {
+			return nil, errors.Join(errors.New("failed to request specs"), err)
+		}
+
+		dynSSZ = dynssz.NewDynSsz(specs.Data)
+	}
+
+	var err error
 	switch res.consensusVersion {
 	case spec.DataVersionPhase0:
 		response.Data.Phase0 = &phase0.SignedBeaconBlock{}
-		if err := response.Data.Phase0.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Phase0, res.body)
+		} else {
+			err = response.Data.Phase0.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode phase0 signed beacon block"), err)
 		}
 	case spec.DataVersionAltair:
 		response.Data.Altair = &altair.SignedBeaconBlock{}
-		if err := response.Data.Altair.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Altair, res.body)
+		} else {
+			err = response.Data.Altair.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode altair signed beacon block"), err)
 		}
 	case spec.DataVersionBellatrix:
 		response.Data.Bellatrix = &bellatrix.SignedBeaconBlock{}
-		if err := response.Data.Bellatrix.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Bellatrix, res.body)
+		} else {
+			err = response.Data.Bellatrix.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode bellatrix signed beacon block"), err)
 		}
 	case spec.DataVersionCapella:
 		response.Data.Capella = &capella.SignedBeaconBlock{}
-		if err := response.Data.Capella.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Capella, res.body)
+		} else {
+			err = response.Data.Capella.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode capella signed beacon block"), err)
 		}
 	case spec.DataVersionDeneb:
 		response.Data.Deneb = &deneb.SignedBeaconBlock{}
-		if err := response.Data.Deneb.UnmarshalSSZ(res.body); err != nil {
+		if s.customSpecSupport {
+			err = dynSSZ.UnmarshalSSZ(response.Data.Deneb, res.body)
+		} else {
+			err = response.Data.Deneb.UnmarshalSSZ(res.body)
+		}
+		if err != nil {
 			return nil, errors.Join(errors.New("failed to decode deneb signed block contents"), err)
 		}
 	case spec.DataVersionElectra: