Tweak validator registrations.

Store validator registrations based on hash of the registration (minus
the timestamp).  This avoids duplication of registrations when the same
data is provided to multiple relays, reducing both CPU usage and memory
footprint.

CHANGELOG.md

@@ -2,6 +2,7 @@ dev:
   - add User-Agent header to HTTP requests
   - controller only uses beacon nodes that are used for attestation data
   - fix crash if beacon node returns nil block during cache update
+  - increase speed of validator registration generation, reduce memory usage
 
 1.7.5:
   - add score of block proposals to tracing

go.mod

@@ -10,14 +10,14 @@ require (
 	github.com/holiman/uint256 v1.2.2
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.15.1
+	github.com/prometheus/client_golang v1.16.0
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7
 	github.com/rs/zerolog v1.29.1
 	github.com/sasha-s/go-deadlock v0.3.1
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
-	github.com/stretchr/testify v1.8.3
+	github.com/stretchr/testify v1.8.4
 	github.com/wealdtech/go-bytesutil v1.2.1
 	github.com/wealdtech/go-eth2-types/v2 v2.8.1
 	github.com/wealdtech/go-eth2-wallet v1.15.1
@@ -114,11 +114,11 @@ require (
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/crypto v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sys v0.9.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.125.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -329,8 +329,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
-github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
@@ -380,8 +380,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
@@ -473,8 +474,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
+golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -687,8 +688,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

services/blockrelay/standard/service.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Attestant Limited.
+// Copyright © 2022, 2023 Attestant Limited.
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -35,6 +35,7 @@ import (
 	zerologger "github.com/rs/zerolog/log"
 	e2types "github.com/wealdtech/go-eth2-types/v2"
 	"github.com/wealdtech/go-majordomo"
+	"golang.org/x/sync/semaphore"
 )
 
 // Service is the builder service for Vouch.
@@ -54,7 +55,7 @@ type Service struct {
 	builderBidsCache                          map[string]map[string]*builderspec.VersionedSignedBuilderBid
 	builderBidsCacheMu                        sync.RWMutex
 	timeout                                   time.Duration
-	signedValidatorRegistrations              map[string]*apiv1.SignedValidatorRegistration
+	signedValidatorRegistrations              map[phase0.Root]*apiv1.SignedValidatorRegistration
 	signedValidatorRegistrationsMu            sync.RWMutex
 	secondaryValidatorRegistrationsSubmitters []consensusclient.ValidatorRegistrationsSubmitter
 	logResults                                bool
@@ -66,6 +67,7 @@ type Service struct {
 
 	relayPubkeys   map[phase0.BLSPubKey]*e2types.BLSPublicKey
 	relayPubkeysMu sync.RWMutex
+	activitySem    *semaphore.Weighted
 }
 
 // module-wide log.
@@ -120,14 +122,15 @@ func New(ctx context.Context, params ...Parameter) (*Service, error) {
 		validatingAccountsProvider:   parameters.validatingAccountsProvider,
 		validatorRegistrationSigner:  parameters.validatorRegistrationSigner,
 		timeout:                      parameters.timeout,
-		signedValidatorRegistrations: make(map[string]*apiv1.SignedValidatorRegistration),
+		signedValidatorRegistrations: make(map[phase0.Root]*apiv1.SignedValidatorRegistration),
 		secondaryValidatorRegistrationsSubmitters: parameters.secondaryValidatorRegistrationsSubmitters,
 		logResults:               parameters.logResults,
 		applicationBuilderDomain: domain,
 		releaseVersion:           parameters.releaseVersion,
 		builderBidsCache:         make(map[string]map[string]*builderspec.VersionedSignedBuilderBid),
 		relayPubkeys:             make(map[phase0.BLSPubKey]*e2types.BLSPublicKey),
 		executionConfig:          &v2.ExecutionConfig{Version: 2},
+		activitySem:              semaphore.NewWeighted(1),
 	}
 
 	// Carry out initial fetch of execution configuration.

services/blockrelay/standard/submitvalidatorregistrations.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Attestant Limited.
+// Copyright © 2022, 2023 Attestant Limited.
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -14,7 +14,6 @@
 package standard
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -70,6 +69,13 @@ func (s *Service) submitValidatorRegistrations(ctx context.Context,
 	defer span.End()
 	started := time.Now()
 
+	// Only allow a single registration at a time.
+	if !s.activitySem.TryAcquire(1) {
+		log.Trace().Msg("Another validator registration submission in progress; skipping")
+		return
+	}
+	defer s.activitySem.Release(1)
+
 	epoch := s.chainTime.CurrentEpoch()
 
 	// Fetch the validating accounts for the next epoch, to ensure that we capture any validators
@@ -227,30 +233,28 @@ func (s *Service) generateValidatorRegistrationForRelay(ctx context.Context,
 	*consensusapi.VersionedSignedValidatorRegistration,
 	error,
 ) {
+	// Create a registration without a timestamp, to allow matching its hash
+	// with an existing registration (also registered without timestamp).
+	registration := &apiv1.ValidatorRegistration{
+		FeeRecipient: relayConfig.FeeRecipient,
+		GasLimit:     relayConfig.GasLimit,
+		Pubkey:       pubkey,
+	}
+	registrationRoot, err := registration.HashTreeRoot()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to obtain hash tree root of registration")
+	}
+	// Now add the timestamp, for completeness of the struct.
+	registration.Timestamp = time.Now().Round(time.Second)
+
 	// See if we already have a signed registration that matches this configuration.
-	key := fmt.Sprintf("%x:%s", pubkey, relayConfig.Address)
 	s.signedValidatorRegistrationsMu.RLock()
-	signedRegistration, exists := s.signedValidatorRegistrations[key]
+	signedRegistration, exists := s.signedValidatorRegistrations[registrationRoot]
 	s.signedValidatorRegistrationsMu.RUnlock()
 	if exists {
-		// See if the details of the pre-signed registration match our parameters.
-		if bytes.Equal(relayConfig.FeeRecipient[:], signedRegistration.Message.FeeRecipient[:]) &&
-			relayConfig.GasLimit == signedRegistration.Message.GasLimit {
-			monitorRegistrationsGeneration("cache")
-		} else {
-			signedRegistration = nil
-		}
-	}
-	if signedRegistration == nil {
-		log.Trace().Msg("No signed registration; creating one")
-		// Need to build and sign a new registration.
-		registration := &apiv1.ValidatorRegistration{
-			FeeRecipient: relayConfig.FeeRecipient,
-			GasLimit:     relayConfig.GasLimit,
-			Timestamp:    time.Now().Round(time.Second),
-			Pubkey:       pubkey,
-		}
-
+		monitorRegistrationsGeneration("cache")
+	} else {
+		log.Trace().Msg("Signing the validator registration")
 		sig, err := s.validatorRegistrationSigner.SignValidatorRegistration(ctx, account, &builderapi.VersionedValidatorRegistration{
 			Version: builderspec.BuilderVersionV1,
 			V1:      registration,
@@ -264,7 +268,7 @@ func (s *Service) generateValidatorRegistrationForRelay(ctx context.Context,
 			Signature: sig,
 		}
 		s.signedValidatorRegistrationsMu.Lock()
-		s.signedValidatorRegistrations[key] = signedRegistration
+		s.signedValidatorRegistrations[registrationRoot] = signedRegistration
 		s.signedValidatorRegistrationsMu.Unlock()
 		monitorRegistrationsGeneration("generation")
 	}