[Merged by Bors] - Simplify switch to different poet services

README.md

@@ -202,7 +202,7 @@ the build folder you need to ensure that you have the gpu setup dynamic library
 binary. The simplest way to do this is just copy the library file to be in the
 same directory as the go-spacemesh binary. Alternatively you can modify your
 system's library search paths (e.g. LD_LIBRARY_PATH) to ensure that the
-library is found._
+library is found.
 
 go-spacemesh is p2p software which is designed to form a decentralized network by connecting to other instances of
 go-spacemesh running on remote computers.

activation/activation_errors.go

@@ -12,6 +12,10 @@ var (
 	ErrPoetServiceUnstable = &PoetSvcUnstableError{}
 	// ErrPoetProofNotReceived is returned when no poet proof was received.
 	ErrPoetProofNotReceived = errors.New("builder: didn't receive any poet proof")
+	// ErrNoRegistrationForGivenPoetFound is returned when there are existing registrations for given node id
+	// in current poet round, but for other poet services and poet round has already started.
+	// So poet proof will not be fetched.
+	ErrNoRegistrationForGivenPoetFound = errors.New("builder: no registration found for given poet set")
 )
 
 // PoetSvcUnstableError means there was a problem communicating

activation/nipost.go

@@ -8,10 +8,13 @@
 	"math/rand/v2"
 	"time"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/spacemeshos/merkle-tree"
 	"github.com/spacemeshos/poet/shared"
 	postshared "github.com/spacemeshos/post/shared"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/spacemeshos/go-spacemesh/activation/metrics"
@@ -86,8 +89,7 @@
 	opts ...NIPostBuilderOption,
 ) (*NIPostBuilder, error) {
 	b := &NIPostBuilder{
-		localDB: db,
-
+		localDB:     db,
 		postService: postService,
 		logger:      lg,
 		poetCfg:     poetCfg,
@@ -210,7 +212,7 @@
 
 	poetRoundStart := nb.layerClock.LayerToTime((postChallenge.PublishEpoch - 1).FirstLayer()).
 		Add(nb.poetCfg.PhaseShift)
-	poetRoundEnd := nb.layerClock.LayerToTime(postChallenge.PublishEpoch.FirstLayer()).
+	curPoetRoundEnd := nb.layerClock.LayerToTime(postChallenge.PublishEpoch.FirstLayer()).
 		Add(nb.poetCfg.PhaseShift).
 		Add(-nb.poetCfg.CycleGap)
 
@@ -224,41 +226,20 @@
 
 	logger.Info("building nipost",
 		zap.Time("poet round start", poetRoundStart),
-		zap.Time("poet round end", poetRoundEnd),
+		zap.Time("poet round end", curPoetRoundEnd),
 		zap.Time("publish epoch end", publishEpochEnd),
 		zap.Uint32("publish epoch", postChallenge.PublishEpoch.Uint32()),
 	)
 
 	// Phase 0: Submit challenge to PoET services.
-	count, err := nipost.PoetRegistrationCount(nb.localDB, signer.NodeID())
+	// Deadline: start of PoET round: we will not accept registrations after that
+	submitedRegistrations, err := nb.submitPoetChallenges(
+		ctx,
+		signer,
+		poetProofDeadline,
+		poetRoundStart, challenge.Bytes())
 	if err != nil {
-		return nil, fmt.Errorf("failed to get poet registration count: %w", err)
-	}
-	if count == 0 {
-		now := time.Now()
-		// Deadline: start of PoET round for publish epoch. PoET won't accept registrations after that.
-		if poetRoundStart.Before(now) {
-			return nil, fmt.Errorf(
-				"%w: poet round has already started at %s (now: %s)",
-				ErrATXChallengeExpired,
-				poetRoundStart,
-				now,
-			)
-		}
-
-		submitCtx, cancel := context.WithDeadline(ctx, poetRoundStart)
-		defer cancel()
-		err := nb.submitPoetChallenges(submitCtx, signer, poetProofDeadline, challenge.Bytes())
-		if err != nil {
-			return nil, fmt.Errorf("submitting to poets: %w", err)
-		}
-		count, err := nipost.PoetRegistrationCount(nb.localDB, signer.NodeID())
-		if err != nil {
-			return nil, fmt.Errorf("failed to get poet registration count: %w", err)
-		}
-		if count == 0 {
-			return nil, &PoetSvcUnstableError{msg: "failed to submit challenge to any PoET", source: submitCtx.Err()}
-		}
+		return nil, fmt.Errorf("submitting to poets: %w", err)
 	}
 
 	// Phase 1: query PoET services for proofs
@@ -280,8 +261,8 @@
 			)
 		}
 
-		events.EmitPoetWaitProof(signer.NodeID(), postChallenge.PublishEpoch, poetRoundEnd)
-		poetProofRef, membership, err = nb.getBestProof(ctx, signer.NodeID(), challenge, postChallenge.PublishEpoch)
+		events.EmitPoetWaitProof(signer.NodeID(), postChallenge.PublishEpoch, curPoetRoundEnd)
+		poetProofRef, membership, err = nb.getBestProof(ctx, signer.NodeID(), challenge, submitedRegistrations)
 		if err != nil {
 			return nil, &PoetSvcUnstableError{msg: "getBestProof failed", source: err}
 		}
@@ -363,7 +344,7 @@
 	client PoetService,
 	prefix, challenge []byte,
 	signature types.EdSignature,
-) error {
+) (nipost.PoETRegistration, error) {
 	logger := nb.logger.With(
 		log.ZContext(ctx),
 		zap.String("poet", client.Address()),
@@ -377,34 +358,106 @@
 
 	round, err := client.Submit(submitCtx, deadline, prefix, challenge, signature, nodeID)
 	if err != nil {
-		return &PoetSvcUnstableError{msg: "failed to submit challenge to poet service", source: err}
+		return nipost.PoETRegistration{}, &PoetSvcUnstableError{msg: "failed to submit challenge to poet service", source: err}
 	}
 	logger.Info("challenge submitted to poet proving service", zap.String("round", round.ID))
-	return nipost.AddPoetRegistration(nb.localDB, nodeID, nipost.PoETRegistration{
+
+	registration := nipost.PoETRegistration{
 		ChallengeHash: types.Hash32(challenge),
 		Address:       client.Address(),
 		RoundID:       round.ID,
 		RoundEnd:      round.End,
-	})
+	}
+
+	if err := nipost.AddPoetRegistration(nb.localDB, nodeID, registration); err != nil {
+		return nipost.PoETRegistration{}, err
+	}
+
+	return registration, err
 }
 
-// Submit the challenge to all registered PoETs.
+// submitPoetChallenges submit the challenge to registered PoETs
+// if some registrations are missing and PoET round didn't start.
 func (nb *NIPostBuilder) submitPoetChallenges(
 	ctx context.Context,
 	signer *signing.EdSigner,
-	deadline time.Time,
+	poetProofDeadline time.Time,
+	curPoetRoundStartDeadline time.Time,
 	challenge []byte,
-) error {
+) ([]nipost.PoETRegistration, error) {
+	// check if some registrations missing or were removed
+	nodeID := signer.NodeID()
+	registrations, err := nipost.PoetRegistrations(nb.localDB, nodeID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get poet registrations from db: %w", err)
+	}
+
+	registrationsMap := make(map[string]nipost.PoETRegistration)
+	for _, reg := range registrations {
+		registrationsMap[reg.Address] = reg
+	}
+
+	existingRegistrationsMap := make(map[string]nipost.PoETRegistration)
+	missingRegistrationsPoETAddresses := make(map[string]struct{})
+
+	for addr := range nb.poetProvers {
+		if val, ok := registrationsMap[addr]; ok {
+			existingRegistrationsMap[addr] = val
+		} else {
+			missingRegistrationsPoETAddresses[addr] = struct{}{}
+		}
+	}
+
+	existingRegistrations := maps.Values(existingRegistrationsMap)
+	if len(missingRegistrationsPoETAddresses) == 0 {
+		return existingRegistrations, nil
+	}
+
+	now := time.Now()
+
+	if curPoetRoundStartDeadline.Before(now) {
+		if len(existingRegistrations) == 0 {
+			if len(registrations) == 0 {
+				// no existing registration at all, drop current registration challenge
+				err = ErrATXChallengeExpired
+			} else {
+				// no existing registration for given poets set
+				err = ErrNoRegistrationForGivenPoetFound
+				nb.logger.Warn("revert poet configuration to previous is needed immediately",
+					zap.Error(err), log.ZShortStringer("smesherID", nodeID))
+			}
+			return nil, fmt.Errorf(
+				"%w: poet round has already started at %s (now: %s)",
+				err,
+				curPoetRoundStartDeadline,
+				now,
+			)
+		}
+		return existingRegistrations, nil
+	}
+
+	// send registrations to missing addresses
 	signature := signer.Sign(signing.POET, challenge)
 	prefix := bytes.Join([][]byte{signer.Prefix(), {byte(signing.POET)}}, nil)
-	nodeID := signer.NodeID()
-	g, ctx := errgroup.WithContext(ctx)
-	errChan := make(chan error, len(nb.poetProvers))
+
+	submitCtx, cancel := context.WithDeadline(ctx, curPoetRoundStartDeadline)
+	defer cancel()
+
+	g, ctx := errgroup.WithContext(submitCtx)
+	errChan := make(chan error, len(missingRegistrationsPoETAddresses))
+	submittedRegistrations := make([]nipost.PoETRegistration, 0)
+
 	for _, client := range nb.poetProvers {
-		g.Go(func() error {
-			errChan <- nb.submitPoetChallenge(ctx, nodeID, deadline, client, prefix, challenge, signature)
-			return nil
-		})
+		if _, ok := missingRegistrationsPoETAddresses[client.Address()]; ok {
+			g.Go(func() error {
+				registration, err := nb.submitPoetChallenge(ctx, nodeID, poetProofDeadline, client, prefix, challenge, signature)
+				if err == nil {
+					submittedRegistrations = append(submittedRegistrations, registration)
+				}
+				errChan <- err
+				return nil
+			})
+		}
 	}
 	g.Wait()
 	close(errChan)
@@ -423,18 +476,15 @@
 	}
 	if allInvalid {
 		nb.logger.Warn("all poet submits were too late. ATX challenge expires", log.ZShortStringer("smesherID", nodeID))
-		return ErrATXChallengeExpired
+		return nil, ErrATXChallengeExpired
 	}
-	return nil
-}
 
-func (nb *NIPostBuilder) getPoetService(ctx context.Context, address string) PoetService {
-	for _, service := range nb.poetProvers {
-		if address == service.Address() {
-			return service
-		}
+	existingRegistrations = append(existingRegistrations, submittedRegistrations...)
+	if len(existingRegistrations) == 0 {
+		return nil, &PoetSvcUnstableError{msg: "failed to submit challenge to any PoET", source: ctx.Err()}
 	}
-	return nil
+
+	return existingRegistrations, nil
 }
 
 // membersContainChallenge verifies that the challenge is included in proof's members.
@@ -451,16 +501,12 @@
 	ctx context.Context,
 	nodeID types.NodeID,
 	challenge types.Hash32,
-	publishEpoch types.EpochID,
+	registrations []nipost.PoETRegistration,
 ) (types.PoetProofRef, *types.MerkleProof, error) {
 	type poetProof struct {
 		poet       *types.PoetProof
 		membership *types.MerkleProof
 	}
-	registrations, err := nipost.PoetRegistrations(nb.localDB, nodeID)
-	if err != nil {
-		return types.PoetProofRef{}, nil, fmt.Errorf("getting poet registrations: %w", err)
-	}
 	proofs := make(chan *poetProof, len(registrations))
 
 	var eg errgroup.Group
@@ -471,11 +517,13 @@
 			zap.String("poet_address", r.Address),
 			zap.String("round", r.RoundID),
 		)
-		client := nb.getPoetService(ctx, r.Address)
-		if client == nil {
+
+		client, ok := nb.poetProvers[r.Address]
+		if !ok {
 			logger.Warn("poet client not found")
 			continue
 		}
+
 		round := r.RoundID
 		waitDeadline := proofDeadline(r.RoundEnd, nb.poetCfg.CycleGap)
 		eg.Go(func() error {