Add workaround for PoET Round 3 (#4969)

## Motivation
Closes #4967

## Changes
- For PoET round 3 / publish epoch 4 PoET 111 will be added to the local state to fetch proofs from if the node registered to PoET 110 for that round.

## Test Plan
- Added tests to ensure workaround works for the correct epoch / round

## TODO
<!-- This section should be removed when all items are complete -->
- [x] Explain motivation or link existing issue(s)
- [x] Test changes and document test plan
- [x] Update documentation as needed
- [x] Update [changelog](../CHANGELOG.md) as needed

CHANGELOG.md

@@ -10,6 +10,8 @@ See [RELEASE](./RELEASE.md) for workflow instructions.
 
 ### Features
 
+* [#4969](https://github.com/spacemeshos/go-spacemesh/pull/4969) Nodes will also fetch from PoET 111 for round 3 if they were able to register to PoET 110.
+
 ### Improvements
 
 * [#4965](https://github.com/spacemeshos/go-spacemesh/pull/4965) Updates to PoST:
@@ -28,13 +30,13 @@ See [RELEASE](./RELEASE.md) for workflow instructions.
 
 Replacement for original version of hare. Won't be enabled on mainnet for now.
 Otherwise protocol uses significantly less traffic (atlest x20), and will allow
-to set lower expected latency in the network, eventually reducing layer time. 
+to set lower expected latency in the network, eventually reducing layer time.
 
 ### Improvements
 
 * [#4879](https://github.com/spacemeshos/go-spacemesh/pull/4795) Makes majority calculation weighted for optimistic filtering.
 The network will start using the new algorithm at layer 18_000 (2023-09-14 20:00:00 +0000 UTC)
-* [#4923](https://github.com/spacemeshos/go-spacemesh/pull/4923) Faster ballot eligibility validation. Improves sync speed. 
+* [#4923](https://github.com/spacemeshos/go-spacemesh/pull/4923) Faster ballot eligibility validation. Improves sync speed.
 * [#4934](https://github.com/spacemeshos/go-spacemesh/pull/4934) Ensure state is synced before participating in tortoise consensus.
 * [#4939](https://github.com/spacemeshos/go-spacemesh/pull/4939) Make sure to fetch data from peers that are already connected.
 * [#4936](https://github.com/spacemeshos/go-spacemesh/pull/4936) Use correct hare active set after node was synced. Otherwise applied layer may lag slightly behind the rest.

activation/nipost.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spacemeshos/poet/shared"
 	"github.com/spacemeshos/post/proving"
 	"github.com/spacemeshos/post/verifying"
+	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/spacemeshos/go-spacemesh/activation/metrics"
@@ -47,6 +48,8 @@ const (
 // proving clients, and thus enormously reduce the cost-per-proof for PoET since each additional proof adds only
 // a small number of hash evaluations to the total cost.
 type PoetProvingServiceClient interface {
+	Address() string
+
 	PowParams(ctx context.Context) (*PoetPowParams, error)
 
 	// Submit registers a challenge in the proving service current open round.
@@ -84,7 +87,7 @@ type NIPostBuilder struct {
 	nodeID            types.NodeID
 	dataDir           string
 	postSetupProvider postSetupProvider
-	poetProvers       []PoetProvingServiceClient
+	poetProvers       map[string]PoetProvingServiceClient
 	poetDB            poetDbAPI
 	state             *types.NIPostBuilderState
 	log               log.Log
@@ -105,7 +108,10 @@ func WithNipostValidator(v nipostValidator) NIPostBuilderOption {
 // withPoetClients allows to pass in clients directly (for testing purposes).
 func withPoetClients(clients []PoetProvingServiceClient) NIPostBuilderOption {
 	return func(nb *NIPostBuilder) {
-		nb.poetProvers = clients
+		nb.poetProvers = make(map[string]PoetProvingServiceClient, len(clients))
+		for _, client := range clients {
+			nb.poetProvers[client.Address()] = client
+		}
 	}
 }
 
@@ -127,13 +133,13 @@ func NewNIPostBuilder(
 	layerClock layerClock,
 	opts ...NIPostBuilderOption,
 ) (*NIPostBuilder, error) {
-	poetClients := make([]PoetProvingServiceClient, 0, len(poetServers))
+	poetClients := make(map[string]PoetProvingServiceClient, len(poetServers))
 	for _, address := range poetServers {
 		client, err := NewHTTPPoetClient(address, poetCfg)
 		if err != nil {
 			return nil, fmt.Errorf("cannot create poet client: %w", err)
 		}
-		poetClients = append(poetClients, client)
+		poetClients[client.Address()] = client
 	}
 
 	b := &NIPostBuilder{
@@ -168,7 +174,10 @@ func (nb *NIPostBuilder) UpdatePoETProvers(poetProvers []PoetProvingServiceClien
 	nb.state = &types.NIPostBuilderState{
 		NIPost: &types.NIPost{},
 	}
-	nb.poetProvers = poetProvers
+	nb.poetProvers = make(map[string]PoetProvingServiceClient, len(poetProvers))
+	for _, poetProver := range poetProvers {
+		nb.poetProvers[poetProver.Address()] = poetProver
+	}
 	nb.log.With().Info("updated poet proof service clients", log.Int("count", len(nb.poetProvers)))
 }
 
@@ -249,7 +258,7 @@ func (nb *NIPostBuilder) BuildNIPost(ctx context.Context, challenge *types.NIPos
 		}
 
 		events.EmitPoetWaitProof(challenge.PublishEpoch, challenge.TargetEpoch(), time.Until(poetRoundEnd))
-		poetProofRef, membership, err := nb.getBestProof(ctx, nb.state.Challenge)
+		poetProofRef, membership, err := nb.getBestProof(ctx, nb.state.Challenge, challenge.PublishEpoch)
 		if err != nil {
 			return nil, &PoetSvcUnstableError{msg: "getBestProof failed", source: err}
 		}
@@ -397,7 +406,59 @@ func membersContainChallenge(members []types.Member, challenge types.Hash32) (ui
 	return 0, fmt.Errorf("challenge is not a member of the proof")
 }
 
-func (nb *NIPostBuilder) getBestProof(ctx context.Context, challenge types.Hash32) (types.PoetProofRef, *types.MerkleProof, error) {
+// TODO(mafa): remove after epoch 4 ends; https://github.com/spacemeshos/go-spacemesh/issues/4968
+func (nb *NIPostBuilder) addPoet111ForPubEpoch4(ctx context.Context) error {
+	// because PoET 111 had a hardware issue when challenges for round 3 were submitted, no node could submit to it
+	// 111 was recovered with the PoET 110 DB, so all successful submissions to 110 should be able to be fetched from there as well
+
+	client111, ok := nb.poetProvers["https://poet-111.spacemesh.network"]
+	if !ok {
+		// poet 111 is not in the list, no action necessary
+		return nil
+	}
+
+	nb.log.Info("pub epoch 4 mitigation: PoET 111 is in the list of PoETs, adding it to the state as well")
+	client110 := nb.poetProvers["https://poet-110.spacemesh.network"]
+
+	ID110, err := client110.PoetServiceID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get PoET 110 id: %w", err)
+	}
+
+	ID111, err := client111.PoetServiceID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get PoET 111 id: %w", err)
+	}
+
+	if slices.IndexFunc(nb.state.PoetRequests, func(r types.PoetRequest) bool { return bytes.Equal(r.PoetServiceID.ServiceID, ID111.ServiceID) }) != -1 {
+		nb.log.Info("PoET 111 is already in the state, no action necessary")
+		return nil
+	}
+
+	poet110 := slices.IndexFunc(nb.state.PoetRequests, func(r types.PoetRequest) bool {
+		return bytes.Equal(r.PoetServiceID.ServiceID, ID110.ServiceID)
+	})
+	if poet110 == -1 {
+		return fmt.Errorf("poet 110 is not in the state, cannot add poet 111")
+	}
+
+	poet111 := nb.state.PoetRequests[poet110]
+	poet111.PoetServiceID.ServiceID = ID111.ServiceID
+	nb.state.PoetRequests = append(nb.state.PoetRequests, poet111)
+	nb.persistState()
+	nb.log.Info("pub epoch 4 mitigation: PoET 111 added to the state")
+	return nil
+}
+
+func (nb *NIPostBuilder) getBestProof(ctx context.Context, challenge types.Hash32, publishEpoch types.EpochID) (types.PoetProofRef, *types.MerkleProof, error) {
+	// TODO(mafa): remove after next PoET round; https://github.com/spacemeshos/go-spacemesh/issues/4968
+	if publishEpoch == 4 {
+		err := nb.addPoet111ForPubEpoch4(ctx)
+		if err != nil {
+			nb.log.With().Error("pub epoch 4 mitigation: failed to add PoET 111 to state for pub epoch 4", log.Err(err))
+		}
+	}
+
 	type poetProof struct {
 		poet       *types.PoetProofMessage
 		membership *types.MerkleProof