Merge branch 'master' into graffitti-append

CHANGELOG.md

@@ -15,8 +15,11 @@ the [releases page](https://github.com/Consensys/teku/releases).
 ### Additions and Improvements
 - Introduced [Validator Slashing Prevention feature](https://docs.teku.consensys.io/how-to/prevent-slashing/detect-slashing).
 - If the EL supports the `engine_getClientVersionV1` Engine API method, the default graffiti (when no graffiti has been configured by the validator) will include EL as well as CL version information. For more details, please see https://github.com/ethereum/execution-apis/pull/517.
-- `—p2p-private-key-file` command line option supports reading a binary private key file.
+- `--p2p-private-key-file` command line option supports reading a binary private key file.
 - Updated libp2p seen cache configuration to reflect EIP-7045 spec changes. This reduces CPU and network bandwidth consumption.
-- Increased the attestation cache capacity to allow teku a bigger pool of attestations when block building.
+- Increased the attestation cache capacity to allow Teku a bigger pool of attestations when block building.
+- Defaulted `--builder-bid-compare-factor` to 90. This makes it necessary for external block builders to give at least 10% additional profit compared to a local build before being taken into consideration. If you would like to go back to the previous default, set `--builder-bid-compare-factor` to 100.
+- Added `--p2p-direct-peers` command line option to configure explicit peers as per [Explicit Peering Agreements](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#explicit-peering-agreements) libp2p spec.
 
 ### Bug Fixes
+- Fix incompatibility between Teku validator client and Lighthouse beacon nodes [#8117](https://github.com/Consensys/teku/pull/8117)

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java

@@ -707,12 +707,7 @@ public void waitForAttestationBeingGossiped(
 
           Set<UInt64> attesterIndicesInAttestations =
               block.getMessage().getBody().attestations.stream()
-                  .map(
-                      a ->
-                          spec.getAttestingIndices(
-                              state,
-                              a.asInternalAttestation(spec).getData(),
-                              a.asInternalAttestation(spec).getAggregationBits()))
+                  .map(a -> spec.getAttestingIndices(state, a.asInternalAttestation(spec)))
                   .flatMap(Collection::stream)
                   .map(UInt64::valueOf)
                   .collect(toSet());

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactory.java

@@ -13,7 +13,8 @@
 
 package tech.pegasys.teku.validator.coordinator;
 
-import com.google.common.base.Preconditions;
+import static com.google.common.base.Preconditions.checkState;
+
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
@@ -117,9 +118,7 @@ public Function<BeaconBlockBodyBuilder, SafeFuture<Void>> createSelector(
 
       final SszList<Attestation> attestations =
           attestationPool.getAttestationsForBlock(
-              blockSlotState,
-              new AttestationForkChecker(spec, blockSlotState),
-              spec.createAttestationWorthinessChecker(blockSlotState));
+              blockSlotState, new AttestationForkChecker(spec, blockSlotState));
 
       // Collect slashings to include
       final Set<UInt64> exitedValidators = new HashSet<>();
@@ -453,20 +452,28 @@ public Function<SignedBlockContainer, List<BlobSidecar>> createBlobSidecarsSelec
 
       final SszList<Blob> blobs;
       final SszList<SszKZGProof> proofs;
+      final SszList<SszKZGCommitment> blockCommitments =
+          block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow();
 
       if (blockContainer.isBlinded()) {
         // need to use the builder BlobsBundle for the blinded flow, because the
         // blobs and the proofs wouldn't be part of the BlockContainer
         final tech.pegasys.teku.spec.datastructures.builder.BlobsBundle blobsBundle =
             getCachedBuilderBlobsBundle(slot);
+
+        blobs = blobsBundle.getBlobs();
+        proofs = blobsBundle.getProofs();
+
         // consistency check because the BlobsBundle comes from an external source (a builder)
-        final SszList<SszKZGCommitment> blockCommitments =
-            block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow();
-        Preconditions.checkState(
+        checkState(
             blobsBundle.getCommitments().hashTreeRoot().equals(blockCommitments.hashTreeRoot()),
             "Commitments in the builder BlobsBundle don't match the commitments in the block");
-        blobs = blobsBundle.getBlobs();
-        proofs = blobsBundle.getProofs();
+        checkState(
+            blockCommitments.size() == proofs.size(),
+            "The number of proofs in BlobsBundle doesn't match the number of commitments in the block");
+        checkState(
+            blockCommitments.size() == blobs.size(),
+            "The number of blobs in BlobsBundle doesn't match the number of commitments in the block");
       } else {
         blobs = blockContainer.getBlobs().orElseThrow();
         proofs = blockContainer.getKzgProofs().orElseThrow();

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/DepositProvider.java

@@ -74,8 +74,8 @@ public class DepositProvider
   private boolean inSync = false;
 
   public DepositProvider(
-      MetricsSystem metricsSystem,
-      RecentChainData recentChainData,
+      final MetricsSystem metricsSystem,
+      final RecentChainData recentChainData,
       final Eth1DataCache eth1DataCache,
       final StorageUpdateChannel storageUpdateChannel,
       final Eth1DepositStorageChannel eth1DepositStorageChannel,
@@ -99,7 +99,7 @@ public DepositProvider(
   }
 
   @Override
-  public synchronized void onDepositsFromBlock(DepositsFromBlockEvent event) {
+  public synchronized void onDepositsFromBlock(final DepositsFromBlockEvent event) {
     event.getDeposits().stream()
         .map(depositUtil::convertDepositEventToOperationDeposit)
         .forEach(
@@ -171,7 +171,7 @@ public void onEth1Block(
   }
 
   @Override
-  public void onMinGenesisTimeBlock(MinGenesisTimeBlockEvent event) {}
+  public void onMinGenesisTimeBlock(final MinGenesisTimeBlockEvent event) {}
 
   @Override
   public void onSlot(final UInt64 slot) {
@@ -201,33 +201,54 @@ public void onSlot(final UInt64 slot) {
         .ifExceptionGetsHereRaiseABug();
   }
 
-  public void onSyncingStatusChanged(boolean inSync) {
+  public void onSyncingStatusChanged(final boolean inSync) {
     this.inSync = inSync;
   }
 
-  public synchronized SszList<Deposit> getDeposits(BeaconState state, Eth1Data eth1Data) {
+  public synchronized SszList<Deposit> getDeposits(
+      final BeaconState state, final Eth1Data eth1Data) {
+    final long maxDeposits = spec.getMaxDeposits(state);
+    final SszListSchema<Deposit, ?> depositsSchema = depositsSchemaCache.get(maxDeposits);
+    // no Eth1 deposits needed if already transitioned to the EIP-6110 mechanism
+    if (spec.isFormerDepositMechanismDisabled(state)) {
+      return depositsSchema.createFromElements(emptyList());
+    }
     final UInt64 eth1DepositCount;
     if (spec.isEnoughVotesToUpdateEth1Data(state, eth1Data, 1)) {
       eth1DepositCount = eth1Data.getDepositCount();
     } else {
       eth1DepositCount = state.getEth1Data().getDepositCount();
     }
-
     final UInt64 eth1DepositIndex = state.getEth1DepositIndex();
 
+    final UInt64 eth1PendingDepositCount =
+        state
+            .toVersionElectra()
+            .map(
+                stateElectra -> {
+                  // EIP-6110
+                  final UInt64 eth1DepositIndexLimit =
+                      eth1DepositCount.min(stateElectra.getDepositReceiptsStartIndex());
+                  return eth1DepositIndexLimit.minusMinZero(eth1DepositIndex).min(maxDeposits);
+                })
+            .orElseGet(
+                () -> {
+                  // Phase0
+                  return eth1DepositCount.minusMinZero(eth1DepositIndex).min(maxDeposits);
+                });
+
+    // No deposits to include
+    if (eth1PendingDepositCount.isZero()) {
+      return depositsSchema.createFromElements(emptyList());
+    }
+
     // We need to have all the deposits that can be included in the state available to ensure
     // the generated proofs are valid
     checkRequiredDepositsAvailable(eth1DepositCount, eth1DepositIndex);
 
-    final long maxDeposits = spec.getMaxDeposits(state);
-    final UInt64 latestDepositIndexWithMaxBlock = eth1DepositIndex.plus(spec.getMaxDeposits(state));
-
-    final UInt64 toDepositIndex =
-        latestDepositIndexWithMaxBlock.isGreaterThan(eth1DepositCount)
-            ? eth1DepositCount
-            : latestDepositIndexWithMaxBlock;
+    final UInt64 toDepositIndex = eth1DepositIndex.plus(eth1PendingDepositCount);
 
-    return getDepositsWithProof(eth1DepositIndex, toDepositIndex, eth1DepositCount, maxDeposits);
+    return getDepositsWithProof(eth1DepositIndex, toDepositIndex, eth1DepositCount, depositsSchema);
   }
 
   protected synchronized List<DepositWithIndex> getAvailableDeposits() {
@@ -261,14 +282,12 @@ public synchronized int getDepositMapSize() {
    * @param eth1DepositCount number of deposits in the merkle tree according to Eth1Data in state
    */
   private SszList<Deposit> getDepositsWithProof(
-      UInt64 fromDepositIndex, UInt64 toDepositIndex, UInt64 eth1DepositCount, long maxDeposits) {
+      final UInt64 fromDepositIndex,
+      final UInt64 toDepositIndex,
+      final UInt64 eth1DepositCount,
+      final SszListSchema<Deposit, ?> depositsSchema) {
     final AtomicReference<UInt64> expectedDepositIndex = new AtomicReference<>(fromDepositIndex);
-    final SszListSchema<Deposit, ?> depositsSchema = depositsSchemaCache.get(maxDeposits);
     final SszBytes32VectorSchema<?> depositProofSchema = Deposit.SSZ_SCHEMA.getProofSchema();
-    // No deposits to include so don't bother rewinding the merkle tree.
-    if (fromDepositIndex.equals(toDepositIndex)) {
-      return depositsSchema.createFromElements(emptyList());
-    }
     if (depositMerkleTree.getDepositCount() < eth1DepositCount.intValue()) {
       throw MissingDepositsException.missingRange(
           UInt64.valueOf(depositMerkleTree.getDepositCount()), eth1DepositCount);

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/AbstractBlockFactoryTest.java

@@ -185,7 +185,7 @@ protected BlockContainerAndMetaData assertBlockCreated(
     final BlockFactory blockFactory = createBlockFactory(spec);
 
     when(depositProvider.getDeposits(any(), any())).thenReturn(deposits);
-    when(attestationsPool.getAttestationsForBlock(any(), any(), any())).thenReturn(attestations);
+    when(attestationsPool.getAttestationsForBlock(any(), any())).thenReturn(attestations);
     when(attesterSlashingPool.getItemsForBlock(any(), any(), any())).thenReturn(attesterSlashings);
     when(proposerSlashingPool.getItemsForBlock(any(), any(), any())).thenReturn(proposerSlashings);
     when(voluntaryExitPool.getItemsForBlock(any(), any(), any())).thenReturn(voluntaryExits);

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java

@@ -17,6 +17,7 @@
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.assertThatSafeFuture;
 import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.safeJoin;
@@ -211,7 +212,7 @@ class BlockOperationSelectorFactoryTest {
 
   @BeforeEach
   void setUp() {
-    when(attestationPool.getAttestationsForBlock(any(), any(), any()))
+    when(attestationPool.getAttestationsForBlock(any(), any()))
         .thenReturn(
             beaconBlockSchemaSupplier.apply(UInt64.ZERO).getAttestationsSchema().getDefault());
     when(contributionValidator.validate(any())).thenReturn(SafeFuture.completedFuture(ACCEPT));
@@ -819,7 +820,7 @@ void shouldCreateBlobSidecarsForBlockContents() {
   }
 
   @Test
-  void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleIsNotConsistent() {
+  void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleCommitmentsRootIsNotConsistent() {
     final SszList<SszKZGCommitment> commitments = dataStructureUtil.randomBlobKzgCommitments(3);
     final SignedBeaconBlock signedBlindedBeaconBlock =
         dataStructureUtil.randomSignedBlindedBeaconBlockWithCommitments(commitments);
@@ -838,6 +839,48 @@ void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleIsNotConsistent() {
             "Commitments in the builder BlobsBundle don't match the commitments in the block");
   }
 
+  @Test
+  void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleProofsIsNotConsistent() {
+    final SszList<SszKZGCommitment> commitments = dataStructureUtil.randomBlobKzgCommitments(3);
+    final SignedBeaconBlock signedBlindedBeaconBlock =
+        dataStructureUtil.randomSignedBlindedBeaconBlockWithCommitments(commitments);
+
+    final tech.pegasys.teku.spec.datastructures.builder.BlobsBundle blobsBundle =
+        spy(dataStructureUtil.randomBuilderBlobsBundle(commitments));
+    when(blobsBundle.getBlobs()).thenReturn(dataStructureUtil.randomSszBlobs(2));
+
+    prepareCachedBuilderPayload(
+        signedBlindedBeaconBlock.getSlot(),
+        dataStructureUtil.randomExecutionPayload(),
+        Optional.of(blobsBundle));
+
+    assertThatThrownBy(() -> factory.createBlobSidecarsSelector().apply(signedBlindedBeaconBlock))
+        .isInstanceOf(IllegalStateException.class)
+        .hasMessage(
+            "The number of blobs in BlobsBundle doesn't match the number of commitments in the block");
+  }
+
+  @Test
+  void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleBlobsIsNotConsistent() {
+    final SszList<SszKZGCommitment> commitments = dataStructureUtil.randomBlobKzgCommitments(3);
+    final SignedBeaconBlock signedBlindedBeaconBlock =
+        dataStructureUtil.randomSignedBlindedBeaconBlockWithCommitments(commitments);
+
+    final tech.pegasys.teku.spec.datastructures.builder.BlobsBundle blobsBundle =
+        spy(dataStructureUtil.randomBuilderBlobsBundle(commitments));
+    when(blobsBundle.getProofs()).thenReturn(dataStructureUtil.randomSszKZGProofs(2));
+
+    prepareCachedBuilderPayload(
+        signedBlindedBeaconBlock.getSlot(),
+        dataStructureUtil.randomExecutionPayload(),
+        Optional.of(blobsBundle));
+
+    assertThatThrownBy(() -> factory.createBlobSidecarsSelector().apply(signedBlindedBeaconBlock))
+        .isInstanceOf(IllegalStateException.class)
+        .hasMessage(
+            "The number of proofs in BlobsBundle doesn't match the number of commitments in the block");
+  }
+
   @Test
   void shouldCreateBlobSidecarsForBlindedBlock() {
     final SszList<SszKZGCommitment> commitments = dataStructureUtil.randomBlobKzgCommitments(3);