Refactored externalSigner to use Typedefs (#8541)

 - created a new suite of tests to show a number of different signing bodies.
 - had to do manual testing for external signer becuase no AT i think
 - I was able to get from phase0 -> bellatrix so this should cover most scenarios.
 - I believe most of the wrappers should be available in the api

partially addresses #8527

Signed-off-by: Paul Harris <paul.harris@consensys.net>

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/CapellaRemoteSignerAcceptanceTest.java

@@ -39,10 +39,10 @@ public class CapellaRemoteSignerAcceptanceTest extends AcceptanceTestBase {
   private static final URL JWT_FILE = Resources.getResource("auth/ee-jwt-secret.hex");
 
   @Test
-  void denebWithRemoteSigner() throws Exception {
+  void capellaWithRemoteSigner() throws Exception {
     final UInt64 currentTime = new SystemTimeProvider().getTimeInSeconds();
     final int genesisTime =
-        currentTime.intValue() + 10; // genesis in 10 seconds to give node time to start
+        currentTime.intValue() + 30; // genesis needs added time for nodes to startup
 
     final Web3SignerNode web3SignerNode =
         createWeb3SignerNode(

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/builder/ValidatorRegistration.java

@@ -24,6 +24,7 @@
 
 public class ValidatorRegistration
     extends Container4<ValidatorRegistration, SszByteVector, SszUInt64, SszUInt64, SszPublicKey> {
+  public static final ValidatorRegistrationSchema SSZ_SCHEMA = new ValidatorRegistrationSchema();
 
   protected ValidatorRegistration(
       final ValidatorRegistrationSchema schema, final TreeNode backingNode) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/ForkInfo.java

@@ -13,10 +13,13 @@
 
 package tech.pegasys.teku.spec.datastructures.state;
 
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.BYTES32_TYPE;
+
 import com.google.common.base.MoreObjects;
 import java.util.Objects;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.infrastructure.bytes.Bytes4;
+import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
 import tech.pegasys.teku.spec.Spec;
 
 public class ForkInfo {
@@ -63,6 +66,13 @@ public boolean equals(final Object o) {
         && Objects.equals(genesisValidatorsRoot, forkInfo.genesisValidatorsRoot);
   }
 
+  public static SerializableTypeDefinition<ForkInfo> getJsonTypeDefinition() {
+    return SerializableTypeDefinition.object(ForkInfo.class)
+        .withField("fork", Fork.SSZ_SCHEMA.getJsonTypeDefinition(), ForkInfo::getFork)
+        .withField("genesis_validators_root", BYTES32_TYPE, ForkInfo::getGenesisValidatorsRoot)
+        .build();
+  }
+
   @Override
   public int hashCode() {
     return Objects.hash(fork, genesisValidatorsRoot);

interop-keys/import_keys.sh

@@ -20,9 +20,11 @@ done
 TEMP=`mktemp -d`
 
 function cleanup() {
+    echo "Cleaning up temp folder ${TEMP}."
     rm -rf "${TEMP}"
 }
 trap cleanup EXIT
+echo "TEMP: ${TEMP}"
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
@@ -67,12 +69,20 @@ done
 echo "]
 }" >> "${TEMP}/payload.json"
 
+# annoying but AUTHORIZATION_HEADER below seems to be a problem if its empty, so coding with and without
 echo "Sending payload.json to ${SIGNER_URL}/eth/v1/keystores..."
-
-curl --fail -q -X POST ${SIGNER_URL}/eth/v1/keystores \
-     "${AUTHORIZATION_HEADER:-}" \
-     -H "Content-Type: application/json" \
-     -d "@${TEMP}/payload.json" \
-     -o "${TEMP}/result.json"
+if [ ! -z "${AUTHORIZATION_HEADER:-}" ]
+then
+    curl --fail -q -X POST ${SIGNER_URL}/eth/v1/keystores \
+         "${AUTHORIZATION_HEADER:-}" \
+         -H "Content-Type: application/json" \
+         -d "@${TEMP}/payload.json" \
+         -o "${TEMP}/result.json"
+else
+    curl --fail -q -X POST ${SIGNER_URL}/eth/v1/keystores \
+         -H "Content-Type: application/json" \
+         -d "@${TEMP}/payload.json" \
+         -o "${TEMP}/result.json"
+fi
 
  echo "Wrote result to ${TEMP}/result.json."

validator/api/src/main/java/tech/pegasys/teku/validator/api/signer/AggregationSlotWrapper.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.api.signer;
+
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.UINT64_TYPE;
+
+import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+
+public record AggregationSlotWrapper(UInt64 slot) {
+  public static DeserializableTypeDefinition<AggregationSlotWrapper> getJsonTypeDefinition() {
+    return DeserializableTypeDefinition.object(AggregationSlotWrapper.class, Builder.class)
+        .initializer(Builder::new)
+        .finisher(Builder::build)
+        .withField("slot", UINT64_TYPE, AggregationSlotWrapper::slot, Builder::slot)
+        .build();
+  }
+
+  static class Builder {
+    private UInt64 slot;
+
+    Builder slot(final UInt64 slot) {
+      this.slot = slot;
+      return this;
+    }
+
+    AggregationSlotWrapper build() {
+      return new AggregationSlotWrapper(slot);
+    }
+  }
+}

validator/api/src/main/java/tech/pegasys/teku/validator/api/signer/BlockWrapper.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.api.signer;
+
+import static tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition.enumOf;
+
+import java.util.Optional;
+import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
+import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader;
+
+public record BlockWrapper(
+    SpecMilestone milestone, Optional<BeaconBlock> block, Optional<BeaconBlockHeader> blockHeader) {
+  public SerializableTypeDefinition<BlockWrapper> getJsonTypeDefinition() {
+    return SerializableTypeDefinition.object(BlockWrapper.class)
+        .withField("version", enumOf(SpecMilestone.class), BlockWrapper::milestone)
+        .withOptionalField(SignType.BLOCK.getName(), getMaybeBlockSchema(), BlockWrapper::block)
+        .withOptionalField("block_header", getMaybeBlockHeaderSchema(), BlockWrapper::blockHeader)
+        .build();
+  }
+
+  private SerializableTypeDefinition<BeaconBlock> getMaybeBlockSchema() {
+    return block
+        .<SerializableTypeDefinition<BeaconBlock>>map(
+            beaconBlock -> beaconBlock.getSchema().getJsonTypeDefinition())
+        .orElse(null);
+  }
+
+  private SerializableTypeDefinition<BeaconBlockHeader> getMaybeBlockHeaderSchema() {
+    return blockHeader
+        .<SerializableTypeDefinition<BeaconBlockHeader>>map(
+            beaconBlockHeader -> beaconBlockHeader.getSchema().getJsonTypeDefinition())
+        .orElse(null);
+  }
+}

validator/api/src/main/java/tech/pegasys/teku/validator/api/signer/RandaoRevealWrapper.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.api.signer;
+
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.UINT64_TYPE;
+
+import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+
+public record RandaoRevealWrapper(UInt64 epoch) {
+
+  public static DeserializableTypeDefinition<RandaoRevealWrapper> getJsonTypeDefinition() {
+    return DeserializableTypeDefinition.object(RandaoRevealWrapper.class, Builder.class)
+        .initializer(Builder::new)
+        .finisher(Builder::build)
+        .withField("epoch", UINT64_TYPE, RandaoRevealWrapper::epoch, Builder::epoch)
+        .build();
+  }
+
+  static class Builder {
+    private UInt64 epoch;
+
+    public Builder epoch(final UInt64 epoch) {
+      this.epoch = epoch;
+      return this;
+    }
+
+    public RandaoRevealWrapper build() {
+      return new RandaoRevealWrapper(epoch);
+    }
+  }
+}

validator/api/src/main/java/tech/pegasys/teku/validator/api/signer/SignType.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Consensys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.api.signer;
+
+public enum SignType {
+  RANDAO_REVEAL("randao_reveal"),
+  BLOCK("block"),
+  BLOCK_V2("block_v2"),
+  ATTESTATION("attestation"),
+  AGGREGATION_SLOT("aggregation_slot"),
+  AGGREGATE_AND_PROOF("aggregate_and_proof"),
+  VOLUNTARY_EXIT("voluntary_exit"),
+  SYNC_COMMITTEE_MESSAGE("sync_committee_message"),
+  SYNC_AGGREGATOR_SELECTION_DATA("sync_aggregator_selection_data"),
+  SYNC_COMMITTEE_SELECTION_PROOF("sync_committee_selection_proof"),
+  SYNC_COMMITTEE_CONTRIBUTION_AND_PROOF("sync_committee_contribution_and_proof"),
+  VALIDATOR_REGISTRATION("validator_registration"),
+  CONTRIBUTION_AND_PROOF("contribution_and_proof"),
+  BEACON_BLOCK("beacon_block"),
+  BLOB_SIDECAR("blob_sidecar");
+
+  private final String name;
+
+  SignType(final String name) {
+    this.name = name;
+  }
+
+  public String getName() {
+    return name;
+  }
+}

validator/api/src/main/java/tech/pegasys/teku/validator/api/signer/SyncAggregatorSelectionDataWrapper.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.api.signer;
+
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.UINT64_TYPE;
+
+import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+
+public record SyncAggregatorSelectionDataWrapper(UInt64 slot, UInt64 subcommitteeIndex) {
+  public static DeserializableTypeDefinition<SyncAggregatorSelectionDataWrapper>
+      getJsonTypefinition() {
+    return DeserializableTypeDefinition.object(
+            SyncAggregatorSelectionDataWrapper.class, Builder.class)
+        .initializer(Builder::new)
+        .finisher(Builder::build)
+        .withField("slot", UINT64_TYPE, SyncAggregatorSelectionDataWrapper::slot, Builder::slot)
+        .withField(
+            "subcommittee_index",
+            UINT64_TYPE,
+            SyncAggregatorSelectionDataWrapper::subcommitteeIndex,
+            Builder::subcommitteeIndex)
+        .build();
+  }
+
+  static class Builder {
+    private UInt64 slot;
+    private UInt64 subcommitteeIndex;
+
+    Builder slot(final UInt64 slot) {
+      this.slot = slot;
+      return this;
+    }
+
+    Builder subcommitteeIndex(final UInt64 subcommitteeIndex) {
+      this.subcommitteeIndex = subcommitteeIndex;
+      return this;
+    }
+
+    SyncAggregatorSelectionDataWrapper build() {
+      return new SyncAggregatorSelectionDataWrapper(slot, subcommitteeIndex);
+    }
+  }
+}

validator/api/src/main/java/tech/pegasys/teku/validator/api/signer/SyncCommitteeMessageWrapper.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.api.signer;
+
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.BYTES32_TYPE;
+import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.UINT64_TYPE;
+
+import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+
+public record SyncCommitteeMessageWrapper(Bytes32 blockRoot, UInt64 slot) {
+
+  public static DeserializableTypeDefinition<SyncCommitteeMessageWrapper> getJsonTypeDefinition() {
+    return DeserializableTypeDefinition.object(SyncCommitteeMessageWrapper.class, Builder.class)
+        .initializer(Builder::new)
+        .finisher(Builder::build)
+        .withField("slot", UINT64_TYPE, SyncCommitteeMessageWrapper::slot, Builder::slot)
+        .withField(
+            "beacon_block_root",
+            BYTES32_TYPE,
+            SyncCommitteeMessageWrapper::blockRoot,
+            Builder::blockRoot)
+        .build();
+  }
+
+  static class Builder {
+    private Bytes32 blockRoot;
+    private UInt64 slot;
+
+    Builder blockRoot(final Bytes32 blockRoot) {
+      this.blockRoot = blockRoot;
+      return this;
+    }
+
+    Builder slot(final UInt64 slot) {
+      this.slot = slot;
+      return this;
+    }
+
+    SyncCommitteeMessageWrapper build() {
+      return new SyncCommitteeMessageWrapper(blockRoot, slot);
+    }
+  }
+}