feat: PeerDAS

deploy/local/docker-compose/vector-kafka-clickhouse-libp2p.yaml

@@ -1911,3 +1911,23 @@ sinks:
  healthcheck:
  enabled: true
  skip_unknown_fields: false
+ libp2p_trace_gossipsub_beacon_data_column_sidecar_clickhouse:
+ type: clickhouse
+ inputs:
+ - libp2p_trace_gossipsub_beacon_data_column_sidecar_formatted
+ database: default
+ endpoint: "${CLICKHOUSE_ENDPOINT}"
+ table: libp2p_gossipsub_beacon_data_column_sidecar
+ auth:
+ strategy: basic
+ user: "${CLICKHOUSE_USER}"
+ password: "${CLICKHOUSE_PASSWORD}"
+ batch:
+ max_bytes: 52428800
+ max_events: 200000
+ timeout_secs: 1
+ buffer:
+ max_events: 200000
+ healthcheck:
+ enabled: true
+ skip_unknown_fields: false

go.mod

@@ -2,7 +2,7 @@ module github.com/ethpandaops/xatu
 
 go 1.22.0
 
-replace github.com/probe-lab/hermes => github.com/ethpandaops/hermes v0.0.0-20240527060557-ae23464cdc41
+replace github.com/probe-lab/hermes => github.com/ethpandaops/hermes v0.0.0-20240725023104-113a74f8b721
 
 require (
  github.com/IBM/sarama v1.43.0

go.sum

@@ -266,8 +266,8 @@ github.com/ethpandaops/ethcore v0.0.0-20240422023000-2a5727b18756 h1:8JWjrRfP14m
 github.com/ethpandaops/ethcore v0.0.0-20240422023000-2a5727b18756/go.mod h1:ZvKqL6CKxiraefdXPHeJurV2pDD/f2HF2uklDVdrry8=
 github.com/ethpandaops/ethwallclock v0.3.0 h1:xF5fwtBf+bHFHZKBnwiPFEuelW3sMM7SD3ZNFq1lJY4=
 github.com/ethpandaops/ethwallclock v0.3.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24=
-github.com/ethpandaops/hermes v0.0.0-20240527060557-ae23464cdc41 h1:KM07rlGX9EQm6ZawvL0BDMJ5Rf3brZIJKsmiFS92wI0=
-github.com/ethpandaops/hermes v0.0.0-20240527060557-ae23464cdc41/go.mod h1:uMPOUopuxLk1Qktn66udTjRBLqJJ+CNfiCyxNQda4Ow=
+github.com/ethpandaops/hermes v0.0.0-20240725023104-113a74f8b721 h1:8fwmolf051iMka7QkuiZ7a97wTwTEshNA0RoaLKOy7o=
+github.com/ethpandaops/hermes v0.0.0-20240725023104-113a74f8b721/go.mod h1:IZgi+dl4W0aDbxy0IB/hTfJtiDDEe/AkkDzvt/rgvZ8=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=

pkg/clmimicry/ethereum/config.go

@@ -19,6 +19,7 @@ type CustomNetwork struct {
  ConfigURL string `yaml:"configUrl"`
  BootnodeENRURL string `yaml:"bootnodeEnrUrl"`
  DepositContractBlockURL string `yaml:"depositContractBlockUrl"`
+ GenesisSSZURL string `yaml:"genesisSSZUrl"`
 }
 
 func (c *CustomNetwork) Validate() error {
@@ -34,5 +35,9 @@ func (c *CustomNetwork) Validate() error {
  return errors.New("depositContractBlockUrl is required")
  }
 
+ if c.GenesisSSZURL == "" {
+ return errors.New("genesisSSZUrl is required")
+ }
+
  return nil
 }

pkg/clmimicry/mimicry.go

@@ -99,68 +99,43 @@ func New(ctx context.Context, log logrus.FieldLogger, config *Config) (*Mimicry,
  return mimicry, nil
 }
 
-func (m *Mimicry) deriveNetworkConfig(ctx context.Context) (*eth.GenesisConfig, *params.NetworkConfig, *params.BeaconChainConfig, error) {
+func (m *Mimicry) deriveNetworkConfig(ctx context.Context) (*eth.NetworkConfig, error) {
  if m.Config.Ethereum.CustomNetwork == nil {
- return eth.GetConfigsByNetworkName(m.Config.Ethereum.Network)
+ return eth.DeriveKnownNetworkConfig(ctx, m.Config.Ethereum.Network)
  }
 
  m.log.WithField("network", m.Config.Ethereum.Network).Info("Using a custom Ethereum network")
 
- genesis := m.ethereum.Metadata().Genesis
-
- genesisConfig := &eth.GenesisConfig{
- GenesisValidatorRoot: genesis.GenesisValidatorsRoot[:],
- GenesisTime: genesis.GenesisTime,
- }
-
- beaconChainConfig, err := m.FetchConfigFromURL(ctx, m.Config.Ethereum.CustomNetwork.ConfigURL)
- if err != nil {
- return nil, nil, nil, fmt.Errorf("failed to fetch custom Ethereum network config.yaml: %w", err)
- }
-
- if err := params.SetActive(beaconChainConfig); err != nil {
- return nil, nil, nil, fmt.Errorf("failed to set active network config: %w", err)
- }
-
- params.OverrideBeaconConfig(beaconChainConfig)
-
- networkConfig := params.BeaconNetworkConfig() // TODO: Support others
-
- bootnodes, err := FetchBootnodeENRsFromURL(ctx, m.Config.Ethereum.CustomNetwork.BootnodeENRURL)
- if err != nil {
- return nil, nil, nil, fmt.Errorf("failed to fetch custom Ethereum network bootnodes.yaml: %w", err)
- }
-
- depositContractBlock, err := FetchDepositContractBlockFromURL(ctx, m.Config.Ethereum.CustomNetwork.DepositContractBlockURL)
+ conf, err := eth.DeriveDevnetConfig(ctx, eth.DevnetOptions{
+ ConfigURL: m.Config.Ethereum.CustomNetwork.ConfigURL,
+ BootnodesURL: m.Config.Ethereum.CustomNetwork.BootnodeENRURL,
+ DepositContractBlockURL: m.Config.Ethereum.CustomNetwork.DepositContractBlockURL,
+ GenesisSSZURL: m.Config.Ethereum.CustomNetwork.GenesisSSZURL,
+ })
  if err != nil {
- return nil, nil, nil, fmt.Errorf("failed to fetch custom Ethereum network deposit contract block.yaml: %w", err)
+ return nil, fmt.Errorf("failed to derive devnet config: %w", err)
  }
 
- networkConfig.BootstrapNodes = bootnodes
- networkConfig.ContractDeploymentBlock = depositContractBlock
-
- params.OverrideBeaconNetworkConfig(networkConfig)
-
- return genesisConfig, networkConfig, beaconChainConfig, nil
+ return conf, nil
 }
 
 func (m *Mimicry) startHermes(ctx context.Context) error {
- genConfig, netConfig, beaConfig, err := m.deriveNetworkConfig(ctx)
+ networkConfig, err := m.deriveNetworkConfig(ctx)
  if err != nil {
  return fmt.Errorf("failed to derive network config for %s: %w", m.Config.Ethereum.Network, err)
  }
 
- m.networkConfig = netConfig
- m.beaconConfig = beaConfig
+ m.networkConfig = networkConfig.Network
+ m.beaconConfig = networkConfig.Beacon
 
- genesisRoot := genConfig.GenesisValidatorRoot
- genesisTime := genConfig.GenesisTime
+ genesisRoot := networkConfig.Genesis.GenesisValidatorRoot
+ genesisTime := networkConfig.Genesis.GenesisTime
 
  // compute fork version and fork digest
  currentSlot := slots.Since(genesisTime)
  currentEpoch := slots.ToEpoch(currentSlot)
 
- currentForkVersion, err := eth.GetCurrentForkVersion(currentEpoch, beaConfig)
+ currentForkVersion, err := eth.GetCurrentForkVersion(currentEpoch, m.beaconConfig)
  if err != nil {
  return fmt.Errorf("compute fork version for epoch %d: %w", currentEpoch, err)
  }
@@ -172,14 +147,14 @@ func (m *Mimicry) startHermes(ctx context.Context) error {
 
  // Overriding configuration so that functions like ComputForkDigest take the
  // correct input data from the global configuration.
- params.OverrideBeaconConfig(beaConfig)
- params.OverrideBeaconNetworkConfig(netConfig)
+ params.OverrideBeaconConfig(m.beaconConfig)
+ params.OverrideBeaconNetworkConfig(m.networkConfig)
 
  nodeConfig := m.Config.Node.AsHermesConfig()
 
- nodeConfig.GenesisConfig = genConfig
- nodeConfig.NetworkConfig = netConfig
- nodeConfig.BeaconConfig = beaConfig
+ nodeConfig.GenesisConfig = networkConfig.Genesis
+ nodeConfig.NetworkConfig = m.networkConfig
+ nodeConfig.BeaconConfig = m.beaconConfig
  nodeConfig.ForkDigest = forkDigest
  nodeConfig.ForkVersion = currentForkVersion
  nodeConfig.PubSubSubscriptionRequestLimit = 200