From a5dcabf3f2c40a47f2f234f162999e2ccf558f33 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 22 Aug 2024 21:02:09 +0200 Subject: [PATCH 1/6] fix `ChainService.GetDbBlocksForSlots` & `ChainService.GetDbBlocksByFilter` to show pruned duties properly --- db/slots.go | 70 +++-- handlers/index.go | 11 +- handlers/slots_filtered.go | 7 +- handlers/validator.go | 2 +- handlers/validator_slots.go | 2 +- indexer/beacon/block.go | 11 - services/chainservice_blocks.go | 498 ++++++++++++++++---------------- 7 files changed, 306 insertions(+), 295 deletions(-) diff --git a/db/slots.go b/db/slots.go index 6e145731..f0f60f33 100644 --- a/db/slots.go +++ b/db/slots.go @@ -6,6 +6,7 @@ import ( "math" "strings" + "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/ethpandaops/dora/dbtypes" "github.com/jmoiron/sqlx" "github.com/mitchellh/mapstructure" @@ -83,38 +84,6 @@ func InsertMissingSlot(block *dbtypes.SlotHeader, tx *sqlx.Tx) error { return nil } -func GetSlots(firstSlot uint64, limit uint32, withMissing bool, withOrphaned bool) []*dbtypes.AssignedSlot { - var sql strings.Builder - fmt.Fprintf(&sql, `SELECT slots.slot, slots.proposer`) - blockFields := []string{ - "state_root", "root", "slot", "proposer", "status", "parent_root", "graffiti", "graffiti_text", - "attestation_count", "deposit_count", "exit_count", "withdraw_count", "withdraw_amount", "attester_slashing_count", - "proposer_slashing_count", "bls_change_count", "eth_transaction_count", "eth_block_number", "eth_block_hash", - "eth_block_extra", "eth_block_extra_text", "sync_participation", "fork_id", - } - for _, blockField := range blockFields { - fmt.Fprintf(&sql, ", slots.%v AS \"block.%v\"", blockField, blockField) - } - fmt.Fprintf(&sql, ` FROM slots `) - fmt.Fprintf(&sql, ` WHERE slot <= $1 `) - - if !withMissing { - fmt.Fprintf(&sql, ` AND slots.status != 0 `) - } - if !withOrphaned { - fmt.Fprintf(&sql, ` AND slots.status != 2 `) - } - fmt.Fprintf(&sql, ` ORDER BY slot DESC LIMIT $2`) - - rows, err := ReaderDb.Query(sql.String(), firstSlot, limit) - if err != nil { - logger.WithError(err).Errorf("Error while fetching slots: %v", sql.String()) - return nil - } - - return parseAssignedSlots(rows, blockFields, 2) -} - func GetSlotsRange(firstSlot uint64, lastSlot uint64, withMissing bool, withOrphaned bool) []*dbtypes.AssignedSlot { var sql strings.Builder fmt.Fprintf(&sql, `SELECT slots.slot, slots.proposer`) @@ -184,6 +153,43 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { return &block } +func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { + var sql strings.Builder + fmt.Fprintf(&sql, `SELECT + root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, + attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, + proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, + eth_block_extra, eth_block_extra_text, sync_participation, fork_id + FROM slots + WHERE root IN `) + + argIdx := 0 + args := make([]any, len(roots)) + plcList := make([]string, len(roots)) + for i, root := range roots { + plcList[i] = fmt.Sprintf("$%v", argIdx+1) + args[argIdx] = root + argIdx += 1 + } + fmt.Fprintf(&sql, "(%v)", strings.Join(plcList, ", ")) + + fmt.Fprintf(&sql, " ORDER BY slot DESC") + + slots := []*dbtypes.Slot{} + err := ReaderDb.Select(&slots, sql.String(), args...) + if err != nil { + //logger.Errorf("Error while fetching block by root 0x%x: %v", root, err) + return nil + } + + slotMap := make(map[phase0.Root]*dbtypes.Slot) + for _, slot := range slots { + slotMap[phase0.Root(slot.Root)] = slot + } + + return slotMap +} + func GetBlockHeadByRoot(root []byte) *dbtypes.BlockHead { blockHead := dbtypes.BlockHead{} err := ReaderDb.Get(&blockHead, ` diff --git a/handlers/index.go b/handlers/index.go index 8bcb21ee..185101b3 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -221,7 +221,7 @@ func buildIndexPageData() (*models.IndexPageData, time.Duration) { buildIndexPageRecentEpochsData(pageData, currentEpoch, finalizedEpoch, justifiedEpoch, recentEpochCount) // load recent blocks - buildIndexPageRecentBlocksData(pageData, currentSlot, recentBlockCount) + buildIndexPageRecentBlocksData(pageData, recentBlockCount) // load recent slots buildIndexPageRecentSlotsData(pageData, currentSlot, recentSlotsCount) @@ -257,14 +257,17 @@ func buildIndexPageRecentEpochsData(pageData *models.IndexPageData, currentEpoch pageData.RecentEpochCount = uint64(len(pageData.RecentEpochs)) } -func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, currentSlot phase0.Slot, recentBlockCount int) { +func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, recentBlockCount int) { pageData.RecentBlocks = make([]*models.IndexPageDataBlocks, 0) chainState := services.GlobalBeaconService.GetChainState() - blocksData := services.GlobalBeaconService.GetDbBlocks(uint64(currentSlot), int32(recentBlockCount), false, false) + blocksData := services.GlobalBeaconService.GetDbBlocksByFilter(&dbtypes.BlockFilter{ + WithOrphaned: 0, + WithMissing: 0, + }, 0, uint32(recentBlockCount), 0) for i := 0; i < len(blocksData); i++ { - blockData := blocksData[i] + blockData := blocksData[i].Block if blockData == nil { continue } diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index 8f158cd7..dcb8ce79 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -221,7 +221,12 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string blockFilter.ProposerIndex = &pidx } - dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize)) + withScheduledCount := chainState.GetSpecs().SlotsPerEpoch - uint64(chainState.SlotToSlotIndex(currentSlot)) - 1 + if withScheduledCount > 16 { + withScheduledCount = 16 + } + + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize), withScheduledCount) haveMore := false for idx, dbBlock := range dbBlocks { if idx >= int(pageSize) { diff --git a/handlers/validator.go b/handlers/validator.go index 0de07292..b017c820 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -169,7 +169,7 @@ func buildValidatorPageData(validatorIndex uint64) (*models.ValidatorPageData, t ProposerIndex: &validatorIndex, WithOrphaned: 1, WithMissing: 1, - }, 0, 10) + }, 0, 10, chainState.GetSpecs().SlotsPerEpoch) for _, blockData := range blocksData { var blockStatus dbtypes.SlotStatus if blockData.Block == nil { diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index d0026ea9..c80a7abd 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -104,7 +104,7 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint ProposerIndex: &validator, WithOrphaned: 1, WithMissing: 1, - }, pageIdx, uint32(pageSize)) + }, pageIdx, uint32(pageSize), chainState.GetSpecs().SlotsPerEpoch) haveMore := false for idx, blockAssignment := range dbBlocks { if idx >= int(pageSize) { diff --git a/indexer/beacon/block.go b/indexer/beacon/block.go index 169cd261..10bc1ca3 100644 --- a/indexer/beacon/block.go +++ b/indexer/beacon/block.go @@ -362,17 +362,6 @@ func (block *Block) GetDbConsolidationRequests(indexer *Indexer) []*dbtypes.Cons return indexer.dbWriter.buildDbConsolidationRequests(block, orphaned, nil) } -// GetExecutionExtraData returns the execution extra data of this block. -func (block *Block) GetExecutionExtraData() []byte { - blockBody := block.GetBlock() - if blockBody == nil { - return []byte{} - } - - data, _ := getBlockExecutionExtraData(blockBody) - return data -} - // GetForkId returns the fork ID of this block. func (block *Block) GetForkId() ForkKey { return block.forkId diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index cbb13898..f69a385b 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -214,67 +214,71 @@ func (bs *ChainService) GetBlobSidecarsByBlockRoot(ctx context.Context, blockroo return client.GetClient().GetRPCClient().GetBlobSidecarsByBlockroot(ctx, blockroot) } -func (bs *ChainService) GetDbBlocks(firstSlot uint64, limit int32, withMissing bool, withOrphaned bool) []*dbtypes.Slot { +func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, withMissing bool, withOrphaned bool) []*dbtypes.Slot { + resBlocks := make([]*dbtypes.Slot, 0) + chainState := bs.consensusPool.GetChainState() - resBlocks := make([]*dbtypes.Slot, limit) - resIdx := 0 + finalizedEpoch, prunedEpoch := bs.beaconIndexer.GetBlockCacheState() + prunedSlot := chainState.EpochToSlot(prunedEpoch) + finalizedSlot := chainState.EpochToSlot(finalizedEpoch) - _, prunedEpoch := bs.beaconIndexer.GetBlockCacheState() - idxMinSlot := chainState.EpochToSlot(prunedEpoch) - idxHeadSlot := uint64(chainState.CurrentSlot()) - if firstSlot > idxHeadSlot { - firstSlot = idxHeadSlot + var lastSlot uint64 + if firstSlot > uint64(slotLimit) { + lastSlot = firstSlot - uint64(slotLimit) + } else { + lastSlot = 0 } var proposerAssignments map[phase0.Slot]phase0.ValidatorIndex proposerAssignmentsEpoch := phase0.Epoch(math.MaxInt64) + getCanonicalProposer := func(slot phase0.Slot) phase0.ValidatorIndex { + epoch := chainState.EpochOfSlot(slot) + if epoch != proposerAssignmentsEpoch { + if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { + if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { + proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} + for slotIdx, proposer := range epochStatsValues.ProposerDuties { + slot := chainState.EpochToSlot(epoch) + phase0.Slot(slotIdx) + proposerAssignments[slot] = proposer + } + } + } + proposerAssignmentsEpoch = epoch + } + proposer, ok := proposerAssignments[slot] + if !ok { + proposer = phase0.ValidatorIndex(math.MaxInt64) + } + + return proposer + } + + // get blocks from cache slot := phase0.Slot(firstSlot) - if firstSlot >= uint64(idxMinSlot) { - for slotIdx := int64(slot); slotIdx >= int64(idxMinSlot) && resIdx < int(limit); slotIdx-- { + if slot >= prunedSlot { + for slotIdx := int64(slot); slotIdx >= int64(prunedSlot) && slotIdx >= int64(lastSlot); slotIdx-- { slot = phase0.Slot(slotIdx) - blocks := bs.beaconIndexer.GetBlocksBySlot(slot) - if len(blocks) > 0 { - for bidx := 0; bidx < len(blocks) && resIdx < int(limit); bidx++ { - block := blocks[bidx] - if !withOrphaned && !bs.beaconIndexer.IsCanonicalBlock(block, nil) { - continue - } - - dbBlock := block.GetDbBlock(bs.beaconIndexer) - if dbBlock != nil { - resBlocks[resIdx] = dbBlock - resIdx++ - } + for _, block := range blocks { + if !withOrphaned && !bs.beaconIndexer.IsCanonicalBlock(block, nil) { + continue + } + dbBlock := block.GetDbBlock(bs.beaconIndexer) + if dbBlock != nil { + resBlocks = append(resBlocks, dbBlock) } } if withMissing { - epoch := chainState.EpochOfSlot(slot) - if epoch != proposerAssignmentsEpoch { - if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { - if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { - proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} - for slotIdx, proposer := range epochStatsValues.ProposerDuties { - slot := chainState.EpochToSlot(epoch) + phase0.Slot(slotIdx) - proposerAssignments[slot] = proposer - } - } - } - proposerAssignmentsEpoch = epoch - } - hasCanonicalProposer := false - canonicalProposer := phase0.ValidatorIndex(math.MaxInt64) + canonicalProposer := getCanonicalProposer(slot) if len(blocks) > 0 { if proposerAssignments == nil { hasCanonicalProposer = true } else { - canonicalProposer = proposerAssignments[slot] - for bidx := 0; bidx < len(blocks) && resIdx < int(limit); bidx++ { - block := blocks[bidx] + for _, block := range blocks { header := block.GetHeader() if header == nil { continue @@ -286,17 +290,14 @@ func (bs *ChainService) GetDbBlocks(firstSlot uint64, limit int32, withMissing b } } } - } else if proposerAssignments != nil { - canonicalProposer = proposerAssignments[slot] } - if !hasCanonicalProposer { - resBlocks[resIdx] = &dbtypes.Slot{ + if !hasCanonicalProposer && slot > 0 { + resBlocks = append(resBlocks, &dbtypes.Slot{ Slot: uint64(slot), Proposer: uint64(canonicalProposer), Status: dbtypes.Missing, - } - resIdx++ + }) } } } @@ -305,107 +306,55 @@ func (bs *ChainService) GetDbBlocks(firstSlot uint64, limit int32, withMissing b } } - if resIdx < int(limit) { - dbBlocks := db.GetSlots(uint64(slot), uint32(limit-int32(resIdx)), withMissing, withOrphaned) - for _, dbBlock := range dbBlocks { - - if withMissing { - for ; slot > phase0.Slot(dbBlock.Slot+1); slot-- { - resBlocks[resIdx] = &dbtypes.Slot{ - Slot: uint64(slot), - Proposer: uint64(math.MaxInt64), - Status: dbtypes.Missing, - SyncParticipation: -1, - } - resIdx++ - - if resIdx >= int(limit) { - break - } - } - } - - if resIdx >= int(limit) { - break - } - - if dbBlock.Block != nil { - resBlocks[resIdx] = dbBlock.Block - } else { - resBlocks[resIdx] = &dbtypes.Slot{ - Slot: dbBlock.Slot, - Proposer: dbBlock.Proposer, - Status: dbtypes.Missing, - } - } - resIdx++ - slot = phase0.Slot(dbBlock.Slot) + // get pruned blocks from cache + if uint64(slot) > lastSlot && slot >= finalizedSlot { + unfinalizedLastSlot := phase0.Slot(lastSlot) + if finalizedSlot > unfinalizedLastSlot { + unfinalizedLastSlot = finalizedSlot } - } - - return resBlocks -} -func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, withMissing bool, withOrphaned bool) []*dbtypes.Slot { - resBlocks := make([]*dbtypes.Slot, 0) + // add unfinalized blocks from cache, with block stats from db + blockRoots := make([][]byte, 0) + blockRootsIdx := make([]int, 0) - chainState := bs.consensusPool.GetChainState() - _, prunedEpoch := bs.beaconIndexer.GetBlockCacheState() - idxMinSlot := chainState.EpochToSlot(prunedEpoch) + for slotIdx := int64(slot); slotIdx >= int64(unfinalizedLastSlot); slotIdx-- { + slot = phase0.Slot(slotIdx) - var lastSlot uint64 - if firstSlot > uint64(slotLimit) { - lastSlot = firstSlot - uint64(slotLimit) - } else { - lastSlot = 0 - } + blocks := bs.beaconIndexer.GetBlocksBySlot(slot) + for _, block := range blocks { + blockHeader := block.GetHeader() + if blockHeader == nil { + continue + } - var proposerAssignments map[phase0.Slot]phase0.ValidatorIndex - proposerAssignmentsEpoch := phase0.Epoch(math.MaxInt64) + isCanonical := bs.beaconIndexer.IsCanonicalBlock(block, nil) + if !withOrphaned && !isCanonical { + continue + } - slot := phase0.Slot(firstSlot) - if firstSlot >= uint64(idxMinSlot) { - for slotIdx := int64(slot); slotIdx >= int64(idxMinSlot) && slotIdx >= int64(lastSlot); slotIdx-- { - slot = phase0.Slot(slotIdx) - blocks := bs.beaconIndexer.GetBlocksBySlot(slot) - if len(blocks) > 0 { - for bidx := 0; bidx < len(blocks); bidx++ { - block := blocks[bidx] - if !withOrphaned && !bs.beaconIndexer.IsCanonicalBlock(block, nil) { - continue - } - dbBlock := block.GetDbBlock(bs.beaconIndexer) - if dbBlock != nil { - resBlocks = append(resBlocks, dbBlock) - } + blockStatus := dbtypes.Canonical + if !isCanonical { + blockStatus = dbtypes.Orphaned } + + blockRoots = append(blockRoots, block.Root[:]) + blockRootsIdx = append(blockRootsIdx, len(resBlocks)) + resBlocks = append(resBlocks, &dbtypes.Slot{ + Slot: uint64(slot), + Proposer: uint64(blockHeader.Message.ProposerIndex), + Status: blockStatus, + }) } if withMissing { - epoch := chainState.EpochOfSlot(slot) - if epoch != proposerAssignmentsEpoch { - if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { - if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { - proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} - for slotIdx, proposer := range epochStatsValues.ProposerDuties { - slot := chainState.EpochToSlot(epoch) + phase0.Slot(slotIdx) - proposerAssignments[slot] = proposer - } - } - } - proposerAssignmentsEpoch = epoch - } - hasCanonicalProposer := false - canonicalProposer := phase0.ValidatorIndex(math.MaxInt64) + canonicalProposer := getCanonicalProposer(slot) if len(blocks) > 0 { if proposerAssignments == nil { hasCanonicalProposer = true } else { - canonicalProposer = proposerAssignments[slot] - for bidx := 0; bidx < len(blocks); bidx++ { - block := blocks[bidx] + for _, block := range blocks { header := block.GetHeader() if header == nil { continue @@ -417,8 +366,6 @@ func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, } } } - } else if proposerAssignments != nil { - canonicalProposer = proposerAssignments[slot] } if !hasCanonicalProposer && slot > 0 { @@ -430,20 +377,33 @@ func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, } } } + + // load selected blocks from db + if len(blockRoots) > 0 { + blockMap := db.GetSlotsByRoots(blockRoots) + + for idx, blockRoot := range blockRoots { + if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { + dbBlock.Status = resBlocks[blockRootsIdx[idx]].Status + resBlocks[blockRootsIdx[idx]] = dbBlock + } + } + } + if slot > 0 { slot-- } } + // get finalized blocks from db if uint64(slot) > lastSlot { - dbBlocks := db.GetSlotsRange(uint64(slot), lastSlot, withMissing, withOrphaned) - + dbBlocks := db.GetSlotsRange(uint64(slot), uint64(lastSlot), withMissing, withOrphaned) for _, dbBlock := range dbBlocks { if withMissing { for ; uint64(slot) > dbBlock.Slot+1; slot-- { resBlocks = append(resBlocks, &dbtypes.Slot{ Slot: uint64(slot), - Proposer: uint64(math.MaxInt64), + Proposer: uint64(getCanonicalProposer(slot)), Status: dbtypes.Missing, SyncParticipation: -1, }) @@ -466,7 +426,7 @@ func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, for ; uint64(slot) > lastSlot+1; slot-- { resBlocks = append(resBlocks, &dbtypes.Slot{ Slot: uint64(slot), - Proposer: uint64(math.MaxInt64), + Proposer: uint64(getCanonicalProposer(slot)), Status: dbtypes.Missing, SyncParticipation: -1, }) @@ -480,126 +440,145 @@ func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, type cachedDbBlock struct { slot uint64 proposer uint64 + orphaned bool block *beacon.Block } -func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx uint64, pageSize uint32) []*dbtypes.AssignedSlot { +func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx uint64, pageSize uint32, withScheduledCount uint64) []*dbtypes.AssignedSlot { cachedMatches := make([]cachedDbBlock, 0) chainState := bs.consensusPool.GetChainState() - _, prunedEpoch := bs.beaconIndexer.GetBlockCacheState() - idxMinSlot := chainState.EpochToSlot(prunedEpoch) + finalizedEpoch, prunedEpoch := bs.beaconIndexer.GetBlockCacheState() + prunedSlot := chainState.EpochToSlot(prunedEpoch) + finalizedSlot := chainState.EpochToSlot(finalizedEpoch) + + currentSlot := chainState.CurrentSlot() + startSlot := currentSlot + if withScheduledCount > 0 { + startSlot += phase0.Slot(withScheduledCount) + } + + var proposerAssignments map[phase0.Slot]phase0.ValidatorIndex + proposerAssignmentsEpoch := phase0.Epoch(math.MaxInt64) + getCanonicalProposer := func(slot phase0.Slot) phase0.ValidatorIndex { + epoch := chainState.EpochOfSlot(slot) + if epoch != proposerAssignmentsEpoch { + if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { + if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { + proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} + for slotIdx, proposer := range epochStatsValues.ProposerDuties { + slot := chainState.EpochToSlot(epoch) + phase0.Slot(slotIdx) + proposerAssignments[slot] = proposer + } + } + } + proposerAssignmentsEpoch = epoch + } + + proposer, ok := proposerAssignments[slot] + if !ok { + proposer = phase0.ValidatorIndex(math.MaxInt64) + } - idxHeadSlot := chainState.CurrentSlot() + return proposer + } - proposedMap := map[phase0.Slot]bool{} - for slotIdx := int64(idxHeadSlot); slotIdx >= int64(idxMinSlot); slotIdx-- { + // get blocks from cache + addedMissingBlocks := false + for slotIdx := int64(startSlot); slotIdx >= int64(finalizedSlot); slotIdx-- { slot := phase0.Slot(slotIdx) blocks := bs.beaconIndexer.GetBlocksBySlot(slot) - if blocks != nil { - for bidx := 0; bidx < len(blocks); bidx++ { - block := blocks[bidx] - blockHeader := block.GetHeader() - if blockHeader == nil { + for _, block := range blocks { + blockHeader := block.GetHeader() + if blockHeader == nil { + continue + } + blockIndex := block.GetBlockIndex() + if blockIndex == nil { + continue + } + + isOrphaned := !bs.beaconIndexer.IsCanonicalBlock(block, nil) + if filter.WithOrphaned != 1 { + if filter.WithOrphaned == 0 && isOrphaned { continue } - blockBody := block.GetBlock() - if blockBody == nil { - continue - } - if filter.WithOrphaned != 1 { - isOrphaned := !bs.beaconIndexer.IsCanonicalBlock(block, nil) - if filter.WithOrphaned == 0 && isOrphaned { - continue - } - if filter.WithOrphaned == 2 && !isOrphaned { - continue - } - } - proposedMap[block.Slot] = true - if filter.WithMissing == 2 { + if filter.WithOrphaned == 2 && !isOrphaned { continue } + } - if filter.Graffiti != "" { - graffitiBytes, _ := blockBody.Graffiti() - blockGraffiti := string(graffitiBytes[:]) - if !strings.Contains(blockGraffiti, filter.Graffiti) { - continue - } + if filter.WithMissing == 2 { + continue + } + + if filter.Graffiti != "" { + blockGraffiti := string(blockIndex.Graffiti[:]) + if !strings.Contains(blockGraffiti, filter.Graffiti) { + continue } - if filter.ExtraData != "" { - executionExtraData := block.GetExecutionExtraData() - blockExtraData := string(executionExtraData[:]) - if !strings.Contains(blockExtraData, filter.ExtraData) { - continue - } + } + if filter.ExtraData != "" { + blockExtraData := string(blockIndex.ExecutionExtraData) + if !strings.Contains(blockExtraData, filter.ExtraData) { + continue } - proposer := uint64(blockHeader.Message.ProposerIndex) - if filter.ProposerIndex != nil { - if proposer != *filter.ProposerIndex { - continue - } + } + proposer := uint64(blockHeader.Message.ProposerIndex) + if filter.ProposerIndex != nil { + if proposer != *filter.ProposerIndex { + continue } - if filter.ProposerName != "" { - proposerName := bs.validatorNames.GetValidatorName(proposer) - if !strings.Contains(proposerName, filter.ProposerName) { - continue - } + } + if filter.ProposerName != "" { + proposerName := bs.validatorNames.GetValidatorName(proposer) + if !strings.Contains(proposerName, filter.ProposerName) { + continue } - - cachedMatches = append(cachedMatches, cachedDbBlock{ - slot: uint64(block.Slot), - proposer: uint64(blockHeader.Message.ProposerIndex), - block: block, - }) } + + cachedMatches = append(cachedMatches, cachedDbBlock{ + slot: uint64(block.Slot), + proposer: uint64(blockHeader.Message.ProposerIndex), + orphaned: isOrphaned, + block: block, + }) } - } - if filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 { - // add missed blocks - idxHeadEpoch := chainState.EpochOfSlot(idxHeadSlot) - idxMinEpoch := chainState.EpochOfSlot(idxMinSlot) + if filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 { + hasCanonicalProposer := false + canonicalProposer := getCanonicalProposer(slot) - for epochIdx := int64(idxHeadEpoch); epochIdx >= int64(idxMinEpoch); epochIdx-- { - epoch := phase0.Epoch(epochIdx) - var proposerAssignments map[phase0.Slot]phase0.ValidatorIndex + if len(blocks) > 0 { + if proposerAssignments == nil { + hasCanonicalProposer = true + } else { + for _, block := range blocks { + header := block.GetHeader() + if header == nil { + continue + } - if epochStats := bs.beaconIndexer.GetEpochStats(epoch, nil); epochStats != nil { - if epochStatsValues := epochStats.GetValues(true); epochStatsValues != nil { - proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} - for slotIdx, proposer := range epochStatsValues.ProposerDuties { - slot := chainState.EpochToSlot(epoch) + phase0.Slot(slotIdx) - proposerAssignments[slot] = proposer + if header.Message.ProposerIndex == canonicalProposer { + hasCanonicalProposer = true + break + } } } } - if proposerAssignments == nil { - proposerAssignments = map[phase0.Slot]phase0.ValidatorIndex{} - firstSlot := chainState.EpochToSlot(epoch) - lastSlot := chainState.EpochToSlot(epoch + 1) - for slot := firstSlot; slot < lastSlot; slot++ { - proposerAssignments[slot] = math.MaxInt64 - } - } - - for slot, assigned := range proposerAssignments { - if proposedMap[slot] { - continue - } - if filter.WithMissing == 2 && slot > idxHeadSlot { + if !hasCanonicalProposer && slot > 0 { + if filter.WithMissing == 2 && slot > currentSlot { continue } if filter.ProposerIndex != nil { - if uint64(assigned) != *filter.ProposerIndex { + if uint64(canonicalProposer) != *filter.ProposerIndex { continue } } if filter.ProposerName != "" { - assignedName := bs.validatorNames.GetValidatorName(uint64(assigned)) + assignedName := bs.validatorNames.GetValidatorName(uint64(canonicalProposer)) if assignedName == "" || !strings.Contains(assignedName, filter.ProposerName) { continue } @@ -607,18 +586,26 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx cachedMatches = append(cachedMatches, cachedDbBlock{ slot: uint64(slot), - proposer: uint64(assigned), + proposer: uint64(canonicalProposer), block: nil, }) + addedMissingBlocks = true } - sort.Slice(cachedMatches, func(a, b int) bool { - slotA := cachedMatches[a].slot - slotB := cachedMatches[b].slot - return slotA > slotB - }) + } + + if uint64(len(cachedMatches)) >= uint64(pageIdx+1)*uint64(pageSize) { + break } } + if addedMissingBlocks { + sort.Slice(cachedMatches, func(a, b int) bool { + slotA := cachedMatches[a].slot + slotB := cachedMatches[b].slot + return slotA > slotB + }) + } + cachedMatchesLen := uint64(len(cachedMatches)) cachedPages := cachedMatchesLen / uint64(pageSize) resBlocks := make([]*dbtypes.AssignedSlot, 0) @@ -630,33 +617,54 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx cachedEnd++ } - if cachedPages > 0 && pageIdx < cachedPages { - for _, block := range cachedMatches[cachedStart:cachedEnd] { + blockRoots := make([][]byte, 0) + blockRootsIdx := make([]int, 0) + blockRootsCachedId := make([]uint64, 0) + + if cachedPages > 0 && pageIdx <= cachedPages { + var cachedMatchesRange []cachedDbBlock + if pageIdx == cachedPages { + cachedMatchesRange = cachedMatches[cachedStart:] + } else { + cachedMatchesRange = cachedMatches[cachedStart:cachedEnd] + } + + for cidx, block := range cachedMatchesRange { assignedBlock := dbtypes.AssignedSlot{ Slot: block.slot, Proposer: block.proposer, } if block.block != nil { - assignedBlock.Block = block.block.GetDbBlock(bs.beaconIndexer) + if block.slot >= uint64(prunedSlot) { + assignedBlock.Block = block.block.GetDbBlock(bs.beaconIndexer) + } else { + blockRoots = append(blockRoots, block.block.Root[:]) + blockRootsIdx = append(blockRootsIdx, resIdx) + blockRootsCachedId = append(blockRootsCachedId, cachedStart+uint64(cidx)) + } } resBlocks = append(resBlocks, &assignedBlock) resIdx++ } - } else if pageIdx == cachedPages { - start := pageIdx * uint64(pageSize) - for _, block := range cachedMatches[start:] { - assignedBlock := dbtypes.AssignedSlot{ - Slot: block.slot, - Proposer: block.proposer, - } - if block.block != nil { - assignedBlock.Block = block.block.GetDbBlock(bs.beaconIndexer) + } + + if len(blockRoots) > 0 { + blockMap := db.GetSlotsByRoots(blockRoots) + + for idx, blockRoot := range blockRoots { + if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { + + dbBlock.Status = dbtypes.Canonical + if cachedMatches[blockRootsCachedId[idx]].orphaned { + dbBlock.Status = dbtypes.Orphaned + } + + resBlocks[blockRootsIdx[idx]].Block = dbBlock } - resBlocks = append(resBlocks, &assignedBlock) - resIdx++ } } - if resIdx > int(pageSize) { + + if resIdx >= int(pageSize) { return resBlocks } @@ -665,9 +673,9 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx dbCacheOffset := uint64(pageSize) - (cachedMatchesLen % uint64(pageSize)) var dbBlocks []*dbtypes.AssignedSlot if dbPage == 0 { - dbBlocks = db.GetFilteredSlots(filter, uint64(idxMinSlot), 0, uint32(dbCacheOffset)+1) + dbBlocks = db.GetFilteredSlots(filter, uint64(finalizedSlot), 0, uint32(dbCacheOffset)+1) } else { - dbBlocks = db.GetFilteredSlots(filter, uint64(idxMinSlot), (dbPage-1)*uint64(pageSize)+dbCacheOffset, pageSize+1) + dbBlocks = db.GetFilteredSlots(filter, uint64(finalizedSlot), (dbPage-1)*uint64(pageSize)+dbCacheOffset, pageSize+1) } resBlocks = append(resBlocks, dbBlocks...) From 3f00aa6f57f87672a00847a3aea2ce46ae59bfe4 Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 22 Aug 2024 21:42:18 +0200 Subject: [PATCH 2/6] cleanup & add comments --- services/chainservice_blocks.go | 62 ++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index f69a385b..d6a92a65 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "math" - "sort" "strings" "github.com/attestantio/go-eth2-client/spec" @@ -25,6 +24,11 @@ type CombinedBlockResponse struct { Orphaned bool } +// GetBlockBlob retrieves the blob sidecar for a given block root and commitment. +// It first tries to find a client that has the block root in its cache, and if not found, +// it falls back to a random ready client. It then retrieves the blob sidecars for the block root +// and checks if any of them match the given commitment. If a match is found, it returns the blob sidecar, +// otherwise it returns nil. func (bs *ChainService) GetBlockBlob(ctx context.Context, blockroot phase0.Root, commitment deneb.KZGCommitment) (*deneb.BlobSidecar, error) { client := bs.beaconIndexer.GetReadyClientByBlockRoot(blockroot, true) if client == nil { @@ -49,6 +53,13 @@ func (bs *ChainService) GetBlockBlob(ctx context.Context, blockroot phase0.Root, return nil, nil } +// GetSlotDetailsByBlockroot retrieves the combined block details for a given block root. +// It first checks if the block root is present in the beacon indexer's block cache. +// If found, it constructs a CombinedBlockResponse using the block information from the cache. +// If not found, it checks if the block root is present in the orphaned block database. +// If found, it constructs a CombinedBlockResponse with the orphaned block information. +// If not found in either cache or db, it retrieves the block header and block body from a random +// ready client and constructs a CombinedBlockResponse with the retrieved information. func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot phase0.Root) (*CombinedBlockResponse, error) { var result *CombinedBlockResponse if blockInfo := bs.beaconIndexer.GetBlockByRoot(blockroot); blockInfo != nil { @@ -125,6 +136,11 @@ func (bs *ChainService) GetSlotDetailsByBlockroot(ctx context.Context, blockroot return result, nil } +// GetSlotDetailsBySlot retrieves the combined block details for a given slot. +// It first checks if there are any blocks in the beacon indexer's block cache for the given slot. +// If found, it constructs a CombinedBlockResponse using the block information from the cache. +// If not found, it retrieves the block header and block body from a random ready client +// using the slot and constructs a CombinedBlockResponse with the retrieved information. func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Slot) (*CombinedBlockResponse, error) { var result *CombinedBlockResponse if cachedBlocks := bs.beaconIndexer.GetBlocksBySlot(slot); len(cachedBlocks) > 0 { @@ -205,6 +221,10 @@ func (bs *ChainService) GetSlotDetailsBySlot(ctx context.Context, slot phase0.Sl return result, nil } +// GetBlobSidecarsByBlockRoot retrieves the blob sidecars for a given block root. +// It first tries to find a client that has the block root in its cache, and if not found, +// it falls back to a random ready client. It then retrieves the blob sidecars for the block root +// and returns them. func (bs *ChainService) GetBlobSidecarsByBlockRoot(ctx context.Context, blockroot []byte) ([]*deneb.BlobSidecar, error) { client := bs.beaconIndexer.GetReadyClientByBlockRoot(phase0.Root(blockroot), true) if client == nil { @@ -214,6 +234,12 @@ func (bs *ChainService) GetBlobSidecarsByBlockRoot(ctx context.Context, blockroo return client.GetClient().GetRPCClient().GetBlobSidecarsByBlockroot(ctx, blockroot) } +// GetDbBlocksForSlots retrieves blocks for a range of slots from cache & database. +// The firstSlot parameter specifies the starting slot. +// The slotLimit parameter limits the number of slots to retrieve. +// The withMissing parameter indicates whether to include missing blocks. +// The withOrphaned parameter indicates whether to include orphaned blocks. +// The returned slice contains the retrieved blocks. func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, withMissing bool, withOrphaned bool) []*dbtypes.Slot { resBlocks := make([]*dbtypes.Slot, 0) @@ -444,6 +470,12 @@ type cachedDbBlock struct { block *beacon.Block } +// GetDbBlocksByFilter retrieves a filtered range of blocks from cache & database. +// The filter parameter specifies the filter criteria. +// The pageIdx parameter specifies the page index. +// The pageSize parameter specifies the page size. +// The withScheduledCount parameter specifies the number of scheduled slots to include. +// The returned slice contains the retrieved blocks. func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx uint64, pageSize uint32, withScheduledCount uint64) []*dbtypes.AssignedSlot { cachedMatches := make([]cachedDbBlock, 0) @@ -458,6 +490,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx startSlot += phase0.Slot(withScheduledCount) } + // getCanonicalProposer is a local helper function to get the canonical proposer for a given slot var proposerAssignments map[phase0.Slot]phase0.ValidatorIndex proposerAssignmentsEpoch := phase0.Epoch(math.MaxInt64) getCanonicalProposer := func(slot phase0.Slot) phase0.ValidatorIndex { @@ -484,7 +517,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } // get blocks from cache - addedMissingBlocks := false + // iterate from current slot to finalized slot for slotIdx := int64(startSlot); slotIdx >= int64(finalizedSlot); slotIdx-- { slot := phase0.Slot(slotIdx) blocks := bs.beaconIndexer.GetBlocksBySlot(slot) @@ -501,29 +534,37 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx isOrphaned := !bs.beaconIndexer.IsCanonicalBlock(block, nil) if filter.WithOrphaned != 1 { if filter.WithOrphaned == 0 && isOrphaned { + // only canonical blocks, skip continue } if filter.WithOrphaned == 2 && !isOrphaned { + // only orphaned blocks, skip continue } } if filter.WithMissing == 2 { + // only missing blocks, skip continue } + // filter by graffiti if filter.Graffiti != "" { blockGraffiti := string(blockIndex.Graffiti[:]) if !strings.Contains(blockGraffiti, filter.Graffiti) { continue } } + + // filter by extra data if filter.ExtraData != "" { blockExtraData := string(blockIndex.ExecutionExtraData) if !strings.Contains(blockExtraData, filter.ExtraData) { continue } } + + // filter by proposer proposer := uint64(blockHeader.Message.ProposerIndex) if filter.ProposerIndex != nil { if proposer != *filter.ProposerIndex { @@ -545,10 +586,12 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx }) } + // reconstruct missing blocks from epoch duties if filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 { hasCanonicalProposer := false canonicalProposer := getCanonicalProposer(slot) + // check if canonical proposer has proposed a block if len(blocks) > 0 { if proposerAssignments == nil { hasCanonicalProposer = true @@ -572,6 +615,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx continue } + // filter missing blocks by proposer if filter.ProposerIndex != nil { if uint64(canonicalProposer) != *filter.ProposerIndex { continue @@ -589,7 +633,6 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx proposer: uint64(canonicalProposer), block: nil, }) - addedMissingBlocks = true } } @@ -598,14 +641,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } } - if addedMissingBlocks { - sort.Slice(cachedMatches, func(a, b int) bool { - slotA := cachedMatches[a].slot - slotB := cachedMatches[b].slot - return slotA > slotB - }) - } - + // select range of requested page from matches cachedMatchesLen := uint64(len(cachedMatches)) cachedPages := cachedMatchesLen / uint64(pageSize) resBlocks := make([]*dbtypes.AssignedSlot, 0) @@ -617,6 +653,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx cachedEnd++ } + // build dbtypes.Slot objects for selected cache matches blockRoots := make([][]byte, 0) blockRootsIdx := make([]int, 0) blockRootsCachedId := make([]uint64, 0) @@ -648,6 +685,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } } + // load pruned blocks from database if len(blockRoots) > 0 { blockMap := db.GetSlotsByRoots(blockRoots) @@ -668,7 +706,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx return resBlocks } - // load from db + // load finalized slots from db dbPage := pageIdx - cachedPages dbCacheOffset := uint64(pageSize) - (cachedMatchesLen % uint64(pageSize)) var dbBlocks []*dbtypes.AssignedSlot From 118188b902a7f0412238f9f69fa34030ab30903e Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 22 Aug 2024 22:37:36 +0200 Subject: [PATCH 3/6] fix paging on filteres slots page --- handlers/slots_filtered.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index dcb8ce79..afd93196 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -226,7 +226,7 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string withScheduledCount = 16 } - dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize), withScheduledCount) + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize)+1, withScheduledCount) haveMore := false for idx, dbBlock := range dbBlocks { if idx >= int(pageSize) { From 9fec525fe84c485c8c6a1e6429711dc48fb4bc9d Mon Sep 17 00:00:00 2001 From: pk910 Date: Thu, 22 Aug 2024 23:52:13 +0200 Subject: [PATCH 4/6] revert 118188b and fix paging on filtered slots page properly --- handlers/index.go | 11 ++++++++++- handlers/slots_filtered.go | 2 +- services/chainservice_blocks.go | 11 +++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/handlers/index.go b/handlers/index.go index 185101b3..6efa05a5 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -266,7 +266,12 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, recentBlockC WithOrphaned: 0, WithMissing: 0, }, 0, uint32(recentBlockCount), 0) - for i := 0; i < len(blocksData); i++ { + limit := len(blocksData) + if limit > recentBlockCount { + limit = recentBlockCount + } + + for i := 0; i < limit; i++ { blockData := blocksData[i].Block if blockData == nil { continue @@ -330,6 +335,10 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot pha pageData.RecentSlots = append(pageData.RecentSlots, slotData) blockCount++ buildIndexPageSlotGraph(slotData, &maxOpenFork, openForks) + + if blockCount >= uint64(slotLimit) { + break + } } } pageData.RecentSlotCount = uint64(blockCount) diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index afd93196..dcb8ce79 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -226,7 +226,7 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string withScheduledCount = 16 } - dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize)+1, withScheduledCount) + dbBlocks := services.GlobalBeaconService.GetDbBlocksByFilter(blockFilter, pageIdx, uint32(pageSize), withScheduledCount) haveMore := false for idx, dbBlock := range dbBlocks { if idx >= int(pageSize) { diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index d6a92a65..d1e9a566 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -636,7 +636,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } } - if uint64(len(cachedMatches)) >= uint64(pageIdx+1)*uint64(pageSize) { + if uint64(len(cachedMatches)) > uint64(pageIdx+1)*uint64(pageSize) { break } } @@ -649,7 +649,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx cachedStart := pageIdx * uint64(pageSize) cachedEnd := cachedStart + uint64(pageSize) - if cachedEnd+1 < cachedMatchesLen { + if cachedEnd+1 <= cachedMatchesLen { cachedEnd++ } @@ -702,12 +702,15 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx } } - if resIdx >= int(pageSize) { + if resIdx > int(pageSize) { return resBlocks } // load finalized slots from db - dbPage := pageIdx - cachedPages + dbPage := uint64(0) + if pageIdx > cachedPages { + dbPage = pageIdx - cachedPages + } dbCacheOffset := uint64(pageSize) - (cachedMatchesLen % uint64(pageSize)) var dbBlocks []*dbtypes.AssignedSlot if dbPage == 0 { From 8ded7b53a4477482583f58bb60e4d3a2811f2f32 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 23 Aug 2024 13:07:53 +0200 Subject: [PATCH 5/6] clean up `db.GetSlotsByRoots` --- db/slots.go | 25 +++++++++++++------------ services/chainservice_blocks.go | 26 ++++++++++++++------------ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/db/slots.go b/db/slots.go index f0f60f33..a140d537 100644 --- a/db/slots.go +++ b/db/slots.go @@ -154,15 +154,6 @@ func GetSlotByRoot(root []byte) *dbtypes.Slot { } func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { - var sql strings.Builder - fmt.Fprintf(&sql, `SELECT - root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, - attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, - proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, - eth_block_extra, eth_block_extra_text, sync_participation, fork_id - FROM slots - WHERE root IN `) - argIdx := 0 args := make([]any, len(roots)) plcList := make([]string, len(roots)) @@ -171,14 +162,24 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { args[argIdx] = root argIdx += 1 } - fmt.Fprintf(&sql, "(%v)", strings.Join(plcList, ", ")) - fmt.Fprintf(&sql, " ORDER BY slot DESC") + var sql strings.Builder + fmt.Fprintf(&sql, + `SELECT + root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, + attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, + proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, + eth_block_extra, eth_block_extra_text, sync_participation, fork_id + FROM slots + WHERE root IN (%v) + ORDER BY slot DESC`, + strings.Join(plcList, ", "), + ) slots := []*dbtypes.Slot{} err := ReaderDb.Select(&slots, sql.String(), args...) if err != nil { - //logger.Errorf("Error while fetching block by root 0x%x: %v", root, err) + logger.Errorf("Error while fetching block by roots: %v", err) return nil } diff --git a/services/chainservice_blocks.go b/services/chainservice_blocks.go index d1e9a566..76a73c18 100644 --- a/services/chainservice_blocks.go +++ b/services/chainservice_blocks.go @@ -407,11 +407,12 @@ func (bs *ChainService) GetDbBlocksForSlots(firstSlot uint64, slotLimit uint32, // load selected blocks from db if len(blockRoots) > 0 { blockMap := db.GetSlotsByRoots(blockRoots) - - for idx, blockRoot := range blockRoots { - if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { - dbBlock.Status = resBlocks[blockRootsIdx[idx]].Status - resBlocks[blockRootsIdx[idx]] = dbBlock + if blockMap != nil { + for idx, blockRoot := range blockRoots { + if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { + dbBlock.Status = resBlocks[blockRootsIdx[idx]].Status + resBlocks[blockRootsIdx[idx]] = dbBlock + } } } } @@ -688,16 +689,17 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx // load pruned blocks from database if len(blockRoots) > 0 { blockMap := db.GetSlotsByRoots(blockRoots) + if blockMap != nil { + for idx, blockRoot := range blockRoots { + if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { - for idx, blockRoot := range blockRoots { - if dbBlock, ok := blockMap[phase0.Root(blockRoot)]; ok { + dbBlock.Status = dbtypes.Canonical + if cachedMatches[blockRootsCachedId[idx]].orphaned { + dbBlock.Status = dbtypes.Orphaned + } - dbBlock.Status = dbtypes.Canonical - if cachedMatches[blockRootsCachedId[idx]].orphaned { - dbBlock.Status = dbtypes.Orphaned + resBlocks[blockRootsIdx[idx]].Block = dbBlock } - - resBlocks[blockRootsIdx[idx]].Block = dbBlock } } } From 8a772a2903bb684701613012a33d6c43fe2c1f47 Mon Sep 17 00:00:00 2001 From: pk910 Date: Fri, 23 Aug 2024 13:12:13 +0200 Subject: [PATCH 6/6] simplify `GetSlotsByRoots` --- db/slots.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db/slots.go b/db/slots.go index a140d537..3e2f8064 100644 --- a/db/slots.go +++ b/db/slots.go @@ -163,8 +163,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { argIdx += 1 } - var sql strings.Builder - fmt.Fprintf(&sql, + sql := fmt.Sprintf( `SELECT root, slot, parent_root, state_root, status, proposer, graffiti, graffiti_text, attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, @@ -177,7 +176,7 @@ func GetSlotsByRoots(roots [][]byte) map[phase0.Root]*dbtypes.Slot { ) slots := []*dbtypes.Slot{} - err := ReaderDb.Select(&slots, sql.String(), args...) + err := ReaderDb.Select(&slots, sql, args...) if err != nil { logger.Errorf("Error while fetching block by roots: %v", err) return nil