Skip to content

Commit

Permalink
fix snapshots_cmd
Browse files Browse the repository at this point in the history
  • Loading branch information
@blxdyx
blxdyx committed Aug 22, 2024
1 parent cc8122b commit ef01dc5
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 61 deletions.
152 changes: 104 additions & 48 deletions turbo/snapshotsync/freezeblocks/bsc_snapshots.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,72 +279,128 @@ func (s *BscRoSnapshots) ReadBlobSidecars(blockNum uint64) ([]*types.BlobSidecar
return sidecars, nil
}

var missedBlobs = []uint64{
39565743,
39679711,
39679823,
39688495,
39712447,
39712471,
39712559,
39713015,
39716655,
39716807,
40180114,
40180115,
40208199,
40211855,
40211856,
40212934,
40212939,
40227488,
40231835,
40232104,
40239458,
40297217,
40314598,
40315505,
40316006,
40316698,
40319042,
40320395,
40322006,
40322007,
40693732,
40899197,
41756117,
41756119,
41756123,
41756126,
41756130,
41756132,
41756134,
41756135,
41756138,
41756142,
41756145,
41756147,
41756149,
41756152,
41756156,
41756160,
41756163,
41756164,
41756165,
41756169,
41756172,
}

func checkBlobs(ctx context.Context, blockFrom, blockTo uint64, chainDB kv.RoDB, blobStore services.BlobStorage, blockReader services.FullBlockReader, logger log.Logger) bool {
tx, err := chainDB.BeginRo(ctx)
if err != nil {
return false
}
defer tx.Rollback()
var missedBlobs []uint64
noErr := true
for i := blockFrom; i < blockTo; i++ {
block, err := blockReader.BlockByNumber(ctx, tx, i)
if err != nil {
log.Error("ReadCanonicalHash", "blockNum", i, "blockHash", block.Hash(), "err", err)
noErr = false
}
var blobTxCount uint64

for _, tx := range block.Transactions() {
if tx.Type() != types.BlobTxType {
continue
}
blobTxCount++
}
if blobTxCount == 0 {
continue
}
blobs, found, err := blobStore.ReadBlobSidecars(ctx, i, block.Hash())
if err != nil {
noErr = false
missedBlobs = append(missedBlobs, i)
log.Error("read blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err)
err := blobStore.RemoveBlobSidecars(ctx, i, block.Hash())
log.Error("Remove blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err)
continue
}
if !found {
noErr = false
missedBlobs = append(missedBlobs, i)
log.Error("blob sidecars not found for block ", "blockNumber", i, "count", blobTxCount)
continue
}

if uint64(len(blobs)) != blobTxCount {
missedBlobs = append(missedBlobs, i)
noErr = false
log.Error("blob sidecars not found for block ", "blockNumber", i, "want", blobTxCount, "actual", len(blobs))
continue
}

if i%20_000 == 0 {
logger.Info("Dumping beacon blobs", "progress", i)
}
}
//var missedBlobs []uint64
//noErr := true
//for i := blockFrom; i < blockTo; i++ {
// block, err := blockReader.BlockByNumber(ctx, tx, i)
// if err != nil {
// log.Error("ReadCanonicalHash", "blockNum", i, "blockHash", block.Hash(), "err", err)
// noErr = false
// }
// var blobTxCount uint64
//
// for _, tx := range block.Transactions() {
// if tx.Type() != types.BlobTxType {
// continue
// }
// blobTxCount++
// }
// if blobTxCount == 0 {
// continue
// }
// blobs, found, err := blobStore.ReadBlobSidecars(ctx, i, block.Hash())
// if err != nil {
// noErr = false
// missedBlobs = append(missedBlobs, i)
// log.Error("read blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err)
// err := blobStore.RemoveBlobSidecars(ctx, i, block.Hash())
// log.Error("Remove blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err)
// continue
// }
// if !found {
// noErr = false
// missedBlobs = append(missedBlobs, i)
// log.Error("blob sidecars not found for block ", "blockNumber", i, "count", blobTxCount)
// continue
// }
//
// if uint64(len(blobs)) != blobTxCount {
// missedBlobs = append(missedBlobs, i)
// noErr = false
// log.Error("blob sidecars not found for block ", "blockNumber", i, "want", blobTxCount, "actual", len(blobs))
// continue
// }
//
// if i%20_000 == 0 {
// logger.Info("Dumping beacon blobs", "progress", i)
// }
//}

log.Info("Start query missedBlobs from http")
for _, num := range missedBlobs {
blobs := GetBlobSidecars(num)
hash, err := blockReader.CanonicalHash(ctx, tx, num)
if err != nil {
log.Error("GetBlobSidecars failed", "num", num, "err", err)
return noErr
return false
}
if err = blobStore.WriteBlobSidecars(ctx, hash, blobs); err != nil {
log.Error("WriteBlobSidecars failed", "num", num, "err", err)
}
time.Sleep(1 * time.Second)
}

return noErr
return false
}
27 changes: 14 additions & 13 deletions turbo/snapshotsync/freezeblocks/utill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/ledgerwatch/erigon-lib/common/hexutil"
"github.com/ledgerwatch/erigon/core/types"
"io/ioutil"
"math/big"
"net/http"
)

Expand All @@ -24,16 +23,16 @@ type RPCRequest struct {
}

type BlockResponse struct {
Jsonrpc string `json:"jsonrpc"`
Id int `json:"id"`
Result interface{} `json:"result"`
Error interface{} `json:"error"`
Jsonrpc string `json:"jsonrpc"`
Id int `json:"id"`
Result json.RawMessage `json:"result"`
Error interface{} `json:"error"`
}
type BlobResponse struct {
BlobTxSidecar types.BlobTxSidecar `json:"blobSidecar"`
BlockNumber *big.Int `json:"blockNumber"`
BlockNumber string `json:"blockNumber"`
BlockHash libcommon.Hash `json:"blockHash"`
TxIndex uint64 `json:"txIndex"`
TxIndex string `json:"txIndex"`
TxHash libcommon.Hash `json:"txHash"`
}

Expand Down Expand Up @@ -68,7 +67,7 @@ func GetBlobSidecars(blockNumber uint64) types.BlobSidecars {
var blockResponse BlockResponse
err = json.Unmarshal(responseBody, &blockResponse)
if err != nil {
fmt.Println("Error unmarshalling response:", err)
fmt.Println("Error unmarshalling response a:", err)
return nil
}

Expand All @@ -77,20 +76,22 @@ func GetBlobSidecars(blockNumber uint64) types.BlobSidecars {
return nil
}

var blobResponse []BlobResponse
err = json.Unmarshal(responseBody, &blobResponse)
var blobResponse []*BlobResponse
err = json.Unmarshal(blockResponse.Result, &blobResponse)
if err != nil {
fmt.Println("Error unmarshalling response:", err)
fmt.Println("Error unmarshalling response b:", err)
return nil
}

var blobSidecars types.BlobSidecars
for _, blobSidecar := range blobResponse {
bn, _ := hexutil.DecodeBig(blobSidecar.BlockNumber)
tx, _ := hexutil.DecodeUint64(blobSidecar.TxIndex)
blob := &types.BlobSidecar{
BlobTxSidecar: blobSidecar.BlobTxSidecar,
BlockNumber: blobSidecar.BlockNumber,
BlockNumber: bn,
BlockHash: blobSidecar.BlockHash,
TxIndex: blobSidecar.TxIndex,
TxIndex: tx,
TxHash: blobSidecar.TxHash,
}
blobSidecars = append(blobSidecars, blob)
Expand Down
29 changes: 29 additions & 0 deletions turbo/snapshotsync/freezeblocks/utill_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package freezeblocks

import (
"github.com/ledgerwatch/erigon/core/types"
"reflect"
"testing"
)

func TestGetBlobSidecars(t *testing.T) {

tests := []struct {
name string
blockNumber uint64
want types.BlobSidecars
}{
{
name: "test1",
blockNumber: uint64(39565743),
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := GetBlobSidecars(tt.blockNumber); !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetBlobSidecars() = %v, want %v", got, tt.want)
}
})
}
}

0 comments on commit ef01dc5

Please sign in to comment.