From ef01dc505966e732c1d41302f39222921a4166ca Mon Sep 17 00:00:00 2001 From: blxdyx Date: Thu, 22 Aug 2024 20:33:13 +0800 Subject: [PATCH] fix snapshots_cmd --- .../freezeblocks/bsc_snapshots.go | 152 ++++++++++++------ turbo/snapshotsync/freezeblocks/utill.go | 27 ++-- turbo/snapshotsync/freezeblocks/utill_test.go | 29 ++++ 3 files changed, 147 insertions(+), 61 deletions(-) create mode 100644 turbo/snapshotsync/freezeblocks/utill_test.go diff --git a/turbo/snapshotsync/freezeblocks/bsc_snapshots.go b/turbo/snapshotsync/freezeblocks/bsc_snapshots.go index 887116154f9..2bccb89b813 100644 --- a/turbo/snapshotsync/freezeblocks/bsc_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/bsc_snapshots.go @@ -279,58 +279,114 @@ func (s *BscRoSnapshots) ReadBlobSidecars(blockNum uint64) ([]*types.BlobSidecar return sidecars, nil } +var missedBlobs = []uint64{ + 39565743, + 39679711, + 39679823, + 39688495, + 39712447, + 39712471, + 39712559, + 39713015, + 39716655, + 39716807, + 40180114, + 40180115, + 40208199, + 40211855, + 40211856, + 40212934, + 40212939, + 40227488, + 40231835, + 40232104, + 40239458, + 40297217, + 40314598, + 40315505, + 40316006, + 40316698, + 40319042, + 40320395, + 40322006, + 40322007, + 40693732, + 40899197, + 41756117, + 41756119, + 41756123, + 41756126, + 41756130, + 41756132, + 41756134, + 41756135, + 41756138, + 41756142, + 41756145, + 41756147, + 41756149, + 41756152, + 41756156, + 41756160, + 41756163, + 41756164, + 41756165, + 41756169, + 41756172, +} + func checkBlobs(ctx context.Context, blockFrom, blockTo uint64, chainDB kv.RoDB, blobStore services.BlobStorage, blockReader services.FullBlockReader, logger log.Logger) bool { tx, err := chainDB.BeginRo(ctx) if err != nil { return false } defer tx.Rollback() - var missedBlobs []uint64 - noErr := true - for i := blockFrom; i < blockTo; i++ { - block, err := blockReader.BlockByNumber(ctx, tx, i) - if err != nil { - log.Error("ReadCanonicalHash", "blockNum", i, "blockHash", block.Hash(), "err", err) - noErr = false - } - var blobTxCount uint64 - - for _, tx := range block.Transactions() { - if tx.Type() != types.BlobTxType { - continue - } - blobTxCount++ - } - if blobTxCount == 0 { - continue - } - blobs, found, err := blobStore.ReadBlobSidecars(ctx, i, block.Hash()) - if err != nil { - noErr = false - missedBlobs = append(missedBlobs, i) - log.Error("read blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err) - err := blobStore.RemoveBlobSidecars(ctx, i, block.Hash()) - log.Error("Remove blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err) - continue - } - if !found { - noErr = false - missedBlobs = append(missedBlobs, i) - log.Error("blob sidecars not found for block ", "blockNumber", i, "count", blobTxCount) - continue - } - - if uint64(len(blobs)) != blobTxCount { - missedBlobs = append(missedBlobs, i) - noErr = false - log.Error("blob sidecars not found for block ", "blockNumber", i, "want", blobTxCount, "actual", len(blobs)) - continue - } - - if i%20_000 == 0 { - logger.Info("Dumping beacon blobs", "progress", i) - } - } + //var missedBlobs []uint64 + //noErr := true + //for i := blockFrom; i < blockTo; i++ { + // block, err := blockReader.BlockByNumber(ctx, tx, i) + // if err != nil { + // log.Error("ReadCanonicalHash", "blockNum", i, "blockHash", block.Hash(), "err", err) + // noErr = false + // } + // var blobTxCount uint64 + // + // for _, tx := range block.Transactions() { + // if tx.Type() != types.BlobTxType { + // continue + // } + // blobTxCount++ + // } + // if blobTxCount == 0 { + // continue + // } + // blobs, found, err := blobStore.ReadBlobSidecars(ctx, i, block.Hash()) + // if err != nil { + // noErr = false + // missedBlobs = append(missedBlobs, i) + // log.Error("read blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err) + // err := blobStore.RemoveBlobSidecars(ctx, i, block.Hash()) + // log.Error("Remove blob sidecars:", "blockNum", i, "blobTxCount", blobTxCount, "err", err) + // continue + // } + // if !found { + // noErr = false + // missedBlobs = append(missedBlobs, i) + // log.Error("blob sidecars not found for block ", "blockNumber", i, "count", blobTxCount) + // continue + // } + // + // if uint64(len(blobs)) != blobTxCount { + // missedBlobs = append(missedBlobs, i) + // noErr = false + // log.Error("blob sidecars not found for block ", "blockNumber", i, "want", blobTxCount, "actual", len(blobs)) + // continue + // } + // + // if i%20_000 == 0 { + // logger.Info("Dumping beacon blobs", "progress", i) + // } + //} log.Info("Start query missedBlobs from http") for _, num := range missedBlobs { @@ -338,7 +394,7 @@ func checkBlobs(ctx context.Context, blockFrom, blockTo uint64, chainDB kv.RoDB, hash, err := blockReader.CanonicalHash(ctx, tx, num) if err != nil { log.Error("GetBlobSidecars failed", "num", num, "err", err) - return noErr + return false } if err = blobStore.WriteBlobSidecars(ctx, hash, blobs); err != nil { log.Error("WriteBlobSidecars failed", "num", num, "err", err) @@ -346,5 +402,5 @@ func checkBlobs(ctx context.Context, blockFrom, blockTo uint64, chainDB kv.RoDB, time.Sleep(1 * time.Second) } - return noErr + return false } diff --git a/turbo/snapshotsync/freezeblocks/utill.go b/turbo/snapshotsync/freezeblocks/utill.go index fef96628780..e216a674d6e 100644 --- a/turbo/snapshotsync/freezeblocks/utill.go +++ b/turbo/snapshotsync/freezeblocks/utill.go @@ -8,7 +8,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutil" "github.com/ledgerwatch/erigon/core/types" "io/ioutil" - "math/big" "net/http" ) @@ -24,16 +23,16 @@ type RPCRequest struct { } type BlockResponse struct { - Jsonrpc string `json:"jsonrpc"` - Id int `json:"id"` - Result interface{} `json:"result"` - Error interface{} `json:"error"` + Jsonrpc string `json:"jsonrpc"` + Id int `json:"id"` + Result json.RawMessage `json:"result"` + Error interface{} `json:"error"` } type BlobResponse struct { BlobTxSidecar types.BlobTxSidecar `json:"blobSidecar"` - BlockNumber *big.Int `json:"blockNumber"` + BlockNumber string `json:"blockNumber"` BlockHash libcommon.Hash `json:"blockHash"` - TxIndex uint64 `json:"txIndex"` + TxIndex string `json:"txIndex"` TxHash libcommon.Hash `json:"txHash"` } @@ -68,7 +67,7 @@ func GetBlobSidecars(blockNumber uint64) types.BlobSidecars { var blockResponse BlockResponse err = json.Unmarshal(responseBody, &blockResponse) if err != nil { - fmt.Println("Error unmarshalling response:", err) + fmt.Println("Error unmarshalling response a:", err) return nil } @@ -77,20 +76,22 @@ func GetBlobSidecars(blockNumber uint64) types.BlobSidecars { return nil } - var blobResponse []BlobResponse - err = json.Unmarshal(responseBody, &blobResponse) + var blobResponse []*BlobResponse + err = json.Unmarshal(blockResponse.Result, &blobResponse) if err != nil { - fmt.Println("Error unmarshalling response:", err) + fmt.Println("Error unmarshalling response b:", err) return nil } var blobSidecars types.BlobSidecars for _, blobSidecar := range blobResponse { + bn, _ := hexutil.DecodeBig(blobSidecar.BlockNumber) + tx, _ := hexutil.DecodeUint64(blobSidecar.TxIndex) blob := &types.BlobSidecar{ BlobTxSidecar: blobSidecar.BlobTxSidecar, - BlockNumber: blobSidecar.BlockNumber, + BlockNumber: bn, BlockHash: blobSidecar.BlockHash, - TxIndex: blobSidecar.TxIndex, + TxIndex: tx, TxHash: blobSidecar.TxHash, } blobSidecars = append(blobSidecars, blob) diff --git a/turbo/snapshotsync/freezeblocks/utill_test.go b/turbo/snapshotsync/freezeblocks/utill_test.go new file mode 100644 index 00000000000..14ec5b03a0b --- /dev/null +++ b/turbo/snapshotsync/freezeblocks/utill_test.go @@ -0,0 +1,29 @@ +package freezeblocks + +import ( + "github.com/ledgerwatch/erigon/core/types" + "reflect" + "testing" +) + +func TestGetBlobSidecars(t *testing.T) { + + tests := []struct { + name string + blockNumber uint64 + want types.BlobSidecars + }{ + { + name: "test1", + blockNumber: uint64(39565743), + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetBlobSidecars(tt.blockNumber); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetBlobSidecars() = %v, want %v", got, tt.want) + } + }) + } +}