Merge pull request #83 from xssnick/v1.4dev

V1.4

README.md

@@ -3,7 +3,7 @@
 <img align="right" width="425px" src="https://github.com/xssnick/props/blob/master/logoimg.png?raw=true">
 
 [![Based on TON][ton-svg]][ton]
-![Coverage](https://img.shields.io/badge/Coverage-70.7%25-brightgreen)
+![Coverage](https://img.shields.io/badge/Coverage-70.2%25-brightgreen)
 
 Golang library for interacting with TON blockchain.
 

example/block-scan/main.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"github.com/xssnick/tonutils-go/address"
 	"github.com/xssnick/tonutils-go/liteclient"
 	"github.com/xssnick/tonutils-go/tlb"
 	"github.com/xssnick/tonutils-go/ton"
 	"log"
+	"time"
 )
 
 func main() {
@@ -32,6 +34,9 @@ func main() {
 	// if it will go down, another lite server will be used
 	ctx := api.Client().StickyContext(context.Background())
 
+	// storage for last seen shard seqno
+	shardLastSeqno := map[string]uint32{}
+
 	for {
 		log.Printf("scanning %d master block...\n", master.SeqNo)
 
@@ -42,6 +47,32 @@ func main() {
 			return
 		}
 
+		// shards in master block may have holes, e.g. shard seqno 2756461, then 2756463, and no 2756462 in master chain
+		// thus we need to scan a bit back in case of discovering a hole, till last seen, to fill the misses.
+		var ext []*tlb.BlockInfo
+		for _, shard := range shards {
+			id := fmt.Sprintf("%d|%d", shard.Workchain, shard.Shard)
+			have, ok := shardLastSeqno[id]
+			if ok && have < shard.SeqNo-1 {
+				// find every shard from first missed till previous of new shard, and add them to scan list
+				for x := have + 1; x < shard.SeqNo; x++ {
+					for {
+						missed, err := api.LookupBlock(ctx, shard.Workchain, shard.Shard, x)
+						if err != nil {
+							log.Printf("lookupBlock old shards %d %d %d err: %v", shard.Workchain, shard.Shard, x, err.Error())
+							time.Sleep(1 * time.Second)
+							continue
+						}
+						log.Printf("discovered missed shard from chain %d %d %d", shard.Workchain, shard.Shard, x)
+						ext = append(ext, missed)
+						break
+					}
+				}
+			}
+			shardLastSeqno[id] = shard.SeqNo
+		}
+		shards = append(shards, ext...)
+
 		var txList []*tlb.Transaction
 
 		// for each shard block getting transactions

example/deploy-nft-collection/main.go

@@ -88,11 +88,13 @@ func getContractData(collectionOwnerAddr, royaltyAddr *address.Address) *cell.Ce
 		MustStoreAddr(royaltyAddr).
 		EndCell()
 
-	collectionContent := nft.ContentOffchain{URI: "https://tonutils.com"}
+	// collection data
+	collectionContent := nft.ContentOffchain{URI: "https://tonutils.com/collection.json"}
 	collectionContentCell, _ := collectionContent.ContentCell()
 
-	commonContent := nft.ContentOffchain{URI: "https://tonutils.com/nft/"}
-	commonContentCell, _ := commonContent.ContentCell()
+	// prefix for NFTs data
+	uri := "https://tonutils.com/nft/"
+	commonContentCell := cell.BeginCell().MustStoreStringSnake(uri).EndCell()
 
 	contentRef := cell.BeginCell().
 		MustStoreRef(collectionContentCell).

tlb/block.go

@@ -3,6 +3,7 @@ package tlb
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 
 	"github.com/xssnick/tonutils-go/tvm/cell"
 )
@@ -63,6 +64,7 @@ type McBlockExtra struct {
 }
 
 type BlockExtra struct {
+	_                  Magic         `tlb:"#4a33f6fd"`
 	InMsgDesc          *cell.Cell    `tlb:"^"`
 	OutMsgDesc         *cell.Cell    `tlb:"^"`
 	ShardAccountBlocks *cell.Cell    `tlb:"^"`
@@ -74,7 +76,7 @@ type BlockExtra struct {
 type Block struct {
 	_           Magic       `tlb:"#11ef55aa"`
 	GlobalID    int32       `tlb:"## 32"`
-	BlockInfo   *cell.Cell  `tlb:"^"`
+	BlockInfo   BlockHeader `tlb:"^"`
 	ValueFlow   *cell.Cell  `tlb:"^"`
 	StateUpdate StateUpdate `tlb:"^"`
 	Extra       *BlockExtra `tlb:"^"`
@@ -83,3 +85,213 @@ type Block struct {
 type AllShardsInfo struct {
 	ShardHashes *cell.Dictionary `tlb:"dict 32"`
 }
+
+type BlockHeader struct { // BlockInfo from block.tlb
+	blockInfoPart
+	GenSoftware *GlobalVersion
+	MasterRef   *ExtBlkRef
+	PrevRef     BlkPrevInfo
+	PrevVertRef *BlkPrevInfo
+}
+
+type blockInfoPart struct {
+	_                         Magic      `tlb:"#9bc7a987"`
+	Version                   uint32     `tlb:"## 32"`
+	NotMaster                 bool       `tlb:"bool"`
+	AfterMerge                bool       `tlb:"bool"`
+	BeforeSplit               bool       `tlb:"bool"`
+	AfterSplit                bool       `tlb:"bool"`
+	WantSplit                 bool       `tlb:"bool"`
+	WantMerge                 bool       `tlb:"bool"`
+	KeyBlock                  bool       `tlb:"bool"`
+	VertSeqnoIncr             bool       `tlb:"bool"`
+	Flags                     uint32     `tlb:"## 8"`
+	SeqNo                     uint32     `tlb:"## 32"`
+	VertSeqNo                 uint32     `tlb:"## 32"`
+	Shard                     ShardIdent `tlb:"."`
+	GenUtime                  uint32     `tlb:"## 32"`
+	StartLt                   uint64     `tlb:"## 64"`
+	EndLt                     uint64     `tlb:"## 64"`
+	GenValidatorListHashShort uint32     `tlb:"## 32"`
+	GenCatchainSeqno          uint32     `tlb:"## 32"`
+	MinRefMcSeqno             uint32     `tlb:"## 32"`
+	PrevKeyBlockSeqno         uint32     `tlb:"## 32"`
+}
+
+type ExtBlkRef struct {
+	EndLt    uint64 `tlb:"## 64"`
+	SeqNo    uint32 `tlb:"## 32"`
+	RootHash []byte `tlb:"bits 256"`
+	FileHash []byte `tlb:"bits 256"`
+}
+
+type GlobalVersion struct {
+	_            Magic  `tlb:"#c4"`
+	Version      uint32 `tlb:"## 32"`
+	Capabilities uint64 `tlb:"## 64"`
+}
+
+type BlkPrevInfo struct {
+	Prev1 ExtBlkRef
+	Prev2 *ExtBlkRef
+}
+
+func (h *BlockHeader) LoadFromCell(loader *cell.Slice) error {
+	var infoPart blockInfoPart
+	err := LoadFromCell(&infoPart, loader)
+	if err != nil {
+		return err
+	}
+	h.blockInfoPart = infoPart
+
+	if infoPart.Flags&1 == 1 {
+		var globalVer GlobalVersion
+		err = LoadFromCell(&globalVer, loader)
+		if err != nil {
+			return err
+		}
+		h.GenSoftware = &globalVer
+	}
+
+	if infoPart.NotMaster {
+		var masterRef ExtBlkRef
+		l, err := loader.LoadRef()
+		if err != nil {
+			return err
+		}
+		err = LoadFromCell(&masterRef, l)
+		if err != nil {
+			return err
+		}
+		h.MasterRef = &masterRef
+	}
+
+	l, err := loader.LoadRef()
+	if err != nil {
+		return err
+	}
+	prevRef, err := loadBlkPrevInfo(l, infoPart.AfterMerge)
+	if err != nil {
+		return err
+	}
+	h.PrevRef = *prevRef
+
+	if infoPart.VertSeqnoIncr {
+		l, err := loader.LoadRef()
+		if err != nil {
+			return err
+		}
+		prevVertRef, err := loadBlkPrevInfo(l, false)
+		if err != nil {
+			return err
+		}
+		h.PrevVertRef = prevVertRef
+	}
+	return nil
+}
+
+func loadBlkPrevInfo(loader *cell.Slice, afterMerge bool) (*BlkPrevInfo, error) {
+	var res BlkPrevInfo
+
+	if !afterMerge {
+		var blkRef ExtBlkRef
+		err := LoadFromCell(&blkRef, loader)
+		if err != nil {
+			return nil, err
+		}
+		res.Prev1 = blkRef
+		return &res, nil
+	}
+
+	var blkRef1, blkRef2 ExtBlkRef
+	prev1, err := loader.LoadRef()
+	if err != nil {
+		return nil, err
+	}
+	prev2, err := loader.LoadRef()
+	if err != nil {
+		return nil, err
+	}
+	err = LoadFromCell(&blkRef1, prev1)
+	if err != nil {
+		return nil, err
+	}
+	err = LoadFromCell(&blkRef2, prev2)
+	if err != nil {
+		return nil, err
+	}
+
+	res.Prev1 = blkRef1
+	res.Prev2 = &blkRef2
+	return &res, nil
+}
+
+func ConvertShardIdentToShard(si ShardIdent) (workchain int32, shard uint64) {
+	shard = si.ShardPrefix
+	pow2 := uint64(1) << (63 - si.PrefixBits)
+	shard |= pow2
+	return si.WorkchainID, shard
+}
+
+func shardChild(shard uint64, left bool) uint64 {
+	x := lowerBit64(shard) >> 1
+	if left {
+		return shard - x
+	}
+	return shard + x
+}
+
+func shardParent(shard uint64) uint64 {
+	x := lowerBit64(shard)
+	return (shard - x) | (x << 1)
+}
+
+func lowerBit64(x uint64) uint64 {
+	return x & bitsNegate64(x)
+}
+
+func bitsNegate64(x uint64) uint64 {
+	return ^x + 1
+}
+
+func (h *BlockHeader) GetParentBlocks() ([]*BlockInfo, error) {
+	var parents []*BlockInfo
+	workchain, shard := ConvertShardIdentToShard(h.Shard)
+
+	if !h.AfterMerge && !h.AfterSplit {
+		return []*BlockInfo{{
+			Workchain: workchain,
+			SeqNo:     h.PrevRef.Prev1.SeqNo,
+			RootHash:  h.PrevRef.Prev1.RootHash,
+			FileHash:  h.PrevRef.Prev1.FileHash,
+			Shard:     int64(shard),
+		}}, nil
+	} else if !h.AfterMerge && h.AfterSplit {
+		return []*BlockInfo{{
+			Workchain: workchain,
+			SeqNo:     h.PrevRef.Prev1.SeqNo,
+			RootHash:  h.PrevRef.Prev1.RootHash,
+			FileHash:  h.PrevRef.Prev1.FileHash,
+			Shard:     int64(shardParent(shard)),
+		}}, nil
+	}
+
+	if h.PrevRef.Prev2 == nil {
+		return nil, fmt.Errorf("must be 2 parent blocks after merge")
+	}
+	parents = append(parents, &BlockInfo{
+		Workchain: workchain,
+		SeqNo:     h.PrevRef.Prev1.SeqNo,
+		RootHash:  h.PrevRef.Prev1.RootHash,
+		FileHash:  h.PrevRef.Prev1.FileHash,
+		Shard:     int64(shardChild(shard, true)),
+	})
+	parents = append(parents, &BlockInfo{
+		Workchain: workchain,
+		SeqNo:     h.PrevRef.Prev2.SeqNo,
+		RootHash:  h.PrevRef.Prev2.RootHash,
+		FileHash:  h.PrevRef.Prev2.FileHash,
+		Shard:     int64(shardChild(shard, false)),
+	})
+	return parents, nil
+}

tlb/loader.go

@@ -211,7 +211,7 @@ func LoadFromCell(v any, loader *cell.Slice) error {
 			}
 
 			if ldMagic != uint64(magic) {
-				return fmt.Errorf("magic is not correct")
+				return fmt.Errorf("magic is not correct for %s, want %x, got %x", rv.Type().String(), magic, ldMagic)
 			}
 			continue
 		} else if settings[0] == "dict" {