From 7523a7052287b23ecbc6554124caf763525cf5d9 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Wed, 28 Aug 2024 20:45:01 +0300 Subject: [PATCH 1/9] pruned cells and merkle proof support --- boc/cell.go | 32 ++++++++++++++++++++++++++++++++ liteapi/client.go | 3 +++ tlb/bintree.go | 3 +++ tlb/decoder.go | 6 ++++++ tlb/proof.go | 28 ++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+) diff --git a/boc/cell.go b/boc/cell.go index cb7e06fb..ddc82444 100644 --- a/boc/cell.go +++ b/boc/cell.go @@ -436,5 +436,37 @@ func (c *Cell) GetMerkleRoot() ([32]byte, error) { var hash [32]byte copy(hash[:], bytes[1:]) return hash, nil +} + +// TODO: move to deserializer +func (c *Cell) isValidMerkleProofCell() bool { + return c.cellType == MerkleProofCell && c.RefsSize() == 1 && c.BitSize() == 280 +} + +func (c *Cell) CalculateMerkleProofMeta() (int, [32]byte, error) { + if !c.isValidMerkleProofCell() { + return 0, [32]byte{}, errors.New("not valid merkle proof cell") + } + imc, err := newImmutableCell(c.Refs()[0], map[*Cell]*immutableCell{}) + if err != nil { + return 0, [32]byte{}, fmt.Errorf("get immutable cell: %w", err) + } + h := imc.Hash(0) + var hash [32]byte + copy(hash[:], h) + depth := imc.Depth(0) + return depth, hash, nil +} +// TODO: or add level as optional parameter to Hash256() +func (c *Cell) Hash256WithLevel(level int) ([32]byte, error) { + // TODO: or check for pruned cell and read hash directly from cell + imc, err := newImmutableCell(c, map[*Cell]*immutableCell{}) + if err != nil { + return [32]byte{}, err + } + b := imc.Hash(level) + var h [32]byte + copy(h[:], b) + return h, nil } diff --git a/liteapi/client.go b/liteapi/client.go index 3d101427..1fb3f4db 100644 --- a/liteapi/client.go +++ b/liteapi/client.go @@ -571,6 +571,9 @@ func (c *Client) GetAccountStateRaw(ctx context.Context, accountID ton.AccountID if err != nil { return liteclient.LiteServerAccountStateC{}, err } + if res.Id.ToBlockIdExt() != blockID { + return liteclient.LiteServerAccountStateC{}, errors.New("invalid block ID") + } return res, nil } diff --git a/tlb/bintree.go b/tlb/bintree.go index 183f37ab..0f547682 100644 --- a/tlb/bintree.go +++ b/tlb/bintree.go @@ -54,6 +54,9 @@ func (b *BinTree[T]) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error { } b.Values = make([]T, 0, len(dec)) for _, i := range dec { + if i.CellType() == boc.PrunedBranchCell { + continue + } var t T err := decoder.Unmarshal(i, &t) if err != nil { diff --git a/tlb/decoder.go b/tlb/decoder.go index d3812629..85e215c2 100644 --- a/tlb/decoder.go +++ b/tlb/decoder.go @@ -116,6 +116,9 @@ func decode(c *boc.Cell, tag string, val reflect.Value, decoder *Decoder) error return fmt.Errorf("library cell as a ref is not implemented") } if c.CellType() == boc.PrunedBranchCell { + if val.Kind() == reflect.Struct && val.Type() == bocCellType { + return decodeCell(c, val) + } return nil } case t.IsMaybe: @@ -137,6 +140,9 @@ func decode(c *boc.Cell, tag string, val reflect.Value, decoder *Decoder) error return fmt.Errorf("library cell as a ref is not implemented") } if c.CellType() == boc.PrunedBranchCell { + if val.Kind() == reflect.Struct && val.Type() == bocCellType { + return decodeCell(c, val) + } return nil } } diff --git a/tlb/proof.go b/tlb/proof.go index 554c0ee7..225b32c1 100644 --- a/tlb/proof.go +++ b/tlb/proof.go @@ -1,6 +1,7 @@ package tlb import ( + "errors" "fmt" "github.com/tonkeeper/tongo/boc" @@ -15,6 +16,33 @@ type MerkleProof[T any] struct { VirtualRoot T `tlb:"^"` } +func (p *MerkleProof[T]) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error { + depth, hash, err := c.CalculateMerkleProofMeta() + if err != nil { + return err + } + // TODO: remove duplicates + type merkleProof[T any] struct { + Magic Magic `tlb:"!merkle_proof#03"` + VirtualHash Bits256 + Depth uint16 + VirtualRoot T `tlb:"^"` + } + var res merkleProof[T] + err = decoder.Unmarshal(c, &res) + if err != nil { + return err + } + if res.VirtualHash != hash { + return errors.New("invalid virtual hash") + } + if int(res.Depth) != depth { + return errors.New("invalid depth") + } + *p = MerkleProof[T](res) + return nil +} + type MerkleUpdate[T any] struct { Magic Magic `tlb:"!merkle_update#04"` FromHash Bits256 From caaa3930dd15ebd9f5353317fb19e7339e48b256 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Wed, 28 Aug 2024 20:46:23 +0300 Subject: [PATCH 2/9] get account state with proofs --- liteapi/account.go | 213 ++++++++++++++++++++++++++++++++++++++++ liteapi/account_test.go | 46 +++++++++ 2 files changed, 259 insertions(+) create mode 100644 liteapi/account.go create mode 100644 liteapi/account_test.go diff --git a/liteapi/account.go b/liteapi/account.go new file mode 100644 index 00000000..b3545d38 --- /dev/null +++ b/liteapi/account.go @@ -0,0 +1,213 @@ +package liteapi + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/tonkeeper/tongo/boc" + "github.com/tonkeeper/tongo/tlb" + "github.com/tonkeeper/tongo/ton" +) + +// blockTrimmed stripped-down version of the tlb.Block with pruned cell instead of ShardState and skip some decoding +type blockTrimmed struct { + Magic tlb.Magic `tlb:"block#11ef55aa"` + GlobalId int32 + Info boc.Cell `tlb:"^"` + ValueFlow boc.Cell `tlb:"^"` + StateUpdate tlb.MerkleUpdate[boc.Cell] `tlb:"^"` + Extra boc.Cell `tlb:"^"` +} + +// shardAccountPruned stripped-down version of the tlb.ShardAccount with pruned cell instead of Account +type shardAccountPruned struct { + Account boc.Cell `tlb:"^"` + LastTransHash tlb.Bits256 + LastTransLt uint64 +} + +// shardStateUnsplitTrimmed stripped-down version of the ShardStateUnsplit structure for extracting Account proof +type shardStateUnsplitTrimmed struct { + Magic tlb.Magic `tlb:"shard_state#9023afe2"` + GlobalID int32 + ShardID tlb.ShardIdent + SeqNo uint32 + VertSeqNo uint32 + GenUtime uint32 + GenLt uint64 + MinRefMcSeqno uint32 + OutMsgQueueInfo boc.Cell `tlb:"^"` + BeforeSplit bool + Accounts tlb.HashmapAugE[tlb.Bits256, shardAccountPruned, tlb.DepthBalanceInfo] `tlb:"^"` + Other tlb.ShardStateUnsplitOther `tlb:"^"` + Custom tlb.Maybe[tlb.Ref[boc.Cell]] +} + +// TODO: use proof merger instead of trimmed +type shardStateUnsplit struct { + ShardStateTrimmed shardStateUnsplitTrimmed + ShardState tlb.ShardStateUnsplit +} + +// GetAccountWithProof +// For safe operation, always use GetAccountWithProof with WithBlock(proofedBlock ton.BlockIDExt), as the proof of masterchain cashed blocks is not implemented yet! +func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountID) (*tlb.ShardAccount, *tlb.ShardStateUnsplit, error) { // TODO: return merged tlb.ShardStateUnsplit (proof+account) + res, err := c.GetAccountStateRaw(ctx, accountID) // TODO: add proof check for masterHead + if err != nil { + return nil, nil, err + } + blockID := res.Id.ToBlockIdExt() + if len(res.Proof) == 0 { + return nil, nil, errors.New("empty proof") + } + var shardHash *ton.Bits256 + if accountID.Workchain != -1 { // TODO: set masterchain constant + if len(res.ShardProof) == 0 { + return nil, nil, errors.New("empty shard proof") + } // TODO: change logic for shard blockID (proof shard to proofed master) + if res.Shardblk.RootHash == [32]byte{} { // TODO: how to check for empty shard? + return nil, nil, errors.New("shard block not passed") + } + h := ton.Bits256(res.Shardblk.RootHash) + shardHash = &h + } + blockHash := blockID.RootHash + if shardHash != nil { // we need shard proof only for not masterchain + if err := checkShardInMasterProof(blockID, res.ShardProof, accountID.Workchain, *shardHash); err != nil { + return nil, nil, fmt.Errorf("shard proof is incorrect: %w", err) + } + blockHash = *shardHash + } + shardState, err := checkBlockShardStateProof(res.Proof, blockHash) + if err != nil { + return nil, nil, fmt.Errorf("incorrect block proof: %w", err) + } + values := shardState.ShardStateTrimmed.Accounts.Values() + keys := shardState.ShardStateTrimmed.Accounts.Keys() + for i, k := range keys { + if bytes.Equal(k[:], accountID.Address[:]) { + acc, err := decodeAccount(res.State, values[i]) + if err != nil { + return nil, nil, err + } + return acc, &shardState.ShardState, nil + } + } + if len(res.State) == 0 { + return &tlb.ShardAccount{Account: tlb.Account{SumType: "AccountNone"}}, &shardState.ShardState, nil + } + return nil, nil, errors.New("invalid account state") +} + +func decodeAccount(state []byte, shardAccount shardAccountPruned) (*tlb.ShardAccount, error) { + stateCells, err := boc.DeserializeBoc(state) + if err != nil { + return nil, err + } + if len(stateCells) != 1 { + return nil, boc.ErrNotSingleRoot + } + accountHash, err := stateCells[0].Hash256() + if err != nil { + return nil, err + } + shardAccountHash, err := shardAccount.Account.Hash256WithLevel(0) + if err != nil { + return nil, err + } + if accountHash != shardAccountHash { + return nil, errors.New("invalid account hash") + } + var acc tlb.Account + err = tlb.Unmarshal(stateCells[0], &acc) + if err != nil { + return nil, err + } + // do not check account balance from tlb.DepthBalanceInfo + res := tlb.ShardAccount{Account: acc, LastTransHash: shardAccount.LastTransHash, LastTransLt: shardAccount.LastTransLt} + return &res, nil +} + +func checkShardInMasterProof(master ton.BlockIDExt, shardProof []byte, workchain int32, shardRootHash ton.Bits256) error { + shardState, err := checkBlockShardStateProof(shardProof, master.RootHash) + if err != nil { + return fmt.Errorf("check block proof failed: %w", err) + } + if !shardState.ShardState.ShardStateUnsplit.Custom.Exists { + return fmt.Errorf("not a masterchain block") + } + stateExtra := shardState.ShardState.ShardStateUnsplit.Custom.Value.Value + keys := stateExtra.ShardHashes.Keys() + values := stateExtra.ShardHashes.Values() + for i, k := range keys { + binTreeValues := values[i].Value.BinTree.Values + for _, b := range binTreeValues { + switch b.SumType { + case "Old": + if int32(k) == workchain && ton.Bits256(b.Old.RootHash) == shardRootHash { + return nil + } + case "New": + if int32(k) == workchain && ton.Bits256(b.New.RootHash) == shardRootHash { + return nil + } + } + } + } + return fmt.Errorf("required shard hash not found in proof") +} + +func checkBlockShardStateProof(proof []byte, blockRootHash ton.Bits256) (*shardStateUnsplit, error) { + proofCells, err := boc.DeserializeBoc(proof) + if err != nil { + return nil, err + } + if len(proofCells) != 2 { + return nil, errors.New("must be two root cells") + } + block, err := checkBlockProof(proofCells[0], blockRootHash) + if err != nil { + return nil, fmt.Errorf("incorrect block proof: %w", err) + } + var stateTrimmedProof struct { + Proof tlb.MerkleProof[shardStateUnsplitTrimmed] + } + err = tlb.Unmarshal(proofCells[1], &stateTrimmedProof) // cells order must be strictly defined + if err != nil { + return nil, err + } + proofCells[1].ResetCounters() + var stateProof struct { + Proof tlb.MerkleProof[tlb.ShardStateUnsplit] + } + err = tlb.Unmarshal(proofCells[1], &stateProof) + if err != nil { + return nil, err + } + toRootHash, err := block.StateUpdate.ToRoot.Hash256WithLevel(0) + if err != nil { + return nil, err + } + if stateTrimmedProof.Proof.VirtualHash != toRootHash { + return nil, errors.New("invalid virtual hash") + } + res := shardStateUnsplit{ + ShardStateTrimmed: stateTrimmedProof.Proof.VirtualRoot, + ShardState: stateProof.Proof.VirtualRoot, + } + return &res, nil +} + +func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*blockTrimmed, error) { + var res tlb.MerkleProof[blockTrimmed] + err := tlb.Unmarshal(proof, &res) // merkle hash and depth checks inside + if err != nil { + return nil, fmt.Errorf("failed to unmarshal block proof: %w", err) + } + if ton.Bits256(res.VirtualHash) != blockRootHash { + return nil, fmt.Errorf("invalid block root hash") + } + block := res.VirtualRoot + return &block, nil +} diff --git a/liteapi/account_test.go b/liteapi/account_test.go new file mode 100644 index 00000000..485ab025 --- /dev/null +++ b/liteapi/account_test.go @@ -0,0 +1,46 @@ +package liteapi + +import ( + "context" + "fmt" + "github.com/tonkeeper/tongo/ton" + "testing" +) + +func TestGetAccountWithProof(t *testing.T) { + api, err := NewClient(Testnet(), FromEnvs()) + if err != nil { + t.Fatal(err) + } + testCases := []struct { + name string + accountID string + }{ + { + name: "account from masterchain", + accountID: "-1:34517c7bdf5187c55af4f8b61fdc321588c7ab768dee24b006df29106458d7cf", + }, + { + name: "active account from basechain", + accountID: "0:e33ed33a42eb2032059f97d90c706f8400bb256d32139ca707f1564ad699c7dd", + }, + { + name: "nonexisted from basechain", + accountID: "0:5f00decb7da51881764dc3959cec60609045f6ca1b89e646bde49d492705d77c", + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + accountID, err := ton.AccountIDFromRaw(tt.accountID) + if err != nil { + t.Fatal("AccountIDFromRaw() failed: %w", err) + } + acc, st, err := api.GetAccountWithProof(context.TODO(), accountID) + if err != nil { + t.Fatal(err) + } + fmt.Printf("Account status: %v\n", acc.Account.Status()) + fmt.Printf("Last proof utime: %v\n", st.ShardStateUnsplit.GenUtime) + }) + } +} From b87536e88afb7fd6797854c5bce7b22cd9ef1795 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Mon, 2 Sep 2024 15:29:28 +0300 Subject: [PATCH 3/9] proof resolver --- boc/cell.go | 37 +++++++++++++++++++ liteapi/account_test.go | 79 +++++++++++++++++++++++++++++++++++++++++ tlb/bintree.go | 6 +++- tlb/decoder.go | 64 +++++++++++++++++++++++++++++---- tlb/hashmap.go | 12 +++++-- tlb/primitives.go | 10 ++++-- tlb/proof.go | 20 +++++++++-- 7 files changed, 214 insertions(+), 14 deletions(-) diff --git a/boc/cell.go b/boc/cell.go index ddc82444..447e2c14 100644 --- a/boc/cell.go +++ b/boc/cell.go @@ -470,3 +470,40 @@ func (c *Cell) Hash256WithLevel(level int) ([32]byte, error) { copy(h[:], b) return h, nil } + +// NonPrunedCells returns a map of all non-pruned cells, where the key is the hash of the cell. It can be used to resolve cells in the proofs. +func (c *Cell) NonPrunedCells() (map[[32]byte]*Cell, error) { + // TODO: mutable cell map may change during resolving. It may be necessary to make full copies of cells. + res := map[[32]byte]*Cell{} + if c.CellType() == PrunedBranchCell { + return res, nil + } + h, err := c.Hash256() + if err != nil { + return nil, err + } + res[h] = c + err = collectCells(c, res) + if err != nil { + return nil, err + } + return res, nil +} + +func collectCells(c *Cell, m map[[32]byte]*Cell) error { + refs := c.refs + for _, r := range refs { + if r != nil && r.CellType() != PrunedBranchCell { + h, err := r.Hash256() + if err != nil { + return err + } + m[h] = c + err = collectCells(r, m) + if err != nil { + return err + } + } + } + return nil +} diff --git a/liteapi/account_test.go b/liteapi/account_test.go index 485ab025..259c5332 100644 --- a/liteapi/account_test.go +++ b/liteapi/account_test.go @@ -1,8 +1,13 @@ package liteapi import ( + "bytes" "context" + "encoding/base64" + "errors" "fmt" + "github.com/tonkeeper/tongo/boc" + "github.com/tonkeeper/tongo/tlb" "github.com/tonkeeper/tongo/ton" "testing" ) @@ -44,3 +49,77 @@ func TestGetAccountWithProof(t *testing.T) { }) } } + +func TestUnmarshallingProofWithPrunedResolver(t *testing.T) { + testCases := []struct { + name string + accountID string + state string + proof string + }{ + { + name: "account from masterchain", + accountID: "-1:34517c7bdf5187c55af4f8b61fdc321588c7ab768dee24b006df29106458d7cf", + state: "te6ccgEBAQEANwAAac/zRRfHvfUYfFWvT4th/cMhWIx6t2je4ksAbfKRBkWNfPICV8MiQ7WQAAAnwcjbgQjD0JAE", + proof: "te6ccgECMQIABnwBAAlGAzm5ngf8wRtgCPSbEv1KYCOfL3YI9/HjNbeRsayNPbNBAcQCCUYDNjyZxQ6TS+uioSqhEmArXFMzcJ0iBgOO8gRScN1HDdQAFykkW5Ajr+L////9AP////8AAAAAAAAAAAFZWXUAAAAAZtWqnAAAFybEzYeEAVlZcmADBAUGKEgBAaHRVuPHjzLUmEYd44x/vzWcCD0Yz14taK8lFYkyYi16AAEiE4IRdMqOqEN5qjAHJyIzAAAAAAAAAAD//////////4RdMqOqEN5qiCgnKChIAQGSq4StFUmWS1wEONBSEQt3Wuup9Nhbdrp5fmk6oPx7cgAbIxMBCLplR1QhvNUYCAknIxMBCLADRttzmtl4CgsnKEgBASOab6P8maV1G2OhFqlTgNjoroG1i5MO5qtsoOqgY1ViAcAjEwEIqC8wZsH7ingMDScoSAEB6CVtOp/z9Kpj+SrDgasFP3kzGT7kZL/D7DV+kXZw3p4BwChIAQFIWAk+xGmmFk4X6v0lmgWpnk4YjEI/Tam9kXRPNLTjdgEPIhEA8MDW7kYtTOgODyhIAQFvFG13TDMvzWbO+0LVJbC2JHWAiHGL4nE6ztkqqPbefQENIhEA8BRymICH9QgQESIRAPAErnsrX9VoEhMoSAEB3EtPbrrXUOrrKC6PIjfYhj6sGE/sozDHyHhOqJco3MEAHShIAQEzaqPXOwDDYGRMpiB4f56Ep+oa0eJgHewJ3AzRahxDIgAaIhEA4GnDcXrbdIgUFSIRAOBpsy/ekzFIFhcoSAEBNEqqYfkIiFNMQLEWcnTl6stVZGy5ErSafIxVo9y43QYAGCINAKISMHbNyBgZKEgBAb3/C7yo+BlD9hObGRxmV/KM5e4Dx2dwvpYwbKHtaWxnABciDQChOnTEp4gaGyhIAQGefqFE75DubH/tAm07waI/ZJXXaUbA9TAjnJiXm5bFCQAYKEgBAWXR2cYWGVGOhRr0zx2IbfGvk/PYc1y+oSjg/RG7M23AABAiDQCgymlW3sgcHSINAKBa4jVqKB4fKEgBAScXOibV9zcakxfRicgWyCdTJFdgVU+FKgRvmF+aGkzaAAooSAEBgCBa9skgBATVk+IARArXssVqC0fkYdTeesmbGCNG2QIAEyIJAGHoSAggISIJAGHoSAgiIyhIAQG0d/c3vjpwCZ/skE9GxWaQW2p12p94naEbq+nI8BwxkAALIgkAYehICCQlKEgBAbcMTUahqWzuFpxMXlh6PQ1nbe5GZl+2H6cojgVpqdJGAAohl7xvj3vqMPirXp8Ww/uGQrEY9W7RvcSWANvlIgyLGvngMPQkBwpuh9/a8Q8liD7J/k6cDctwlpd7/wLfSj2ZkV3vQyWYAABPg5G3AgwmKEgBAdYv7OF3BCOejG/QrCN6d0U1gfBqoWUQPs3lCtkW0bqjAAgoSAEBfNH4EBmmQQpGeuMDQrdiJrcg1/foaCtvYt8A2eULqDYAAChIAQGyDjajs2pM3uYBEGxkLpBxiwpY2vIAdT27MYn5VrSUtgABKEgBAfXuY7WxrmX661SjWg783A5GU3G2ZUuk96jxo07FszvmAcAkEBHvVar////9KissLQGgm8ephwAAAAAEAQFZWXUAAAAAAP////8AAAAAAAAAAGbVqpwAABcmxM2HgAAAFybEzYeEjIe6qAAErqgBWVlyAVlQJsQAAAAIAAAAAAAAAe4uKEgBASkmYQFN/IwKZw+6jDvG7Hla0bypRJASLgCSLllgZYYxAAMqigQUxb9ElcbqpVZzQQGVtjJWkzZu/gqQV6cRwEqnJT7ljzm5ngf8wRtgCPSbEv1KYCOfL3YI9/HjNbeRsayNPbNBAcQBxC8wKEgBAfZANQckARh74l3KoHg6MIoIlCtXCklSokH5oFnYkvWaAAcAmAAAFybEvkVEAVlZdIw41mHg1tiTlZWmUEC5Zs1iJSaJiU/PG7sL/HsqfBj+zYXmULmtzn4TRGwnVVC5tKAhaIUDbFZrLZ+xVZ8cOhpojAEDFMW/RJXG6qVWc0EBlbYyVpM2bv4KkFenEcBKpyU+5Y+0LNwg0RHTx+GvVrTHWlXSAsJOr1Re1+VF1o0FxmRgmwHEABVojAEDObmeB/zBG2AI9JsS/UpgI58vdgj38eM1t5GxrI09s0EncQaO3Qwlxbnasj2PyljXoXXcs0VfOqaRU3MLD/XjOwHEABU=", + }, + { + name: "active account from basechain", + accountID: "0:e33ed33a42eb2032059f97d90c706f8400bb256d32139ca707f1564ad699c7dd", + state: "te6ccgECRgEACUQAAnPADjPtM6QusgMgWfl9kMcG+EALslbTITnKcH8VZK1pnH3SjJBKAzalNagAAFyBA05ODYC7pLkMcNNAAQIBFP8A9KQT9LzyyAsDAgAdHgIBYgQFAgLMBgcCASAXGAIBIAgJAgEgExQCASAKCwIBIA8QAW1CDHAJSED/Lw3gHQ0wMBcbCSXwPg+kAwAdMf7UTQ1NQwMSLAAOMCECRfBIIQNw/sUbrchA/y8IDAIBIA0OANAy+CMgghBi5EBpvPLgxwHwBCDXSSDCGPLgyCCBA/C78uDJIHipCMAA8uDKIfAF8uDLWPAHFL7y4Mwi+QGAUPgzIG6zjhDQ9AQwUhCDB/QOb6Ex8tDNkTDiyFAEzxbJyFADzxYSzMnwDAANHDIywHJ0IAAzHCfAdMHAcAAILOUAqYIAt4S5jEgwADy0MmACASAREgIBIDQ1AE8yI4gIddJEtcYWc8WIddKIMAAILObAcAB8uDKAtQw0AKRMeLmMcnQgAH8cCHXSY41XLogs44uMALTByHALSPCALAkpvhSQLmwIsIvI8E6sLEiwmADwXsTsBKxsyCzlAKmCALeE97mbBK6gAgFYFRYAOdLPgFOBD4BbvADGRlgqxnizh9AWW15mZkwCB9gEAC0AcjL//gozxbJcCDIywET9AD0AMsAyYAAbPkAdMjLAhLKB8v/ydCACASAZGgIBIBscAAe4tdMYAB+6ej7UTQ1NQwMfAKcAHwC4ABu5Bb7UTQ1NQwMH/wAhKACdujDDAg10l4qQjAAPLgRiDXCgfAACHXScAIUhCwk1t4beAglQHTBzEB3iHwA1Ei1xgw+QGCALqTyMsPAYIBZ6PtQ9jPFskBkXiRcOISoAGABIAWh0dHBzOi8vZG5zLnRvbi5vcmcvY29sbGVjdGlvbi5qc29uART/APSkE/S88sgLHwIBYiAhAgLMIiMCASA8PQIBICQlAgFINjcCASAmJwIBWDQ1AgEgKCkADUcMjLAcnQgB9z4J28QAtDTAwFxsJJfBOD6QPpAMfoAMXHXIfoAMfoAMPAKJ7OOTl8FbCI0UjLHBfLhlQH6QNQwbXDIywf0AMn4I4IQYuRAaaGCCCeNAKkEIMIMkzCADN6BASyBAPBYqIAMqQSh+CMBoPACRHfwCRA1+CPwC+BTWccFGLCAqABE+kQwcLry4U2AD+I40EJtfC/pAMHAg+CVtgEBwgBDIywVQB88WUAX6AhXLahLLH8s/Im6zlFjPFwGRMuIByQH7AOApxwCRcJUJ0x9QquIh8Aj4IyG8JMAAjp40Ojo7jhY2Njc3N1E1xwXy4ZYQJRAkECP4I/AL4w7gMQ3TPyVusx+wkmwh4w0rLC0A/jAmgGmAZKmEUrC+8uGXghA7msoAUqChUnC8mTaCEDuaygAZoZM5CAXiIMIAjjKCEFV86iD4JRA5bXFwgBDIywVQB88WUAX6AhXLahLLH8s/Im6zlFjPFwGRMuIByQH7AJIwNuKAPCP4I6GhIMIAkxOgApEw4kR08AkQJPgj8AsA0jQ2U82hghA7msoAUhChUnC8mTaCEDuaygAWoZIwBeIgwgCON4IQNw/sUW1yKVE0VEdDcIAQyMsFUAfPFlAF+gIVy2oSyx/LPyJus5RYzxcBkTLiAckB+wAcoQuRMOJtVHdlVHdjLvALAgTIghBfzD0UUiC6jpUxNztTcscF8uGREJoQSRA4RwZAFQTgghAaC51RUiC6jhlbMjU1NzdRNccF8uGaA9QwQBUEUDP4I/AL4CGCEE6x8Pm64wI7IIIQRL6uQbrjAjgnghBO0UtlujEuLzAAiFs2Njg4UUfHBfLhmwTT/yDXSsIAB9DTBwHAAPLhnPQEMAeY1DBAFoMH9BeYMFAFgwf0WzDicMjLB/QAyRA1QBT4I/ALAf4wNjokbvLhnYBQ+DPQ9AQwUkCDB/QOb6Hy4Z/TByHAACLAAbHy4aAhwACOkSQQmxBoUXoQVxBGEFxDFEzdljAQOjlfB+IBwAGOMnCCEDcP7FFYbYEAoHCAEMjLBVAHzxZQBfoCFctqEssfyz8ibrOUWM8XAZEy4gHJAfsAkVviMQH+jno3+CNQBqGBAli8Bm4WsPLhniPQ10n4I/AHUpC+8uGXUXihghA7msoAoSDCAI4yECeCEE7RS2VYB21ycIAQyMsFUAfPFlAF+gIVy2oSyx/LPyJus5RYzxcBkTLiAckB+wCTMDU14vgjgQEsoPACRHfwCRBFEDQS+CPwC+BfBDMB8DUC+kAh8AH6QNIAMfoAghA7msoAHaEhlFMUoKHeItcLAcMAIJIFoZE14iDC//LhkiGOPoIQBRONkchQC88WUA3PFnEkSxRUSMBwgBDIywVQB88WUAX6AhXLahLLH8s/Im6zlFjPFwGRMuIByQH7ABBplBAsOVviATIAio41KPABghDVMnbbEDlGCW1xcIAQyMsFUAfPFlAF+gIVy2oSyx/LPyJus5RYzxcBkTLiAckB+wCTODQw4hBFEDQS+CPwCwCaMjU1ghAvyyaiuo46cIIQi3cXNQTIy/9QBc8WFEMwgEBwgBDIywVQB88WUAX6AhXLahLLH8s/Im6zlFjPFwGRMuIByQH7AOBfBIQP8vAAkwgwASWMIED6IBk4CDABZYwgQH0gDLgIMAGljCBAZCAKOAgwAeWMIEBLIAe4CDACJYwgQDIgBTgIMAJlDCAZHrgwAqTgDJ14HpxgAGkAasC8AYBghA7msoAqAGCEDuaygCoAoIQYuRAaaGCCCeNAKkEIMIVkVvgbBKWp1qAZKkE5IAIBIDg5AgEgOjsAIQgbpQwbXAg4ND6QPoA0z8wgABcyFADzxYB+gLLP8mAAUTtRNDT//pAINdJwgCffwH6QNTU9ATTPzAQVxBW4DBwbW1tbSQQVxBWgACsBsjL/1AFzxZQA88WzMz0AMs/ye1UgAgEgPj8CASBCQwATu7OfAKF18H8AiAICdEBBABCodPAKEEdfBwAMqVnwCmxxAA24/P8ApfA4AgEgREUAE7ZKXgFCBOvg+hAAx7RhhDrpJA8VIRgAHlwI3gFCBuvg+hpg4DgAHlwznoCGAHrhQPgAHlwzuEERxGYQXgM+BIg9yxH7ZN3ElkrRuga4eSQNwjVy83zFyqqxQ6L/+8QYABJmDwA8ADBg/oHt9CYPADA=", + proof: "te6ccgECPwIACDsBAAlGA13wIJUiI7PTg32Ejju+cdCEhP3rVfdUykAsuzXJC+XeAhoCCUYDjCFt8RcRp3CSKwob3sWjzrlRTNWeNVuLJlIfcVPa8CsAHTYjW5Ajr+L////9AgAAAADAAAAAAAAAAAFyPHcAAAAAZtWq5wAAFybGxRHCAVlZkCADBAUoSAEBiyK6Ydkff6GhmEvUf7v2ypzoO80QfF1X31CgOrDi9NkAASERgcZ7gbxJWKSQBgDXAAAAAAAAAAD//////////3Ge4G8SVikji03qk7ZraRAAAXJsa1z4QBWVmQQHydZ7bv6t6cGWTc3mk/vill/h79hpOKKfqDLKYVV2UIGVEYrOM3Wj2tBpeo0h3v1D9oVZi+yPKb3hs1ZHIKP4IhJsDjPcDeJKxSQHCChIAQFV/Gx1KlYcCS7JoHnYKwxWUvcHSieeSmxWO4uodXNNAAIXIhEA4LZ6enx+bWgJCiIRAOBaroXhVZgICwwoSAEBjCXGi3ABTEA16fNsFq6NoDhjFI0NMoPkohAjH+1qLLEB+CIRAOAksERHiOeoDQ4oSAEBKwFZZ4JbhmBFD0mdW0SfSTNG6kvDu3q3m/GIC+nY4zIB+SIPANc4OPbi8mgPEChIAQFxKVccovU7jHX+7gMJQXv9jCCrCiQGA6Y+QxKqTUBOhgH2KEgBAarJ0lCkaVdnW8kHI8DI6r+f2A/fTt8z22IAYXPQ809ZAG0iDwDQN70ViI0IERIoSAEBYKCiCBXVsTmcFm80Mdta68M1YYjRCTuZc93Triz7m9gBgyIPAMKMtn9VX0gTFCIPAMFcoYfNXGgVFihIAQGfX/bkZ1jS0/86yXx7DUfHatLji8tg3KF1vqPCk7IuVgB1Ig8AwOFqAW4syBcYKEgBAcUv4J2EF0tP6D+5WhVgwDMLTkVX7DMpqFjgQDi6E9jhACgoSAEB3U3tRh2vNqt3+VIQEWsJZvr1iIynQifDUaSkLWptWeEAViIPAMCaEhFwVKgZGihIAQEgbSkgGVjN4KMCds3fZKZrTlcAovsrbsmf4H7z/kMvmAAlIg8AwHvkcXhAiBscKEgBASVq+znmXDdANt51HB6R1aLUiNC2dO0HT1iUqtS8Z6F5ADEiDwDAa8Yggk9oHR4oSAEBUYTWv2vPRimCXtZAgNy5iMY3b0wHigalk4BeV1BC+VUAHiIPAMBjmLIoEUgfIChIAQFkFPWNkgIntiWCGsnRF2KAeS1zN7xJz25ijFXeaks1OwAbIg8AwGBRncod6CEiIg8AwF7ekhx2aCMkKEgBAZmbgpiPVKGvOf/WEC7nxevwaoe7MjdmUaGfGElg+1PWAB8oSAEBi34yMErN+/f9620NGVPyNM4YLA7kNUNBzi6/M5+T+rIAGyIPAMBeRJvtTOglJihIAQGuzAJ2VhPM6ZjvDnlUhxmUUW2KVATVTIxDBiyxWEnGjgAaIg8AwF4SHEYnaCcoIg8AwF3qmKb5yCkqKEgBATwM4rRbTvXMTygQOAAN0P3je99htC5985672M3oFzKrABAoSAEBmvQ7T6WK4sikPEF56dRPfYMoKohn6AD14iHPu0PIXG0AESIPAMBd2lpx5QgrLCIPAMBd0wBKaYgtLihIAQGAJDolEz6vxu7T4bZ5nycsMPnhWdrLb6YHk6DtX6clHwAPIg8AwF3SzXUuCC8wKEgBAZj5RGzisK6cu7D5HEuUfV3XH9l7kh+YJkUvHmOD8x+OAAQoSAEBen2eXEL+HfMUrc8tTs/QzzywQbk7BwkhbrJcKjmjKuwAECIPAMBd0pT/QGgxMihIAQFGfJ9GFmcwJQAG2fIW1sw7PoTSSnnu73zaAArcog2ekQAKIg8AwF3SbPOwaDM0IZu53SF1kBkCz8vshjg3wgBdkraZCc5Tg/irJWtM4+6BgLukuQxwz+nllEVa9/Mu2AwDiKHA4s/62dmELHS3FsVCWMVhGZ2gAALkCBpycDA1KEgBATtYLqw6Myl/aB80aal2YMmYp4nlmVgzmLq4OM6C27smAAIoSAEBBw2M4JYgqdIbA+zPjO/RlQITYyUb3l6fcGI6GsYk/xQADSQQEe9Vqv////03ODk6AqCbx6mHAAAAAIQBAXI8dwAAAAACAAAAAMAAAAAAAAAAZtWq5wAAFybGxRHAAAAXJsbFEcLqGGx8AASwigFZWZABWVAmxAAAAAgAAAAAAAAB7js8KEgBAdbtVydD+uFhNgWoW1/6GJjaj2d2eMYE36jWJ+9zq7BIAAEqigSQjubvByF4zveT/d62LW5Uzl9wAVo+9ozG55Q4oZr/kF3wIJUiI7PTg32Ejju+cdCEhP3rVfdUykAsuzXJC+XeAhoCGj0+KEgBAdzATRK2nM22UX18oy9xQvXwCJ9Zf54Cg36vVlziEuIcAAgAmAAAFybGtc+EAVlZkEB8nWe27+renBlk3N5pP74pZf4e/YaTiin6gyymFVdlCBlRGKzjN1o9rQaXqNId79Q/aFWYvsjym94bNWRyCj8AmAAAFybGtc+CAXI8dnR1t2tl9ygPFKytIrYccschqEVLVJKRzfGoXydZLkF/V+9JqqYAgWOFo1SWBohYySfyS4Jzv7iZCQya5q+vgNJojAEDkI7m7wcheM73k/3eti1uVM5fcAFaPvaMxueUOKGa/5Au1SPeq+s/fkBEbdDR9O8KVspDwcDI3pfnn1mShOrlfAIaABpojAEDXfAglSIjs9ODfYSOO75x0ISE/etV91TKQCy7NckL5d4d2zEQnvPwYNp0OmphoUWBv1hhDLQJv0uX98Ed7i21wgIaABs=", + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + accountID, err := ton.AccountIDFromRaw(tt.accountID) + if err != nil { + t.Fatal("AccountIDFromRaw() failed: %w", err) + } + state, err := base64.StdEncoding.DecodeString(tt.state) + if err != nil { + t.Fatal("base64 decoding failed: %w", err) + } + proof, err := base64.StdEncoding.DecodeString(tt.proof) + if err != nil { + t.Fatal("base64 decoding failed: %w", err) + } + stateCells, err := boc.DeserializeBoc(state) + if err != nil { + t.Fatal("DeserializeBoc() failed: %w", err) + } + proofCells, err := boc.DeserializeBoc(proof) + if err != nil { + t.Fatal("DeserializeBoc() failed: %w", err) + } + cellMap, err := stateCells[0].NonPrunedCells() + if err != nil { + t.Fatal("Get NonPrunedCells() failed: %w", err) + } + decoder := tlb.NewDecoder().WithDebug().WithPrunedResolver(func(hash tlb.Bits256) (*boc.Cell, error) { + if cellMap == nil { + return nil, fmt.Errorf("failed to fetch library: no resolver provided") + } + cell, ok := cellMap[hash] + if ok { + return cell, nil + } + return nil, errors.New("not found") + }) + var stateProof struct { + Proof tlb.MerkleProof[tlb.ShardStateUnsplit] + } + err = decoder.Unmarshal(proofCells[1], &stateProof) + if err != nil { + t.Fatal("proof unmarshalling failed: %w", err) + } + values := stateProof.Proof.VirtualRoot.ShardStateUnsplit.Accounts.Values() + keys := stateProof.Proof.VirtualRoot.ShardStateUnsplit.Accounts.Keys() + for i, k := range keys { + if bytes.Equal(k[:], accountID.Address[:]) { + fmt.Printf("Account status: %v\n", values[i].Account.Status()) + } + } + }) + } +} diff --git a/tlb/bintree.go b/tlb/bintree.go index 0f547682..8d36af4a 100644 --- a/tlb/bintree.go +++ b/tlb/bintree.go @@ -55,7 +55,11 @@ func (b *BinTree[T]) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error { b.Values = make([]T, 0, len(dec)) for _, i := range dec { if i.CellType() == boc.PrunedBranchCell { - continue + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell == nil { + continue + } + i = cell } var t T err := decoder.Unmarshal(i, &t) diff --git a/tlb/decoder.go b/tlb/decoder.go index 85e215c2..a7621b38 100644 --- a/tlb/decoder.go +++ b/tlb/decoder.go @@ -8,13 +8,15 @@ import ( ) type resolveLib func(hash Bits256) (*boc.Cell, error) +type resolvePruned func(hash Bits256) (*boc.Cell, error) // Decoder unmarshals a cell into a golang type. type Decoder struct { - hasher *boc.Hasher - withDebug bool - debugPath []string - resolveLib resolveLib + hasher *boc.Hasher + withDebug bool + debugPath []string + resolveLib resolveLib + resolvePruned resolvePruned } func (d *Decoder) WithDebug() *Decoder { @@ -28,6 +30,12 @@ func (d *Decoder) WithLibraryResolver(resolveLib resolveLib) *Decoder { return d } +// WithPrunedResolver provides a function which is used to fetch a pruned cell by its hash. +func (d *Decoder) WithPrunedResolver(resolvePruned resolvePruned) *Decoder { + d.resolvePruned = resolvePruned + return d +} + // NewDecoder returns a new Decoder. func NewDecoder() *Decoder { return &Decoder{ @@ -67,6 +75,25 @@ func decode(c *boc.Cell, tag string, val reflect.Value, decoder *Decoder) error decoder.debugPath = decoder.debugPath[:len(decoder.debugPath)-1] }() } + if c.CellType() == boc.PrunedBranchCell { + if val.Kind() == reflect.Ptr && val.Type() == bocCellPointerType { + // this is a pruned cell, and we unmarshal it to a cell. + // let's not resolve it and keep it as is + val.Elem().Set(reflect.ValueOf(c).Elem()) + return nil + } + if val.Kind() == reflect.Ptr && val.Type() == bocTlbANyPointerType { + // same as lib resolve + //todo: remove + a := Any(*c) + val.Elem().Set(reflect.ValueOf(a)) + return nil + } + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell != nil { + c = cell + } + } if c.IsLibrary() { if val.Kind() == reflect.Ptr && val.Type() == bocCellPointerType { // this is a library cell, and we unmarshal it to a cell. @@ -116,10 +143,14 @@ func decode(c *boc.Cell, tag string, val reflect.Value, decoder *Decoder) error return fmt.Errorf("library cell as a ref is not implemented") } if c.CellType() == boc.PrunedBranchCell { + cell := resolvePrunedCell(c, decoder.resolvePruned) + // TODO: maybe check for pointer too if val.Kind() == reflect.Struct && val.Type() == bocCellType { return decodeCell(c, val) + } else if cell == nil { + return nil } - return nil + c = cell } case t.IsMaybe: tag = "" @@ -140,10 +171,14 @@ func decode(c *boc.Cell, tag string, val reflect.Value, decoder *Decoder) error return fmt.Errorf("library cell as a ref is not implemented") } if c.CellType() == boc.PrunedBranchCell { + cell := resolvePrunedCell(c, decoder.resolvePruned) + // TODO: maybe check for pointer too if val.Kind() == reflect.Struct && val.Type() == bocCellType { return decodeCell(c, val) + } else if cell == nil { + return nil } - return nil + c = cell } } i, ok := val.Interface().(UnmarshalerTLB) @@ -345,3 +380,20 @@ func decodeBitString(c *boc.Cell, val reflect.Value) error { func (dec *Decoder) Hasher() *boc.Hasher { return dec.hasher } + +func resolvePrunedCell(c *boc.Cell, resolver resolvePruned) *boc.Cell { + if resolver == nil { + return nil + } + hash, err := c.Hash256WithLevel(0) + if err != nil { + return nil + } + cell, err := resolver(hash) + if err != nil { + return nil + } + // TODO: we need to reset all counters for cells or deep copy all cells + cell.ResetCounters() + return cell +} diff --git a/tlb/hashmap.go b/tlb/hashmap.go index 1f29d019..1ea68556 100644 --- a/tlb/hashmap.go +++ b/tlb/hashmap.go @@ -223,7 +223,11 @@ func (h *Hashmap[keyT, T]) mapInner(keySize, leftKeySize int, c *boc.Cell, keyPr var err error var size int if c.CellType() == boc.PrunedBranchCell { - return nil + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell == nil { + return nil + } + c = cell } size, keyPrefix, err = loadLabel(leftKeySize, c, keyPrefix) if err != nil { @@ -444,7 +448,11 @@ func (h *HashmapAug[keyT, T1, T2]) mapInner(keySize, leftKeySize int, c *boc.Cel var err error var size int if c.CellType() == boc.PrunedBranchCell { - return nil + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell == nil { + return nil + } + c = cell } size, keyPrefix, err = loadLabel(leftKeySize, c, keyPrefix) if err != nil { diff --git a/tlb/primitives.go b/tlb/primitives.go index 7a5c5db6..efdf4c2d 100644 --- a/tlb/primitives.go +++ b/tlb/primitives.go @@ -228,9 +228,13 @@ func (m *Ref[T]) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error { return err } if r.CellType() == boc.PrunedBranchCell { - var value T - m.Value = value - return nil + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell == nil { + var value T + m.Value = value + return nil + } + r = cell } err = decoder.Unmarshal(r, &m.Value) if err != nil { diff --git a/tlb/proof.go b/tlb/proof.go index 225b32c1..584f96de 100644 --- a/tlb/proof.go +++ b/tlb/proof.go @@ -120,7 +120,15 @@ func (s *ShardState) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error { return err } } else { - s.SplitState.Left = ShardStateUnsplit{} + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell == nil { + s.SplitState.Left = ShardStateUnsplit{} + } else { + err = decoder.Unmarshal(cell, &s.SplitState.Left) + if err != nil { + return err + } + } } c1, err = c.NextRef() if err != nil { @@ -131,7 +139,15 @@ func (s *ShardState) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error { return err } } else { - s.SplitState.Right = ShardStateUnsplit{} + cell := resolvePrunedCell(c, decoder.resolvePruned) + if cell == nil { + s.SplitState.Right = ShardStateUnsplit{} + } else { + err = decoder.Unmarshal(cell, &s.SplitState.Right) + if err != nil { + return err + } + } } s.SumType = "SplitState" break From 119705c991c29a6a3ba50ebdf86d1e53cc5a664a Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Mon, 9 Sep 2024 21:12:44 +0300 Subject: [PATCH 4/9] pruned resolver in account proof verification --- liteapi/account.go | 149 ++++++++++++++------------------------------- 1 file changed, 46 insertions(+), 103 deletions(-) diff --git a/liteapi/account.go b/liteapi/account.go index b3545d38..9e960581 100644 --- a/liteapi/account.go +++ b/liteapi/account.go @@ -10,46 +10,6 @@ import ( "github.com/tonkeeper/tongo/ton" ) -// blockTrimmed stripped-down version of the tlb.Block with pruned cell instead of ShardState and skip some decoding -type blockTrimmed struct { - Magic tlb.Magic `tlb:"block#11ef55aa"` - GlobalId int32 - Info boc.Cell `tlb:"^"` - ValueFlow boc.Cell `tlb:"^"` - StateUpdate tlb.MerkleUpdate[boc.Cell] `tlb:"^"` - Extra boc.Cell `tlb:"^"` -} - -// shardAccountPruned stripped-down version of the tlb.ShardAccount with pruned cell instead of Account -type shardAccountPruned struct { - Account boc.Cell `tlb:"^"` - LastTransHash tlb.Bits256 - LastTransLt uint64 -} - -// shardStateUnsplitTrimmed stripped-down version of the ShardStateUnsplit structure for extracting Account proof -type shardStateUnsplitTrimmed struct { - Magic tlb.Magic `tlb:"shard_state#9023afe2"` - GlobalID int32 - ShardID tlb.ShardIdent - SeqNo uint32 - VertSeqNo uint32 - GenUtime uint32 - GenLt uint64 - MinRefMcSeqno uint32 - OutMsgQueueInfo boc.Cell `tlb:"^"` - BeforeSplit bool - Accounts tlb.HashmapAugE[tlb.Bits256, shardAccountPruned, tlb.DepthBalanceInfo] `tlb:"^"` - Other tlb.ShardStateUnsplitOther `tlb:"^"` - Custom tlb.Maybe[tlb.Ref[boc.Cell]] -} - -// TODO: use proof merger instead of trimmed -type shardStateUnsplit struct { - ShardStateTrimmed shardStateUnsplitTrimmed - ShardState tlb.ShardStateUnsplit -} - // GetAccountWithProof // For safe operation, always use GetAccountWithProof with WithBlock(proofedBlock ton.BlockIDExt), as the proof of masterchain cashed blocks is not implemented yet! func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountID) (*tlb.ShardAccount, *tlb.ShardStateUnsplit, error) { // TODO: return merged tlb.ShardStateUnsplit (proof+account) @@ -79,65 +39,44 @@ func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountI } blockHash = *shardHash } - shardState, err := checkBlockShardStateProof(res.Proof, blockHash) + cellsMap := make(map[[32]byte]*boc.Cell) + if len(res.State) > 0 { + stateCells, err := boc.DeserializeBoc(res.State) + if err != nil { + return nil, nil, fmt.Errorf("state deserialization failed: %w", err) + } + hash, err := stateCells[0].Hash256() + if err != nil { + return nil, nil, fmt.Errorf("get hash err: %w", err) + } + cellsMap[hash] = stateCells[0] + } + shardState, err := checkBlockShardStateProof(res.Proof, blockHash, cellsMap) if err != nil { return nil, nil, fmt.Errorf("incorrect block proof: %w", err) } - values := shardState.ShardStateTrimmed.Accounts.Values() - keys := shardState.ShardStateTrimmed.Accounts.Keys() + values := shardState.ShardStateUnsplit.Accounts.Values() + keys := shardState.ShardStateUnsplit.Accounts.Keys() for i, k := range keys { if bytes.Equal(k[:], accountID.Address[:]) { - acc, err := decodeAccount(res.State, values[i]) - if err != nil { - return nil, nil, err - } - return acc, &shardState.ShardState, nil + return &values[i], shardState, nil } } if len(res.State) == 0 { - return &tlb.ShardAccount{Account: tlb.Account{SumType: "AccountNone"}}, &shardState.ShardState, nil + return &tlb.ShardAccount{Account: tlb.Account{SumType: "AccountNone"}}, shardState, nil } return nil, nil, errors.New("invalid account state") } -func decodeAccount(state []byte, shardAccount shardAccountPruned) (*tlb.ShardAccount, error) { - stateCells, err := boc.DeserializeBoc(state) - if err != nil { - return nil, err - } - if len(stateCells) != 1 { - return nil, boc.ErrNotSingleRoot - } - accountHash, err := stateCells[0].Hash256() - if err != nil { - return nil, err - } - shardAccountHash, err := shardAccount.Account.Hash256WithLevel(0) - if err != nil { - return nil, err - } - if accountHash != shardAccountHash { - return nil, errors.New("invalid account hash") - } - var acc tlb.Account - err = tlb.Unmarshal(stateCells[0], &acc) - if err != nil { - return nil, err - } - // do not check account balance from tlb.DepthBalanceInfo - res := tlb.ShardAccount{Account: acc, LastTransHash: shardAccount.LastTransHash, LastTransLt: shardAccount.LastTransLt} - return &res, nil -} - func checkShardInMasterProof(master ton.BlockIDExt, shardProof []byte, workchain int32, shardRootHash ton.Bits256) error { - shardState, err := checkBlockShardStateProof(shardProof, master.RootHash) + shardState, err := checkBlockShardStateProof(shardProof, master.RootHash, nil) if err != nil { return fmt.Errorf("check block proof failed: %w", err) } - if !shardState.ShardState.ShardStateUnsplit.Custom.Exists { + if !shardState.ShardStateUnsplit.Custom.Exists { return fmt.Errorf("not a masterchain block") } - stateExtra := shardState.ShardState.ShardStateUnsplit.Custom.Value.Value + stateExtra := shardState.ShardStateUnsplit.Custom.Value.Value keys := stateExtra.ShardHashes.Keys() values := stateExtra.ShardHashes.Values() for i, k := range keys { @@ -158,7 +97,7 @@ func checkShardInMasterProof(master ton.BlockIDExt, shardProof []byte, workchain return fmt.Errorf("required shard hash not found in proof") } -func checkBlockShardStateProof(proof []byte, blockRootHash ton.Bits256) (*shardStateUnsplit, error) { +func checkBlockShardStateProof(proof []byte, blockRootHash ton.Bits256, cellsMap map[[32]byte]*boc.Cell) (*tlb.ShardStateUnsplit, error) { proofCells, err := boc.DeserializeBoc(proof) if err != nil { return nil, err @@ -166,41 +105,46 @@ func checkBlockShardStateProof(proof []byte, blockRootHash ton.Bits256) (*shardS if len(proofCells) != 2 { return nil, errors.New("must be two root cells") } - block, err := checkBlockProof(proofCells[0], blockRootHash) + stateUpdate, err := checkBlockProof(proofCells[0], blockRootHash) if err != nil { return nil, fmt.Errorf("incorrect block proof: %w", err) } - var stateTrimmedProof struct { - Proof tlb.MerkleProof[shardStateUnsplitTrimmed] - } - err = tlb.Unmarshal(proofCells[1], &stateTrimmedProof) // cells order must be strictly defined - if err != nil { - return nil, err - } - proofCells[1].ResetCounters() var stateProof struct { Proof tlb.MerkleProof[tlb.ShardStateUnsplit] } - err = tlb.Unmarshal(proofCells[1], &stateProof) + decoder := tlb.NewDecoder() + if cellsMap != nil { + decoder = decoder.WithPrunedResolver(func(hash tlb.Bits256) (*boc.Cell, error) { + cell, ok := cellsMap[hash] + if ok { + return cell, nil + } + return nil, errors.New("not found") + }) + } + err = decoder.Unmarshal(proofCells[1], &stateProof) if err != nil { return nil, err } - toRootHash, err := block.StateUpdate.ToRoot.Hash256WithLevel(0) + toRootHash, err := stateUpdate.ToRoot.Hash256WithLevel(0) if err != nil { return nil, err } - if stateTrimmedProof.Proof.VirtualHash != toRootHash { + if stateProof.Proof.VirtualHash != toRootHash { return nil, errors.New("invalid virtual hash") } - res := shardStateUnsplit{ - ShardStateTrimmed: stateTrimmedProof.Proof.VirtualRoot, - ShardState: stateProof.Proof.VirtualRoot, - } - return &res, nil + return &stateProof.Proof.VirtualRoot, nil } -func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*blockTrimmed, error) { - var res tlb.MerkleProof[blockTrimmed] +func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*tlb.MerkleUpdate[boc.Cell], error) { + // stripped-down version of the tlb.Block with pruned cell instead of ShardState and skip some decoding + var res tlb.MerkleProof[struct { + Magic tlb.Magic `tlb:"block#11ef55aa"` + GlobalId int32 + Info boc.Cell `tlb:"^"` + ValueFlow boc.Cell `tlb:"^"` + StateUpdate tlb.MerkleUpdate[boc.Cell] `tlb:"^"` + }] err := tlb.Unmarshal(proof, &res) // merkle hash and depth checks inside if err != nil { return nil, fmt.Errorf("failed to unmarshal block proof: %w", err) @@ -208,6 +152,5 @@ func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*blockTrimmed, if ton.Bits256(res.VirtualHash) != blockRootHash { return nil, fmt.Errorf("invalid block root hash") } - block := res.VirtualRoot - return &block, nil + return &res.VirtualRoot.StateUpdate, nil } From 07d26524a7bebddbf43896d0abb7373953d140a5 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Mon, 9 Sep 2024 21:25:27 +0300 Subject: [PATCH 5/9] deleted unused methods --- boc/cell.go | 37 ------------------------------------- liteapi/account_test.go | 11 ++++++----- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/boc/cell.go b/boc/cell.go index 447e2c14..ddc82444 100644 --- a/boc/cell.go +++ b/boc/cell.go @@ -470,40 +470,3 @@ func (c *Cell) Hash256WithLevel(level int) ([32]byte, error) { copy(h[:], b) return h, nil } - -// NonPrunedCells returns a map of all non-pruned cells, where the key is the hash of the cell. It can be used to resolve cells in the proofs. -func (c *Cell) NonPrunedCells() (map[[32]byte]*Cell, error) { - // TODO: mutable cell map may change during resolving. It may be necessary to make full copies of cells. - res := map[[32]byte]*Cell{} - if c.CellType() == PrunedBranchCell { - return res, nil - } - h, err := c.Hash256() - if err != nil { - return nil, err - } - res[h] = c - err = collectCells(c, res) - if err != nil { - return nil, err - } - return res, nil -} - -func collectCells(c *Cell, m map[[32]byte]*Cell) error { - refs := c.refs - for _, r := range refs { - if r != nil && r.CellType() != PrunedBranchCell { - h, err := r.Hash256() - if err != nil { - return err - } - m[h] = c - err = collectCells(r, m) - if err != nil { - return err - } - } - } - return nil -} diff --git a/liteapi/account_test.go b/liteapi/account_test.go index 259c5332..da8a14b2 100644 --- a/liteapi/account_test.go +++ b/liteapi/account_test.go @@ -92,15 +92,16 @@ func TestUnmarshallingProofWithPrunedResolver(t *testing.T) { if err != nil { t.Fatal("DeserializeBoc() failed: %w", err) } - cellMap, err := stateCells[0].NonPrunedCells() + hash, err := stateCells[0].Hash256() + if err != nil { + t.Fatal("Get hash failed: %w", err) + } + cellsMap := map[[32]byte]*boc.Cell{hash: stateCells[0]} if err != nil { t.Fatal("Get NonPrunedCells() failed: %w", err) } decoder := tlb.NewDecoder().WithDebug().WithPrunedResolver(func(hash tlb.Bits256) (*boc.Cell, error) { - if cellMap == nil { - return nil, fmt.Errorf("failed to fetch library: no resolver provided") - } - cell, ok := cellMap[hash] + cell, ok := cellsMap[hash] if ok { return cell, nil } From bcac29b358e40fa4f26fdadff1c56888309f3319 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Sun, 15 Sep 2024 02:30:29 +0300 Subject: [PATCH 6/9] simplify checkBlockProof --- liteapi/account.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/liteapi/account.go b/liteapi/account.go index 9e960581..f80f005b 100644 --- a/liteapi/account.go +++ b/liteapi/account.go @@ -105,7 +105,7 @@ func checkBlockShardStateProof(proof []byte, blockRootHash ton.Bits256, cellsMap if len(proofCells) != 2 { return nil, errors.New("must be two root cells") } - stateUpdate, err := checkBlockProof(proofCells[0], blockRootHash) + newStateHash, err := checkBlockProof(proofCells[0], blockRootHash) if err != nil { return nil, fmt.Errorf("incorrect block proof: %w", err) } @@ -126,25 +126,14 @@ func checkBlockShardStateProof(proof []byte, blockRootHash ton.Bits256, cellsMap if err != nil { return nil, err } - toRootHash, err := stateUpdate.ToRoot.Hash256WithLevel(0) - if err != nil { - return nil, err - } - if stateProof.Proof.VirtualHash != toRootHash { + if stateProof.Proof.VirtualHash != *newStateHash { return nil, errors.New("invalid virtual hash") } return &stateProof.Proof.VirtualRoot, nil } -func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*tlb.MerkleUpdate[boc.Cell], error) { - // stripped-down version of the tlb.Block with pruned cell instead of ShardState and skip some decoding - var res tlb.MerkleProof[struct { - Magic tlb.Magic `tlb:"block#11ef55aa"` - GlobalId int32 - Info boc.Cell `tlb:"^"` - ValueFlow boc.Cell `tlb:"^"` - StateUpdate tlb.MerkleUpdate[boc.Cell] `tlb:"^"` - }] +func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*tlb.Bits256, error) { + var res tlb.MerkleProof[tlb.Block] err := tlb.Unmarshal(proof, &res) // merkle hash and depth checks inside if err != nil { return nil, fmt.Errorf("failed to unmarshal block proof: %w", err) @@ -152,5 +141,5 @@ func checkBlockProof(proof *boc.Cell, blockRootHash ton.Bits256) (*tlb.MerkleUpd if ton.Bits256(res.VirtualHash) != blockRootHash { return nil, fmt.Errorf("invalid block root hash") } - return &res.VirtualRoot.StateUpdate, nil + return &res.VirtualRoot.StateUpdate.ToHash, nil // return new_hash field of MerkleUpdate of ShardState } From 918ea03db814c0633c78012d3624481a62bf43f3 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Sun, 15 Sep 2024 04:13:33 +0300 Subject: [PATCH 7/9] skip check for known shard block --- liteapi/account.go | 19 +++++++++-------- liteapi/account_test.go | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/liteapi/account.go b/liteapi/account.go index f80f005b..be4f7d6e 100644 --- a/liteapi/account.go +++ b/liteapi/account.go @@ -21,23 +21,22 @@ func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountI if len(res.Proof) == 0 { return nil, nil, errors.New("empty proof") } - var shardHash *ton.Bits256 - if accountID.Workchain != -1 { // TODO: set masterchain constant + + var blockHash ton.Bits256 + if accountID.Workchain == -1 || blockID == res.Shardblk.ToBlockIdExt() { + blockHash = blockID.RootHash + } else { if len(res.ShardProof) == 0 { return nil, nil, errors.New("empty shard proof") - } // TODO: change logic for shard blockID (proof shard to proofed master) + } if res.Shardblk.RootHash == [32]byte{} { // TODO: how to check for empty shard? return nil, nil, errors.New("shard block not passed") } - h := ton.Bits256(res.Shardblk.RootHash) - shardHash = &h - } - blockHash := blockID.RootHash - if shardHash != nil { // we need shard proof only for not masterchain - if err := checkShardInMasterProof(blockID, res.ShardProof, accountID.Workchain, *shardHash); err != nil { + shardHash := ton.Bits256(res.Shardblk.RootHash) + if err := checkShardInMasterProof(blockID, res.ShardProof, accountID.Workchain, shardHash); err != nil { return nil, nil, fmt.Errorf("shard proof is incorrect: %w", err) } - blockHash = *shardHash + blockHash = shardHash } cellsMap := make(map[[32]byte]*boc.Cell) if len(res.State) > 0 { diff --git a/liteapi/account_test.go b/liteapi/account_test.go index da8a14b2..8b72c3f8 100644 --- a/liteapi/account_test.go +++ b/liteapi/account_test.go @@ -124,3 +124,48 @@ func TestUnmarshallingProofWithPrunedResolver(t *testing.T) { }) } } + +func TestGetAccountWithProofForBlock(t *testing.T) { + api, err := NewClient(Testnet(), FromEnvs()) + if err != nil { + t.Fatal(err) + } + testCases := []struct { + name string + accountID string + block string + }{ + { + name: "active account from basechain", + accountID: "0:e33ed33a42eb2032059f97d90c706f8400bb256d32139ca707f1564ad699c7dd", + block: "(0,e000000000000000,24681072)", + }, + { + name: "account from masterchain", + accountID: "-1:34517c7bdf5187c55af4f8b61fdc321588c7ab768dee24b006df29106458d7cf", + block: "(-1,8000000000000000,23040403)", + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + accountID, err := ton.AccountIDFromRaw(tt.accountID) + if err != nil { + t.Fatal("AccountIDFromRaw() failed: %w", err) + } + b, err := ton.ParseBlockID(tt.block) + if err != nil { + t.Fatal("ParseBlockID() failed: %w", err) + } + block, _, err := api.LookupBlock(context.TODO(), b, 1, nil, nil) + if err != nil { + t.Fatal("LookupBlock() failed: %w", err) + } + acc, st, err := api.WithBlock(block).GetAccountWithProof(context.TODO(), accountID) + if err != nil { + t.Fatal(err) + } + fmt.Printf("Account status: %v\n", acc.Account.Status()) + fmt.Printf("Last proof utime: %v\n", st.ShardStateUnsplit.GenUtime) + }) + } +} From a66b5cdbd6bf9badb2bf3b6aeaeb3a5bdd6100b3 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Mon, 16 Sep 2024 22:44:15 +0300 Subject: [PATCH 8/9] comparison fixes --- liteapi/account.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/liteapi/account.go b/liteapi/account.go index be4f7d6e..75921601 100644 --- a/liteapi/account.go +++ b/liteapi/account.go @@ -1,7 +1,6 @@ package liteapi import ( - "bytes" "context" "errors" "fmt" @@ -23,7 +22,7 @@ func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountI } var blockHash ton.Bits256 - if accountID.Workchain == -1 || blockID == res.Shardblk.ToBlockIdExt() { + if (accountID.Workchain == -1 && blockID.Workchain == -1) || blockID == res.Shardblk.ToBlockIdExt() { blockHash = blockID.RootHash } else { if len(res.ShardProof) == 0 { @@ -57,7 +56,7 @@ func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountI values := shardState.ShardStateUnsplit.Accounts.Values() keys := shardState.ShardStateUnsplit.Accounts.Keys() for i, k := range keys { - if bytes.Equal(k[:], accountID.Address[:]) { + if k == accountID.Address { return &values[i], shardState, nil } } From 79764178ee79e44a3a961ead06b466b62c82cce7 Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Mon, 16 Sep 2024 22:52:36 +0300 Subject: [PATCH 9/9] todo fix --- liteapi/account.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liteapi/account.go b/liteapi/account.go index 75921601..edd467aa 100644 --- a/liteapi/account.go +++ b/liteapi/account.go @@ -11,7 +11,7 @@ import ( // GetAccountWithProof // For safe operation, always use GetAccountWithProof with WithBlock(proofedBlock ton.BlockIDExt), as the proof of masterchain cashed blocks is not implemented yet! -func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountID) (*tlb.ShardAccount, *tlb.ShardStateUnsplit, error) { // TODO: return merged tlb.ShardStateUnsplit (proof+account) +func (c *Client) GetAccountWithProof(ctx context.Context, accountID ton.AccountID) (*tlb.ShardAccount, *tlb.ShardStateUnsplit, error) { res, err := c.GetAccountStateRaw(ctx, accountID) // TODO: add proof check for masterHead if err != nil { return nil, nil, err