[x/programs] Fix program calling another program on the Rust side (#866)

* support program passing in param

* fix inter-program interaction

* update fixture

* keep using `SmartPtr` for now

* remove the need of a new Param type

* lint

* remove param

* fix nits

* remove whitespace

* Cached state values in the program state (#840)

* add cache

* implement cache reads

* actually cache reads

* implement cache delete and handle borsch errors

* use only one cache map

* remove flushed bool

* root of transactions by stateless merkledb

* make root generation a function

* preallocate memory for merkle array and consumebytes flag

* add <*.code-workspace> to .gitignore and remove it from git commit

* move root generation func to merkle package, tx root by items of [txID + result]

* rebase & blk marshal/unmarshal & merkleroot to ids.ID

* write benches for the merkle package

* use crypto/rand, fix var name, report allocs

* put the 10k bench back

* pass config by parameter

* happy clippy

* borrow V

* add TODO

* Revert "pass config by parameter"

This reverts commit 4aec589.

* Revert "put the 10k bench back"

This reverts commit 058d7e7.

* Revert "use crypto/rand, fix var name, report allocs"

This reverts commit 214005b.

* Revert "write benches for the merkle package"

This reverts commit 07993bf.

* Revert "rebase & blk marshal/unmarshal & merkleroot to ids.ID"

This reverts commit 7442836.

* Revert "move root generation func to merkle package, tx root by items of [txID + result]"

This reverts commit e551960.

* Revert "add <*.code-workspace> to .gitignore and remove it from git commit"

This reverts commit ce00289.

* Revert "preallocate memory for merkle array and consumebytes flag"

This reverts commit 68e49b6.

* Revert "make root generation a function"

This reverts commit aa44f97.

* Revert "pass config by parameter"

This reverts commit 4aec589.

* Revert "move root generation func to merkle package, tx root by items of [txID + result]"

This reverts commit e551960.

* Revert "preallocate memory for merkle array and consumebytes flag"

This reverts commit 68e49b6.

* Revert "make root generation a function"

This reverts commit aa44f97.

* merge main!

* merge imports

---------

Co-authored-by: bianyuanop <chen.me.nan@gmail.com>
Co-authored-by: Richard Pringle <richard.pringle@avalabs.org>

* [x/programs] safe wrapper around C ffi interface (#869)

* macro skeleton

* write macro for ffi-safe bindings

* avoid panicking todo!

Signed-off-by: Franfran <51274081+iFrostizz@users.noreply.github.com>

* remove unused arms

---------

Signed-off-by: Franfran <51274081+iFrostizz@users.noreply.github.com>

---------

Signed-off-by: Franfran <51274081+iFrostizz@users.noreply.github.com>
Co-authored-by: bianyuanop <chen.me.nan@gmail.com>
Co-authored-by: Richard Pringle <richard.pringle@avalabs.org>

x/programs/cmd/simulator/cmd/plan.go

@@ -275,15 +275,29 @@ func (c *runCmd) createCallParams(ctx context.Context, db state.Immutable, param
 	for _, param := range params {
 		switch param.Type {
 		case String, ID:
-			val, ok := param.Value.(string)
+			stepIdStr, ok := param.Value.(string)
 			if !ok {
 				return nil, fmt.Errorf("%w: %s", ErrFailedParamTypeCast, param.Type)
 			}
-			val, err := c.verifyProgramIDStr(val)
-			if err != nil {
-				return nil, err
+			if strings.HasPrefix(stepIdStr, "step_") {
+				programIdStr, ok := c.programIDStrMap[stepIdStr]
+				if !ok {
+					return nil, fmt.Errorf("failed to map to id: %s", stepIdStr)
+				}
+				programId, err := ids.FromString(programIdStr)
+				if err != nil {
+					return nil, err
+				}
+				cp = append(cp, actions.CallParam{Value: programId})
+			} else {
+				programId, err := ids.FromString(stepIdStr)
+				if err == nil {
+					cp = append(cp, actions.CallParam{Value: programId})
+				} else {
+					// this is a path to the wasm program
+					cp = append(cp, actions.CallParam{Value: stepIdStr})
+				}
 			}
-			cp = append(cp, actions.CallParam{Value: val})
 		case Bool:
 			val, ok := param.Value.(bool)
 			if !ok {
@@ -334,28 +348,6 @@ func (c *runCmd) createCallParams(ctx context.Context, db state.Immutable, param
 	return cp, nil
 }
 
-// verifyProgramIDStr verifies a string is a valid ID and checks the programIDStrMap for
-// the synthetic identifier `step_N` where N is the step the id was created from
-// execution.
-func (c *runCmd) verifyProgramIDStr(idStr string) (string, error) {
-	// if the id is valid
-	_, err := ids.FromString(idStr)
-	if err == nil {
-		return idStr, nil
-	}
-
-	// check if the id is a synthetic identifier
-	if strings.HasPrefix(idStr, "step_") {
-		stepID, ok := c.programIDStrMap[idStr]
-		if !ok {
-			return "", fmt.Errorf("failed to map to id: %s", idStr)
-		}
-		return stepID, nil
-	}
-
-	return idStr, nil
-}
-
 // generateRandomID creates a unique ID.
 // Note: ids.GenerateID() is not used because the IDs are not unique and will
 // collide.

x/programs/cmd/simulator/vm/actions/program_execute.go

@@ -5,6 +5,7 @@ package actions
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/near/borsh-go"
@@ -22,9 +23,9 @@ import (
 	"github.com/ava-labs/hypersdk/state"
 	"github.com/ava-labs/hypersdk/utils"
 
-	"github.com/ava-labs/hypersdk/x/programs/cmd/simulator/vm/storage"
 	importProgram "github.com/ava-labs/hypersdk/x/programs/examples/imports/program"
 	"github.com/ava-labs/hypersdk/x/programs/examples/imports/pstate"
+	"github.com/ava-labs/hypersdk/x/programs/examples/storage"
 	"github.com/ava-labs/hypersdk/x/programs/runtime"
 )
 
@@ -67,17 +68,16 @@ func (t *ProgramExecute) Execute(
 		return false, 1, OutputValueZero, nil
 	}
 
-	programIDStr, ok := t.Params[0].Value.(string)
+	programID, ok := t.Params[0].Value.(ids.ID)
 	if !ok {
 		return false, 1, utils.ErrBytes(fmt.Errorf("invalid call param: must be ID")), nil
 	}
 
 	// TODO: take fee out of balance?
-	programID, err := ids.FromString(programIDStr)
-	if err != nil {
-		return false, 1, utils.ErrBytes(err), nil
+	programBytes, exists, err := storage.GetProgram(context.Background(), mu, programID)
+	if !exists {
+		err = errors.New("unknown program")
 	}
-	programBytes, err := storage.GetProgram(ctx, mu, programID)
 	if err != nil {
 		return false, 1, utils.ErrBytes(err), nil
 	}

x/programs/cmd/simulator/vm/storage/storage.go

@@ -15,6 +15,8 @@ import (
 	"github.com/ava-labs/hypersdk/crypto/ed25519"
 	"github.com/ava-labs/hypersdk/fees"
 	"github.com/ava-labs/hypersdk/state"
+
+	"github.com/ava-labs/hypersdk/x/programs/examples/storage"
 )
 
 const (
@@ -58,10 +60,18 @@ func GetProgram(
 	programID ids.ID,
 ) (
 	[]byte, // program bytes
+	bool, // exists
 	error,
 ) {
 	k := ProgramKey(programID)
-	return db.GetValue(ctx, k)
+	v, err := db.GetValue(ctx, k)
+	if errors.Is(err, database.ErrNotFound) {
+		return nil, false, nil
+	}
+	if err != nil {
+		return nil, false, err
+	}
+	return v, true, nil
 }
 
 // setProgram stores [program] at [programID]
@@ -71,8 +81,7 @@ func SetProgram(
 	programID ids.ID,
 	program []byte,
 ) error {
-	k := ProgramKey(programID)
-	return mu.Insert(ctx, k, program)
+	return storage.SetProgram(ctx, mu, programID, program)
 }
 
 //

x/programs/examples/counter_test.go

@@ -28,7 +28,7 @@ import (
 func TestCounterProgram(t *testing.T) {
 	require := require.New(t)
 	db := newTestDB()
-	maxUnits := uint64(80000)
+	maxUnits := uint64(10000000)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -102,7 +102,7 @@ func TestCounterProgram(t *testing.T) {
 	require.NoError(err)
 
 	// define max units to transfer to second runtime
-	unitsTransfer := uint64(20000)
+	unitsTransfer := uint64(2000000)
 
 	// transfer the units from the original runtime to the new runtime before
 	// any calls are made.
@@ -180,7 +180,7 @@ func TestCounterProgram(t *testing.T) {
 	target, err := writeToMem(programID2, mem)
 	require.NoError(err)
 
-	maxUnitsProgramToProgram := int64(10000)
+	maxUnitsProgramToProgram := int64(1000000)
 	maxUnitsProgramToProgramPtr, err := writeToMem(maxUnitsProgramToProgram, mem)
 	require.NoError(err)
 

x/programs/examples/imports/program/program.go

@@ -6,6 +6,7 @@ package program
 import (
 	"context"
 	"encoding/binary"
+	"errors"
 
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/utils/logging"
@@ -217,6 +218,7 @@ func getProgramWasmBytes(log logging.Logger, db state.Immutable, idBytes []byte)
 	bytes, exists, err := storage.GetProgram(context.Background(), db, id)
 	if !exists {
 		log.Debug("key does not exist", zap.String("id", id.String()))
+		return nil, errors.New("unknown program")
 	}
 	if err != nil {
 		return nil, err

x/programs/examples/storage/storage.go

@@ -15,7 +15,8 @@ import (
 )
 
 const (
-	programPrefix = 0x0
+	programPrefix        = 0x0
+	ProgramChunks uint16 = 1
 )
 
 func ProgramPrefixKey(id []byte, key []byte) (k []byte) {

x/programs/rust/examples/counter/Cargo.toml

@@ -8,6 +8,9 @@ edition = "2021"
 [dependencies]
 wasmlanche-sdk = { version = "0.1.0", path = "../../wasmlanche-sdk" }
 
+[dev-dependencies]
+simulator = { path = "../../../cmd/simulator/" }
+
 [build-dependencies]
 wasmlanche-sdk = { path = "../../wasmlanche-sdk", features = ["build"] }