merging w. 6.0.0

.gitignore

@@ -8,6 +8,8 @@
 *.fdb_latexmk
 *.gz
 
+
+yarn.lock
 *.log
 docs/spec/out/
 test-out/

core/shared/src/main/scala/sigma/VersionContext.scala

@@ -23,7 +23,7 @@ case class VersionContext(activatedVersion: Byte, ergoTreeVersion: Byte) {
   def isJitActivated: Boolean = activatedVersion >= JitActivationVersion
 
   /** @return true, if the activated script version of Ergo protocol on the network is
-   * including Evolution update. */
+   * including v6.0 update. */
   def isV6SoftForkActivated: Boolean = activatedVersion >= V6SoftForkVersion
 }
 

core/shared/src/main/scala/sigma/ast/SType.scala

@@ -7,7 +7,7 @@ import sigma.data.OverloadHack.Overloaded1
 import sigma.data.{CBigInt, CUnsignedBigInt, Nullable, SigmaConstants}
 import sigma.reflection.{RClass, RMethod, ReflectionData}
 import sigma.util.Extensions.{IntOps, LongOps, ShortOps}
-import sigma.{AvlTree, BigInt, Box, Coll, Context, Evaluation, GroupElement, Header, PreHeader, SigmaDslBuilder, SigmaProp, UnsignedBigInt}
+import sigma.{AvlTree, BigInt, Box, Coll, Context, Evaluation, GroupElement, Header, PreHeader, SigmaDslBuilder, SigmaProp, UnsignedBigInt, VersionContext}
 
 import java.math.BigInteger
 
@@ -376,6 +376,7 @@ case object SByte extends SPrimType with SEmbeddable with SNumericType with SMon
     case s: Short => s.toByteExact
     case i: Int => i.toByteExact
     case l: Long => l.toByteExact
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toByte // toByteExact from int is called under the hood
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -397,6 +398,7 @@ case object SShort extends SPrimType with SEmbeddable with SNumericType with SMo
     case s: Short => s
     case i: Int => i.toShortExact
     case l: Long => l.toShortExact
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toShort // toShortExact from int is called under the hood
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -420,6 +422,7 @@ case object SInt extends SPrimType with SEmbeddable with SNumericType with SMono
     case s: Short => s.toInt
     case i: Int => i
     case l: Long => l.toIntExact
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toInt
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -445,6 +448,7 @@ case object SLong extends SPrimType with SEmbeddable with SNumericType with SMon
     case s: Short => s.toLong
     case i: Int => i.toLong
     case l: Long => l
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toLong
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -465,24 +469,24 @@ case object SBigInt extends SPrimType with SEmbeddable with SNumericType with SM
   override def numericTypeIndex: Int = 4
 
   override def upcast(v: AnyVal): BigInt = {
-    val bi = v match {
-      case x: Byte => BigInteger.valueOf(x.toLong)
-      case x: Short => BigInteger.valueOf(x.toLong)
-      case x: Int => BigInteger.valueOf(x.toLong)
-      case x: Long => BigInteger.valueOf(x)
+    v match {
+      case x: Byte => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Short => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Int => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Long => CBigInt(BigInteger.valueOf(x))
+      case x: BigInt if VersionContext.current.isV6SoftForkActivated => x
       case _ => sys.error(s"Cannot upcast value $v to the type $this")
     }
-    CBigInt(bi)
   }
   override def downcast(v: AnyVal): BigInt = {
-    val bi = v match {
-      case x: Byte => BigInteger.valueOf(x.toLong)
-      case x: Short => BigInteger.valueOf(x.toLong)
-      case x: Int => BigInteger.valueOf(x.toLong)
-      case x: Long => BigInteger.valueOf(x)
+    v match {
+      case x: Byte => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Short => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Int => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Long => CBigInt(BigInteger.valueOf(x))
+      case x: BigInt if VersionContext.current.isV6SoftForkActivated => x
       case _ => sys.error(s"Cannot downcast value $v to the type $this")
     }
-    CBigInt(bi)
   }
 }
 

data/shared/src/main/scala/sigma/ast/SMethod.scala

@@ -50,31 +50,40 @@ case class MethodIRInfo(
 
 /** Represents method descriptor.
   *
-  * @param objType type or type constructor descriptor
-  * @param name    method name
-  * @param stype   method signature type,
-  *                where `stype.tDom`` - argument type and
-  *                `stype.tRange` - method result type.
-  * @param methodId method code, it should be unique among methods of the same objType.
-  * @param costKind cost descriptor for this method
-  * @param irInfo  meta information connecting SMethod with ErgoTree (see [[MethodIRInfo]])
-  * @param docInfo optional human readable method description data
-  * @param costFunc optional specification of how the cost should be computed for the
-  *                 given method call (See ErgoTreeEvaluator.calcCost method).
+  * @param objType         type or type constructor descriptor
+  * @param name            method name
+  * @param stype           method signature type,
+  *                        where `stype.tDom`` - argument type and
+  *                        `stype.tRange` - method result type.
+  * @param methodId        method code, it should be unique among methods of the same objType.
+  * @param costKind        cost descriptor for this method
+  * @param explicitTypeArgs list of type parameters which require explicit
+  *                        serialization in [[MethodCall]]s (i.e for deserialize[T], getVar[T], getReg[T])
+  * @param irInfo          meta information connecting SMethod with ErgoTree (see [[MethodIRInfo]])
+  * @param docInfo         optional human readable method description data
+  * @param costFunc        optional specification of how the cost should be computed for the
+  *                        given method call (See ErgoTreeEvaluator.calcCost method).
+  * @param userDefinedInvoke optional custom method evaluation function
   */
 case class SMethod(
     objType: MethodsContainer,
     name: String,
     stype: SFunc,
     methodId: Byte,
     costKind: CostKind,
+    explicitTypeArgs: Seq[STypeVar],
     irInfo: MethodIRInfo,
     docInfo: Option[OperationInfo],
-    costFunc: Option[MethodCostFunc]) {
+    costFunc: Option[MethodCostFunc],
+    userDefinedInvoke: Option[SMethod.InvokeHandler]
+) {
 
   /** Operation descriptor of this method. */
   lazy val opDesc = MethodDesc(this)
 
+  /** Return true if this method has runtime type parameters */
+  def hasExplicitTypeArgs: Boolean = explicitTypeArgs.nonEmpty
+
   /** Finds and keeps the [[RMethod]] instance which corresponds to this method descriptor.
     * The lazy value is forced only if irInfo.javaMethod == None
     */
@@ -106,7 +115,12 @@ case class SMethod(
   /** Invoke this method on the given object with the arguments.
     * This is used for methods with FixedCost costKind. */
   def invokeFixed(obj: Any, args: Array[Any]): Any = {
-    javaMethod.invoke(obj, args.asInstanceOf[Array[AnyRef]]:_*)
+    userDefinedInvoke match {
+      case Some(h) =>
+        h(this, obj, args)
+      case None =>
+        javaMethod.invoke(obj, args.asInstanceOf[Array[AnyRef]]:_*)
+    }
   }
 
   // TODO optimize: avoid lookup when this SMethod is created via `specializeFor`
@@ -146,6 +160,11 @@ case class SMethod(
     m
   }
 
+  /** Create a new instance with the given user-defined invoke handler. */
+  def withUserDefinedInvoke(handler: SMethod.InvokeHandler): SMethod = {
+    copy(userDefinedInvoke = Some(handler))
+  }
+
   /** Create a new instance with the given stype. */
   def withSType(newSType: SFunc): SMethod = copy(stype = newSType)
 
@@ -255,6 +274,12 @@ object SMethod {
     */
   type InvokeDescBuilder = SFunc => Seq[SType]
 
+  /** Type of user-defined function which is called to handle method invocation.
+    * Instances of this type can be attached to [[SMethod]] instances.
+    * @see SNumericTypeMethods.ToBytesMethod
+    */
+  type InvokeHandler = (SMethod, Any, Array[Any]) => Any
+
   /** Return [[Method]] descriptor for the given `methodName` on the given `cT` type.
     * @param methodName the name of the method to lookup
     * @param cT the class where to search the methodName
@@ -284,10 +309,12 @@ object SMethod {
   /** Convenience factory method. */
   def apply(objType: MethodsContainer, name: String, stype: SFunc,
       methodId: Byte,
-      costKind: CostKind): SMethod = {
+      costKind: CostKind,
+      explicitTypeArgs: Seq[STypeVar] = Nil
+  ): SMethod = {
     SMethod(
-      objType, name, stype, methodId, costKind,
-      MethodIRInfo(None, None, None), None, None)
+      objType, name, stype, methodId, costKind, explicitTypeArgs,
+      MethodIRInfo(None, None, None), None, None, None)
   }
 
 

data/shared/src/main/scala/sigma/ast/SigmaPredef.scala

@@ -5,7 +5,6 @@ import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress}
 import scorex.util.encode.{Base16, Base58, Base64}
 import sigma.ast.SCollection.{SByteArray, SIntArray}
 import sigma.ast.SOption.SIntOption
-import sigma.ast.SigmaPropConstant
 import sigma.ast.syntax._
 import sigma.data.Nullable
 import sigma.exceptions.InvalidArguments
@@ -561,7 +560,7 @@ object SigmaPredef {
 
     val funcs: Map[String, PredefinedFunc] = globalFuncs ++ infixFuncs ++ unaryFuncs
 
-    /** WARNING: This operations are not used in frontend, and should be be used.
+    /** WARNING: This operations are not used in frontend, and should not be used.
       * They are used in SpecGen only the source of metadata for the corresponding ErgoTree nodes.
       */
     val specialFuncs: Map[String, PredefinedFunc] = Seq(

data/shared/src/main/scala/sigma/serialization/ErgoTreeSerializer.scala

@@ -287,6 +287,9 @@ class ErgoTreeSerializer {
     * allow to use serialized scripts as pre-defined templates.
     * See [[SubstConstants]] for details.
     *
+    * Note, this operation doesn't require (de)serialization of ErgoTree expression,
+    * thus it is more efficient than serialization roundtrip.
+    *
     * @param scriptBytes serialized ErgoTree with ConstantSegregationFlag set to 1.
     * @param positions   zero based indexes in ErgoTree.constants array which
     *                    should be replaced with new values
@@ -304,39 +307,62 @@ class ErgoTreeSerializer {
       s"expected positions and newVals to have the same length, got: positions: ${positions.toSeq},\n newVals: ${newVals.toSeq}")
     val r = SigmaSerializer.startReader(scriptBytes)
     val (header, _, constants, treeBytes) = deserializeHeaderWithTreeBytes(r)
-    val w = SigmaSerializer.startWriter()
-    w.put(header)
+    val nConstants = constants.length
+
+    val resBytes = if (VersionContext.current.isJitActivated) {
+      // need to measure the serialized size of the new constants
+      // by serializing them into a separate writer
+      val constW = SigmaSerializer.startWriter()
 
-    if (VersionContext.current.isJitActivated) {
       // The following `constants.length` should not be serialized when segregation is off
       // in the `header`, because in this case there is no `constants` section in the
       // ErgoTree serialization format. Thus, applying this `substituteConstants` for
       // non-segregated trees will return non-parsable ErgoTree bytes (when
       // `constants.length` is put in `w`).
       if (ErgoTree.isConstantSegregation(header)) {
-        w.putUInt(constants.length)
+        constW.putUInt(constants.length)
       }
 
       // The following is optimized O(nConstants + position.length) implementation
-      val nConstants = constants.length
       if (nConstants > 0) {
         val backrefs = getPositionsBackref(positions, nConstants)
         cfor(0)(_ < nConstants, _ + 1) { i =>
           val c = constants(i)
           val iPos = backrefs(i) // index to `positions`
           if (iPos == -1) {
             // no position => no substitution, serialize original constant
-            constantSerializer.serialize(c, w)
+            constantSerializer.serialize(c, constW)
           } else {
-            assert(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
+            require(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
             val newConst = newVals(iPos)
             require(c.tpe == newConst.tpe,
               s"expected new constant to have the same ${c.tpe} tpe, got ${newConst.tpe}")
-            constantSerializer.serialize(newConst, w)
+            constantSerializer.serialize(newConst, constW)
           }
         }
       }
+
+      val constBytes = constW.toBytes // nConstants + serialized new constants
+
+      // start composing the resulting tree bytes
+      val w = SigmaSerializer.startWriter()
+      w.put(header) // header byte
+
+      if (VersionContext.current.isV6SoftForkActivated) {
+        // fix in v6.0 to save tree size to respect size bit of the original tree
+        if (ErgoTree.hasSize(header)) {
+          val size = constBytes.length + treeBytes.length
+          w.putUInt(size)     // tree size
+        }
+      }
+
+      w.putBytes(constBytes)  // constants section
+      w.putBytes(treeBytes)   // tree section
+      w.toBytes
     } else {
+      val w = SigmaSerializer.startWriter()
+      w.put(header)
+
       // for v4.x compatibility we save constants.length here (see the above comment to
       // understand the consequences)
       w.putUInt(constants.length)
@@ -357,10 +383,12 @@ class ErgoTreeSerializer {
         case (c, _) =>
           constantSerializer.serialize(c, w)
       }
+
+      w.putBytes(treeBytes)
+      w.toBytes
     }
 
-    w.putBytes(treeBytes)
-    (w.toBytes, constants.length)
+    (resBytes, nConstants)
   }
 
 }

data/shared/src/main/scala/sigma/serialization/MethodCallSerializer.scala

@@ -1,13 +1,15 @@
 package sigma.serialization
 
 import sigma.ast.syntax._
-import sigma.ast.{MethodCall, SContextMethods, SMethod, SType, STypeSubst, Value, ValueCompanion}
+import sigma.ast.{MethodCall, SContextMethods, SMethod, SType, STypeSubst, STypeVar, Value, ValueCompanion}
 import sigma.util.safeNewArray
 import SigmaByteWriter._
 import debox.cfor
 import sigma.ast.SContextMethods.BlockchainContextMethodNames
 import sigma.serialization.CoreByteWriter.{ArgInfo, DataInfo}
 
+import scala.collection.compat.immutable.ArraySeq
+
 case class MethodCallSerializer(cons: (Value[SType], SMethod, IndexedSeq[Value[SType]], STypeSubst) => Value[SType])
   extends ValueSerializer[MethodCall] {
   override def opDesc: ValueCompanion = MethodCall
@@ -23,6 +25,10 @@ case class MethodCallSerializer(cons: (Value[SType], SMethod, IndexedSeq[Value[S
     w.putValue(mc.obj, objInfo)
     assert(mc.args.nonEmpty)
     w.putValues(mc.args, argsInfo, argsItemInfo)
+    mc.method.explicitTypeArgs.foreach { a =>
+      val tpe = mc.typeSubst(a)  // existence is checked in MethodCall constructor
+      w.putType(tpe)
+    }
   }
 
   /** The SMethod instances in STypeCompanions may have type STypeIdent in methods types,
@@ -43,9 +49,36 @@ case class MethodCallSerializer(cons: (Value[SType], SMethod, IndexedSeq[Value[S
     val obj = r.getValue()
     val args = r.getValues()
     val method = SMethod.fromIds(typeId, methodId)
-    val nArgs = args.length
 
-    val types: Seq[SType] =
+    val explicitTypes = if (method.hasExplicitTypeArgs) {
+      val nTypes = method.explicitTypeArgs.length
+      val res = safeNewArray[SType](nTypes)
+      cfor(0)(_ < nTypes, _ + 1) { i =>
+        res(i) = r.getType()
+      }
+      ArraySeq.unsafeWrapArray(res)
+    } else SType.EmptySeq
+
+    val explicitTypeSubst = method.explicitTypeArgs.zip(explicitTypes).toMap
+    val specMethod = getSpecializedMethodFor(method, explicitTypeSubst, obj, args)
+
+    var isUsingBlockchainContext = specMethod.objType == SContextMethods &&
+      BlockchainContextMethodNames.contains(method.name)
+    r.wasUsingBlockchainContext ||= isUsingBlockchainContext
+
+    cons(obj, specMethod, args, explicitTypeSubst)
+  }
+
+  def getSpecializedMethodFor(
+    methodTemplate: SMethod,
+    explicitTypeSubst: STypeSubst,
+    obj: SValue,
+    args: Seq[SValue]
+  ): SMethod = {
+    // TODO optimize: avoid repeated transformation of method type
+    val method = methodTemplate.withConcreteTypes(explicitTypeSubst)
+    val nArgs = args.length
+    val argTypes: Seq[SType] =
       if (nArgs == 0) SType.EmptySeq
       else {
         val types = safeNewArray[SType](nArgs)
@@ -55,12 +88,6 @@ case class MethodCallSerializer(cons: (Value[SType], SMethod, IndexedSeq[Value[S
         types
       }
 
-    val specMethod = method.specializeFor(obj.tpe, types)
-
-    var isUsingBlockchainContext = specMethod.objType == SContextMethods &&
-      BlockchainContextMethodNames.contains(method.name)
-    r.wasUsingBlockchainContext ||= isUsingBlockchainContext
-
-    cons(obj, specMethod, args, Map.empty)
+    method.specializeFor(obj.tpe, argTypes)
   }
 }