merging w. 6.0.0

core/shared/src/main/scala/sigma/VersionContext.scala

@@ -23,7 +23,7 @@ case class VersionContext(activatedVersion: Byte, ergoTreeVersion: Byte) {
   def isJitActivated: Boolean = activatedVersion >= JitActivationVersion
 
   /** @return true, if the activated script version of Ergo protocol on the network is
-   * including Evolution update. */
+   * including v6.0 update. */
   def isV6SoftForkActivated: Boolean = activatedVersion >= V6SoftForkVersion
 }
 

core/shared/src/main/scala/sigma/ast/SType.scala

@@ -7,7 +7,7 @@ import sigma.data.OverloadHack.Overloaded1
 import sigma.data.{CBigInt, Nullable, SigmaConstants}
 import sigma.reflection.{RClass, RMethod, ReflectionData}
 import sigma.util.Extensions.{IntOps, LongOps, ShortOps}
-import sigma.{AvlTree, BigInt, Box, Coll, Context, Evaluation, GroupElement, Header, PreHeader, SigmaDslBuilder, SigmaProp}
+import sigma.{AvlTree, BigInt, Box, Coll, Context, Evaluation, GroupElement, Header, PreHeader, SigmaDslBuilder, SigmaProp, VersionContext}
 
 import java.math.BigInteger
 
@@ -375,6 +375,7 @@ case object SByte extends SPrimType with SEmbeddable with SNumericType with SMon
     case s: Short => s.toByteExact
     case i: Int => i.toByteExact
     case l: Long => l.toByteExact
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toByte // toByteExact from int is called under the hood
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -396,6 +397,7 @@ case object SShort extends SPrimType with SEmbeddable with SNumericType with SMo
     case s: Short => s
     case i: Int => i.toShortExact
     case l: Long => l.toShortExact
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toShort // toShortExact from int is called under the hood
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -419,6 +421,7 @@ case object SInt extends SPrimType with SEmbeddable with SNumericType with SMono
     case s: Short => s.toInt
     case i: Int => i
     case l: Long => l.toIntExact
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toInt
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -444,6 +447,7 @@ case object SLong extends SPrimType with SEmbeddable with SNumericType with SMon
     case s: Short => s.toLong
     case i: Int => i.toLong
     case l: Long => l
+    case bi: BigInt if VersionContext.current.isV6SoftForkActivated => bi.toLong
     case _ => sys.error(s"Cannot downcast value $v to the type $this")
   }
 }
@@ -465,24 +469,24 @@ case object SBigInt extends SPrimType with SEmbeddable with SNumericType with SM
   override def numericTypeIndex: Int = 4
 
   override def upcast(v: AnyVal): BigInt = {
-    val bi = v match {
-      case x: Byte => BigInteger.valueOf(x.toLong)
-      case x: Short => BigInteger.valueOf(x.toLong)
-      case x: Int => BigInteger.valueOf(x.toLong)
-      case x: Long => BigInteger.valueOf(x)
+    v match {
+      case x: Byte => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Short => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Int => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Long => CBigInt(BigInteger.valueOf(x))
+      case x: BigInt if VersionContext.current.isV6SoftForkActivated => x
       case _ => sys.error(s"Cannot upcast value $v to the type $this")
     }
-    CBigInt(bi)
   }
   override def downcast(v: AnyVal): BigInt = {
-    val bi = v match {
-      case x: Byte => BigInteger.valueOf(x.toLong)
-      case x: Short => BigInteger.valueOf(x.toLong)
-      case x: Int => BigInteger.valueOf(x.toLong)
-      case x: Long => BigInteger.valueOf(x)
+    v match {
+      case x: Byte => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Short => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Int => CBigInt(BigInteger.valueOf(x.toLong))
+      case x: Long => CBigInt(BigInteger.valueOf(x))
+      case x: BigInt if VersionContext.current.isV6SoftForkActivated => x
       case _ => sys.error(s"Cannot downcast value $v to the type $this")
     }
-    CBigInt(bi)
   }
 }
 

data/shared/src/main/scala/org/ergoplatform/validation/ValidationRules.scala

@@ -155,20 +155,6 @@ object ValidationRules {
     override protected lazy val settings: SigmaValidationSettings = currentSettings
   }
 
-  object CheckMinimalErgoTreeVersion extends ValidationRule(1016,
-    "ErgoTree should have at least required version") with SoftForkWhenReplaced {
-    override protected lazy val settings: SigmaValidationSettings = currentSettings
-
-    final def apply(currentVersion: Byte, minVersion: Byte): Unit = {
-      checkRule()
-      if (currentVersion < minVersion) {
-        throwValidationException(
-          new SigmaException(s"ErgoTree should have at least $minVersion version, but was $currentVersion"),
-          Array(currentVersion, minVersion))
-      }
-    }
-  }
-
   val ruleSpecs: Seq[ValidationRule] = Seq(
     CheckDeserializedScriptType,
     CheckDeserializedScriptIsSigmaProp,
@@ -185,8 +171,7 @@ object ValidationRules {
     CheckHeaderSizeBit,
     CheckCostFuncOperation,
     CheckPositionLimit,
-    CheckLoopLevelInCostFunction,
-    CheckMinimalErgoTreeVersion
+    CheckLoopLevelInCostFunction
   )
 
   /** Validation settings that correspond to the current version of the ErgoScript implementation.

data/shared/src/main/scala/sigma/ast/SMethod.scala

@@ -63,6 +63,7 @@ case class MethodIRInfo(
   * @param docInfo         optional human readable method description data
   * @param costFunc        optional specification of how the cost should be computed for the
   *                        given method call (See ErgoTreeEvaluator.calcCost method).
+  * @param userDefinedInvoke optional custom method evaluation function
   */
 case class SMethod(
     objType: MethodsContainer,
@@ -73,7 +74,9 @@ case class SMethod(
     explicitTypeArgs: Seq[STypeVar],
     irInfo: MethodIRInfo,
     docInfo: Option[OperationInfo],
-    costFunc: Option[MethodCostFunc]) {
+    costFunc: Option[MethodCostFunc],
+    userDefinedInvoke: Option[SMethod.InvokeHandler]
+) {
 
   /** Operation descriptor of this method. */
   lazy val opDesc = MethodDesc(this)
@@ -114,7 +117,12 @@ case class SMethod(
   /** Invoke this method on the given object with the arguments.
     * This is used for methods with FixedCost costKind. */
   def invokeFixed(obj: Any, args: Array[Any]): Any = {
-    javaMethod.invoke(obj, args.asInstanceOf[Array[AnyRef]]:_*)
+    userDefinedInvoke match {
+      case Some(h) =>
+        h(this, obj, args)
+      case None =>
+        javaMethod.invoke(obj, args.asInstanceOf[Array[AnyRef]]:_*)
+    }
   }
 
   // TODO optimize: avoid lookup when this SMethod is created via `specializeFor`
@@ -154,6 +162,11 @@ case class SMethod(
     m
   }
 
+  /** Create a new instance with the given user-defined invoke handler. */
+  def withUserDefinedInvoke(handler: SMethod.InvokeHandler): SMethod = {
+    copy(userDefinedInvoke = Some(handler))
+  }
+
   /** Create a new instance with the given stype. */
   def withSType(newSType: SFunc): SMethod = copy(stype = newSType)
 
@@ -263,6 +276,12 @@ object SMethod {
     */
   type InvokeDescBuilder = SFunc => Seq[SType]
 
+  /** Type of user-defined function which is called to handle method invocation.
+    * Instances of this type can be attached to [[SMethod]] instances.
+    * @see SNumericTypeMethods.ToBytesMethod
+    */
+  type InvokeHandler = (SMethod, Any, Array[Any]) => Any
+
   /** Return [[Method]] descriptor for the given `methodName` on the given `cT` type.
     * @param methodName the name of the method to lookup
     * @param cT the class where to search the methodName
@@ -297,7 +316,7 @@ object SMethod {
   ): SMethod = {
     SMethod(
       objType, name, stype, methodId, costKind, explicitTypeArgs,
-      MethodIRInfo(None, None, None), None, None)
+      MethodIRInfo(None, None, None), None, None, None)
   }
 
 

data/shared/src/main/scala/sigma/ast/SigmaPredef.scala

@@ -5,7 +5,6 @@ import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress}
 import scorex.util.encode.{Base16, Base58, Base64}
 import sigma.ast.SCollection.{SByteArray, SIntArray}
 import sigma.ast.SOption.SIntOption
-import sigma.ast.SigmaPropConstant
 import sigma.ast.syntax._
 import sigma.data.Nullable
 import sigma.exceptions.InvalidArguments

data/shared/src/main/scala/sigma/ast/methods.scala

@@ -1,7 +1,6 @@
 package sigma.ast
 
 import org.ergoplatform._
-import org.ergoplatform.validation.ValidationRules.CheckMinimalErgoTreeVersion
 import org.ergoplatform.validation._
 import sigma._
 import sigma.ast.SCollection.{SBooleanArray, SBoxArray, SByteArray, SByteArray2, SHeaderArray}
@@ -1534,7 +1533,6 @@ case object SGlobalMethods extends MonoTypeMethods {
     */
   def serialize_eval(mc: MethodCall, G: SigmaDslBuilder, value: SType#WrappedType)
       (implicit E: ErgoTreeEvaluator): Coll[Byte] = {
-    CheckMinimalErgoTreeVersion(E.context.currentErgoTreeVersion, VersionContext.V6SoftForkVersion)
 
     E.addCost(SigmaByteWriter.StartWriterCost)
 

data/shared/src/main/scala/sigma/ast/values.scala

@@ -1298,8 +1298,10 @@ case class MethodCall(
     method: SMethod,
     args: IndexedSeq[Value[SType]],
     typeSubst: Map[STypeVar, SType]) extends Value[SType] {
+
   require(method.explicitTypeArgs.forall(tyArg => typeSubst.contains(tyArg)),
     s"Generic method call should have concrete type for each explicit type parameter, but was: $this")
+
   override def companion = if (args.isEmpty) PropertyCall else MethodCall
 
   override def opType: SFunc = SFunc(obj.tpe +: args.map(_.tpe), tpe)

data/shared/src/main/scala/sigma/serialization/ErgoTreeSerializer.scala

@@ -287,6 +287,9 @@ class ErgoTreeSerializer {
     * allow to use serialized scripts as pre-defined templates.
     * See [[SubstConstants]] for details.
     *
+    * Note, this operation doesn't require (de)serialization of ErgoTree expression,
+    * thus it is more efficient than serialization roundtrip.
+    *
     * @param scriptBytes serialized ErgoTree with ConstantSegregationFlag set to 1.
     * @param positions   zero based indexes in ErgoTree.constants array which
     *                    should be replaced with new values
@@ -304,39 +307,62 @@ class ErgoTreeSerializer {
       s"expected positions and newVals to have the same length, got: positions: ${positions.toSeq},\n newVals: ${newVals.toSeq}")
     val r = SigmaSerializer.startReader(scriptBytes)
     val (header, _, constants, treeBytes) = deserializeHeaderWithTreeBytes(r)
-    val w = SigmaSerializer.startWriter()
-    w.put(header)
+    val nConstants = constants.length
+
+    val resBytes = if (VersionContext.current.isJitActivated) {
+      // need to measure the serialized size of the new constants
+      // by serializing them into a separate writer
+      val constW = SigmaSerializer.startWriter()
 
-    if (VersionContext.current.isJitActivated) {
       // The following `constants.length` should not be serialized when segregation is off
       // in the `header`, because in this case there is no `constants` section in the
       // ErgoTree serialization format. Thus, applying this `substituteConstants` for
       // non-segregated trees will return non-parsable ErgoTree bytes (when
       // `constants.length` is put in `w`).
       if (ErgoTree.isConstantSegregation(header)) {
-        w.putUInt(constants.length)
+        constW.putUInt(constants.length)
       }
 
       // The following is optimized O(nConstants + position.length) implementation
-      val nConstants = constants.length
       if (nConstants > 0) {
         val backrefs = getPositionsBackref(positions, nConstants)
         cfor(0)(_ < nConstants, _ + 1) { i =>
           val c = constants(i)
           val iPos = backrefs(i) // index to `positions`
           if (iPos == -1) {
             // no position => no substitution, serialize original constant
-            constantSerializer.serialize(c, w)
+            constantSerializer.serialize(c, constW)
           } else {
-            assert(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
+            require(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
             val newConst = newVals(iPos)
             require(c.tpe == newConst.tpe,
               s"expected new constant to have the same ${c.tpe} tpe, got ${newConst.tpe}")
-            constantSerializer.serialize(newConst, w)
+            constantSerializer.serialize(newConst, constW)
           }
         }
       }
+
+      val constBytes = constW.toBytes // nConstants + serialized new constants
+
+      // start composing the resulting tree bytes
+      val w = SigmaSerializer.startWriter()
+      w.put(header) // header byte
+
+      if (VersionContext.current.isV6SoftForkActivated) {
+        // fix in v6.0 to save tree size to respect size bit of the original tree
+        if (ErgoTree.hasSize(header)) {
+          val size = constBytes.length + treeBytes.length
+          w.putUInt(size)     // tree size
+        }
+      }
+
+      w.putBytes(constBytes)  // constants section
+      w.putBytes(treeBytes)   // tree section
+      w.toBytes
     } else {
+      val w = SigmaSerializer.startWriter()
+      w.put(header)
+
       // for v4.x compatibility we save constants.length here (see the above comment to
       // understand the consequences)
       w.putUInt(constants.length)
@@ -357,10 +383,12 @@ class ErgoTreeSerializer {
         case (c, _) =>
           constantSerializer.serialize(c, w)
       }
+
+      w.putBytes(treeBytes)
+      w.toBytes
     }
 
-    w.putBytes(treeBytes)
-    (w.toBytes, constants.length)
+    (resBytes, nConstants)
   }
 
 }

interpreter/shared/src/main/scala/sigmastate/interpreter/Interpreter.scala

@@ -247,7 +247,12 @@ trait Interpreter {
       val currCost = addCostChecked(context.initCost, deserializeSubstitutionCost, context.costLimit)
       val context1 = context.withInitCost(currCost).asInstanceOf[CTX]
       val (propTree, context2) = trySoftForkable[(SigmaPropValue, CTX)](whenSoftFork = (TrueSigmaProp, context1)) {
-        applyDeserializeContextJITC(context, prop)
+        // Before ErgoTree V3 the deserialization cost was not added to the total cost
+        applyDeserializeContextJITC(if (VersionContext.current.activatedVersion >= VersionContext.V6SoftForkVersion) {
+          context1
+        } else {
+          context
+        }, prop)
       }
 
       // here we assume that when `propTree` is TrueProp then `reduceToCrypto` always succeeds

interpreter/shared/src/test/scala/sigma/serialization/MethodCallSerializerSpecification.scala

@@ -2,7 +2,6 @@ package sigma.serialization
 
 import sigma.VersionContext
 import sigma.ast.SCollection.SByteArray
-import sigma.ast.SType.tT
 import sigma.ast._
 import sigma.validation.ValidationException
 

sc/shared/src/main/scala/sigma/compiler/phases/SigmaTyper.scala

@@ -139,7 +139,8 @@ class SigmaTyper(val builder: SigmaBuilder,
       obj.tpe match {
         case p: SProduct =>
           MethodsContainer.getMethod(p, n) match {
-            case Some(method @ SMethod(_, _, genFunTpe @ SFunc(_, _, _), _, _, _, _, _, _)) =>
+            case Some(method: SMethod) =>
+              val genFunTpe = method.stype
               val subst = Map(genFunTpe.tpeParams.head.ident -> rangeTpe)
               val concrFunTpe = applySubst(genFunTpe, subst)
               val expectedArgs = concrFunTpe.asFunc.tDom.tail

sc/shared/src/test/scala/sigma/LanguageSpecificationBase.scala

@@ -1,10 +1,11 @@
 package sigma
 
 import org.scalatest.BeforeAndAfterAll
-import sigma.ast.JitCost
-import sigma.eval.{EvalSettings, Profiler}
+import sigma.ast.{Apply, FixedCostItem, FuncValue, GetVar, JitCost, OptionGet, ValUse}
+import sigma.eval.{CostDetails, EvalSettings, Profiler}
 import sigmastate.CompilerCrossVersionProps
 import sigmastate.interpreter.CErgoTreeEvaluator
+
 import scala.util.Success
 
 /** Base class for language test suites (one suite for each language version: 5.0, 6.0, etc.)
@@ -123,4 +124,23 @@ abstract class LanguageSpecificationBase extends SigmaDslTesting
     prepareSamples[(PreHeader, PreHeader)]
     prepareSamples[(Header, Header)]
   }
+
+  ///=====================================================
+  ///         CostDetails shared among test cases
+  ///-----------------------------------------------------
+  val traceBase = Array(
+    FixedCostItem(Apply),
+    FixedCostItem(FuncValue),
+    FixedCostItem(GetVar),
+    FixedCostItem(OptionGet),
+    FixedCostItem(FuncValue.AddToEnvironmentDesc, FuncValue.AddToEnvironmentDesc_CostKind),
+    FixedCostItem(ValUse)
+  )
+
+  /** Helper method to create the given expected results for all tree versions. */
+  def expectedSuccessForAllTreeVersions[A](value: A, cost: Int, costDetails: CostDetails) = {
+    val res = ExpectedResult(Success(value), Some(cost)) -> Some(costDetails)
+    Seq(0, 1, 2, 3).map(version => version -> res)
+  }
+
 }