Merge remote-tracking branch 'origin/master' into v5.0.1-RC

CONTRIBUTING.md

@@ -1,3 +1,13 @@
+## Building
+
+Clone the repository and then run tests.
+
+```shell
+git clone git@github.com:ScorexFoundation/sigmastate-interpreter.git
+cd sigmastate-interpreter
+sbt test
+```
+
 ## Releasing
 To publish release version to Sonatype, do the following:
 - make a tag with version number `vX.Y.Z` (used by `sbt-dynver` to set `version` in `build.sbt`);

README.md

@@ -111,6 +111,17 @@ libraryDependencies += "org.scorexfoundation" %% "sigma-state" % "4.0.3"
 | sigma-library | Implementation of graph IR nodes for Sigma types   |
 | sigmastate | Implementation ErgoTree, Interpreter and cost estimation   |
 
+## Acknowledgments
+
+We thank JetBrains for [supporting](https://www.jetbrains.com/buy/opensource/) this project since 2021 by providing All Products Pack subscription.
+
+<img src="https://www.yourkit.com/images/yklogo.png"/>
+
+We thank YourKit for support of open source projects with its full-featured Java Profiler.
+YourKit, LLC is the creator of <a href="https://www.yourkit.com/java/profiler/">YourKit Java Profiler</a>
+and <a href="https://www.yourkit.com/.net/profiler/">YourKit .NET Profiler</a>,
+innovative and intelligent tools for profiling Java and .NET applications.
+
 ## References
 
 - [Ergo Site](https://ergoplatform.org/en/)

sigmastate/src/main/scala/org/ergoplatform/ErgoBoxCandidate.scala

@@ -3,7 +3,7 @@ package org.ergoplatform
 import java.util
 import org.ergoplatform.ErgoBox._
 import org.ergoplatform.settings.ErgoAlgos
-import scorex.util.{bytesToId, ModifierId}
+import scorex.util.{ModifierId, bytesToId}
 import sigmastate.Values._
 import sigmastate._
 import sigmastate.SType.AnyOps
@@ -16,7 +16,7 @@ import sigmastate.serialization.ErgoTreeSerializer.DefaultSerializer
 import sigmastate.util.safeNewArray
 import spire.syntax.all.cfor
 
-import scala.collection.immutable
+import scala.collection.{immutable, mutable}
 import scala.runtime.ScalaRunTime
 
 /**
@@ -100,12 +100,17 @@ class ErgoBoxCandidate(val value: Long,
     s"tokens: (${additionalTokens.map(t => ErgoAlgos.encode(t._1) + ":" + t._2).toArray.mkString(", ")}), " +
     s"$additionalRegisters, creationHeight: $creationHeight)"
 
-  /** Additional tokens stored in the box. */
-  lazy val tokens: Map[ModifierId, Long] =
-    additionalTokens
-      .toArray
-      .map(t => bytesToId(t._1) -> t._2)
-      .toMap
+  /** Additional tokens stored in the box, merged into a Map.
+    * This method is not used in ErgoTree and serialization, not part of consensus.
+    */
+  lazy val tokens: Map[ModifierId, Long] = {
+    val merged = new mutable.HashMap[ModifierId, Long]
+    additionalTokens.foreach { case (id, amount) =>
+      val mId = bytesToId(id)
+      merged.put(mId, java7.compat.Math.addExact(merged.getOrElse(mId, 0L), amount))
+    }
+    merged.toMap
+  }
 }
 
 object ErgoBoxCandidate {
@@ -188,13 +193,20 @@ object ErgoBoxCandidate {
       val value = r.getULong()                  // READ
       val tree = DefaultSerializer.deserializeErgoTree(r, SigmaSerializer.MaxPropositionSize)  // READ
       val creationHeight = r.getUIntExact       // READ
+      // NO-FORK: ^ in v5.x getUIntExact may throw Int overflow exception
+      // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+      // and ErgoBoxCandidate with negative creation height is created, which is then invalidated
+      // during transaction validation. See validation rule # 122 in the Ergo node (ValidationRules.scala)
       val nTokens = r.getUByte()                // READ
       val tokenIds = safeNewArray[Array[Byte]](nTokens)
       val tokenAmounts = safeNewArray[Long](nTokens)
       if (digestsInTx != null) {
         val nDigests = digestsInTx.length
         cfor(0)(_ < nTokens, _ + 1) { i =>
           val digestIndex = r.getUIntExact    // READ
+          // NO-FORK: in v5.x getUIntExact throws Int overflow exception
+          // in v4.x r.getUInt().toInt is used and may return negative Int in which case
+          // the error below is thrown
           if (digestIndex < 0 || digestIndex >= nDigests)
             sys.error(s"failed to find token id with index $digestIndex")
           val amount = r.getULong()           // READ

sigmastate/src/main/scala/org/ergoplatform/ErgoLikeTransaction.scala

@@ -122,9 +122,6 @@ object ErgoLikeTransactionSerializer extends SigmaSerializer[ErgoLikeTransaction
       w.putBytes(input.boxId)
     }
     // Serialize distinct ids of tokens in transaction outputs.
-    // This optimization is crucial to allow up to MaxTokens (== 255) in a box.
-    // Without it total size of all token ids 255 * 32 = 8160,
-    // way beyond MaxBoxSize (== 4K)
     val tokenIds = tx.outputCandidates.toColl
       .flatMap(box => box.additionalTokens.map(t => t._1))
 
@@ -162,6 +159,9 @@ object ErgoLikeTransactionSerializer extends SigmaSerializer[ErgoLikeTransaction
 
     // parse distinct ids of tokens in transaction outputs
     val tokensCount = r.getUIntExact
+    // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+    // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+    // in which case the array allocation will throw NegativeArraySizeException
     val tokens = safeNewArray[Array[Byte]](tokensCount)
     cfor(0)(_ < tokensCount, _ + 1) { i =>
       tokens(i) = r.getBytes(TokenId.size)

sigmastate/src/main/scala/org/ergoplatform/validation/SigmaValidationSettingsSerializer.scala

@@ -4,6 +4,7 @@ import sigmastate.serialization.SigmaSerializer
 import sigmastate.utils.{SigmaByteReader, SigmaByteWriter}
 import scalan.util.Extensions.{IntOps,LongOps}
 
+// TODO v5.x: remove unused class and related json encoders
 /** The rules are serialized ordered by ruleId.
   * This serializer preserves roundtrip identity `deserialize(serialize(_)) = identity`
   * however it may not preserve `serialize(deserialize(_)) = identity` */
@@ -19,7 +20,8 @@ object SigmaValidationSettingsSerializer extends SigmaSerializer[SigmaValidation
   }
 
   override def parse(r: SigmaByteReader): SigmaValidationSettings = {
-    val nRules = r.getUIntExact
+    val nRules = r.getUInt().toInt
+    // Note, when nRules < 0 as a result of Int overflow, the loop is empty
     val parsed = (0 until nRules).map { _ =>
       val ruleId = r.getUShort().toShortExact
       val status = RuleStatusSerializer.parse(r)

sigmastate/src/main/scala/sigmastate/AvlTreeData.scala

@@ -92,8 +92,11 @@ object AvlTreeData {
     override def parse(r: SigmaByteReader): AvlTreeData = {
       val digest = r.getBytes(DigestSize)
       val tf = AvlTreeFlags(r.getByte())
-      val keyLength = r.getUIntExact
-      val valueLengthOpt = r.getOption(r.getUIntExact)
+      val keyLength = r.getUInt().toInt
+      val valueLengthOpt = r.getOption(r.getUInt().toInt)
+      // Note, when keyLength and valueLengthOpt < 0 as a result of Int overflow,
+      // the deserializer succeeds with invalid AvlTreeData
+      // but still some AvlTree operations (remove_eval, update_eval, contains_eval) won't throw
       AvlTreeData(ADDigest @@ digest, tf, keyLength, valueLengthOpt)
     }
   }

sigmastate/src/main/scala/sigmastate/Values.scala

@@ -853,19 +853,26 @@ object Values {
     def tpe = SBox
   }
 
-  // TODO refactor: only Constant make sense to inherit from EvaluatedValue
-
   /** ErgoTree node which converts a collection of expressions into a tuple of data values
     * of different types. Each data value of the resulting collection is obtained by
     * evaluating the corresponding expression in `items`. All items may have different
     * types.
     *
     * @param items source collection of expressions
     */
-  case class Tuple(items: IndexedSeq[Value[SType]]) extends Value[STuple] {
+  case class Tuple(items: IndexedSeq[Value[SType]])
+      extends EvaluatedValue[STuple]  // note, this superclass is required as Tuple can be in a register
+         with EvaluatedCollection[SAny.type, STuple] {
     override def companion = Tuple
     override lazy val tpe = STuple(items.map(_.tpe))
     override def opType: SFunc = ???
+    override def elementType: SAny.type = SAny
+
+    override lazy val value = {
+      val xs = items.cast[EvaluatedValue[SAny.type]].map(_.value)
+      Colls.fromArray(xs.toArray(SAny.classTag.asInstanceOf[ClassTag[SAny.WrappedType]]))(RType.AnyType)
+    }
+
     protected final override def eval(env: DataEnv)(implicit E: ErgoTreeEvaluator): Any = {
       // in v5.0 version we support only tuples of 2 elements to be equivalent with v4.x
       if (items.length != 2)

sigmastate/src/main/scala/sigmastate/serialization/BlockValueSerializer.scala

@@ -26,8 +26,12 @@ case class BlockValueSerializer(cons: (IndexedSeq[BlockItem], Value[SType]) => V
 
   override def parse(r: SigmaByteReader): Value[SType] = {
     val itemsSize = r.getUIntExact
-    val values: IndexedSeq[BlockItem] = if (itemsSize == 0)
+    // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+    // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+    // in which case the array allocation will throw NegativeArraySizeException
+    val values: IndexedSeq[BlockItem] = if (itemsSize == 0) {
       BlockItem.EmptySeq
+    }
     else {
       // HOTSPOT:: allocate new array only if it is not empty
       val buf = safeNewArray[BlockItem](itemsSize)

sigmastate/src/main/scala/sigmastate/serialization/ConstantPlaceholderSerializer.scala

@@ -15,6 +15,9 @@ case class ConstantPlaceholderSerializer(cons: (Int, SType) => Value[SType])
 
   override def parse(r: SigmaByteReader): Value[SType] = {
     val id = r.getUIntExact
+    // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+    // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+    // in which case the constantStore.get will throw ArrayIndexOutOfBoundsException
     val constant = r.constantStore.get(id)
     if (r.resolvePlaceholdersToConstants)
       constant

sigmastate/src/main/scala/sigmastate/serialization/DataSerializer.scala

@@ -100,6 +100,9 @@ object DataSerializer {
       case SLong => r.getLong()
       case SString =>
         val size = r.getUIntExact
+        // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+        // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+        // in which case the getBytes will throw NegativeArraySizeException
         val bytes = r.getBytes(size)
         new String(bytes, StandardCharsets.UTF_8)
       case SBigInt =>

sigmastate/src/main/scala/sigmastate/serialization/ErgoTreeSerializer.scala

@@ -207,7 +207,20 @@ class ErgoTreeSerializer {
     val header = r.getByte()
     CheckHeaderSizeBit(header)
     val sizeOpt = if (ErgoTree.hasSize(header)) {
-      Some(r.getUIntExact)
+      val size = r.getUInt().toInt
+      // Note, when size < 0 as a result of Int overflow nothing happens here and later
+      // when deserialization proceeds normally as sizeOpt is not used on this pass.
+      // However, when ValidationException is thrown in deserializeErgoTree this negative
+      // tree size value will be used in
+      // val val numBytes = bodyPos - startPos + treeSize
+      //            r.position = startPos
+      //            val bytes = r.getBytes(numBytes) = bodyPos - startPos + treeSize
+      // val bytes = r.getBytes(numBytes)
+      // If numBytes < 0 then it throws on getBytes and the whole deserialization fails
+      // On the other hand if numBytes >= 0 then UnparsedErgoTree will be created.
+      // The Reader however will be in some unpredictable state, as not all ErgoTree bytes
+      // are consumed.
+      Some(size)
     } else
       None
     (header, sizeOpt)
@@ -221,7 +234,10 @@ class ErgoTreeSerializer {
   private def deserializeConstants(header: Byte, r: SigmaByteReader): IndexedSeq[Constant[SType]] = {
     val constants: IndexedSeq[Constant[SType]] =
       if (ErgoTree.isConstantSegregation(header)) {
-        val nConsts = r.getUIntExact
+        val nConsts = r.getUInt().toInt
+        // Note, when nConsts < 0 as a result of Int overflow, the empty seq is returned
+        // deserialization will succeed
+
         if (nConsts > 0) {
           // HOTSPOT:: allocate new array only if it is not empty
           val res = safeNewArray[Constant[SType]](nConsts)

sigmastate/src/main/scala/sigmastate/serialization/FuncValueSerializer.scala

@@ -28,9 +28,15 @@ case class FuncValueSerializer(cons: (IndexedSeq[(Int, SType)], Value[SType]) =>
 
   override def parse(r: SigmaByteReader): Value[SType] = {
     val argsSize = r.getUIntExact
+    // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+    // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+    // in which case the array allocation will throw NegativeArraySizeException
     val args = safeNewArray[(Int, SType)](argsSize)
     cfor(0)(_ < argsSize, _ + 1) { i =>
-      val id = r.getUIntExact
+      val id = r.getUInt().toInt
+      // Note, when id < 0 as a result of Int overflow, the r.valDefTypeStore(id) won't throw
+      // More over evaluation of such FuncValue will not throw either if the body contains
+      // ValUse with the same negative id
       val tpe = r.getType()
       r.valDefTypeStore(id) = tpe
       args(i) = (id, tpe)

sigmastate/src/main/scala/sigmastate/serialization/ValDefSerializer.scala

@@ -31,6 +31,10 @@ case class ValDefSerializer(override val opDesc: ValueCompanion) extends ValueSe
 
   override def parse(r: SigmaByteReader): Value[SType] = {
     val id = r.getUIntExact
+    // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+    // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+    // When id < 0 as a result of Int overflow, the r.valDefTypeStore(id) won't throw
+    // but ValDef constructor fails on require(id >= 0, "id must be >= 0")
     val tpeArgs: Seq[STypeVar] = opCode match {
       case FunDefCode =>
         val nTpeArgs = r.getByte()

sigmastate/src/main/scala/sigmastate/serialization/ValUseSerializer.scala

@@ -12,7 +12,12 @@ case class ValUseSerializer(cons: (Int, SType) => Value[SType]) extends ValueSer
   }
 
   override def parse(r: SigmaByteReader): Value[SType] = {
-    val id = r.getUIntExact
+    val id = r.getUInt.toInt
+    // Note, when id < 0 as a result of Int overflow, the r.valDefTypeStore(id) won't throw
+    // and also ValUse node will be created, but then its evaluation will throw (because
+    // there will be no ValDef with negative id in the env.
+    // However, in general, there is no guarantee that this ValUse will ever be executed
+    // as it may be in an `if` branch.
     val tpe = r.valDefTypeStore(id)
     cons(id, tpe)
   }

sigmastate/src/main/scala/sigmastate/serialization/transformers/SigmaTransformerSerializer.scala

@@ -19,6 +19,9 @@ case class SigmaTransformerSerializer[I <: SigmaPropValue, O <: SigmaPropValue]
 
   override def parse(r: SigmaByteReader): SigmaPropValue = {
     val itemsSize = r.getUIntExact
+    // NO-FORK: in v5.x getUIntExact may throw Int overflow exception
+    // in v4.x r.getUInt().toInt is used and may return negative Int instead of the overflow
+    // in which case the array allocation will throw NegativeArraySizeException
     val res = safeNewArray[SigmaPropValue](itemsSize)
     cfor(0)(_ < itemsSize, _ + 1) { i =>
       res(i) = r.getValue().asInstanceOf[SigmaPropValue]

sigmastate/src/test/scala/org/ergoplatform/ErgoLikeTransactionSpec.scala

@@ -3,11 +3,12 @@ package org.ergoplatform
 import org.ergoplatform.ErgoBox.TokenId
 import org.ergoplatform.settings.ErgoAlgos
 import scorex.crypto.hash.Digest32
+import scorex.util.encode.Base16
 import scorex.util.{Random, ModifierId}
 import sigmastate.SCollection.SByteArray
-import sigmastate.{SSigmaProp, SPair, SInt, TrivialProp, SType}
+import sigmastate.{SSigmaProp, TrivialProp, SType, SPair, SInt}
 import sigmastate.Values._
-import sigmastate.interpreter.{ProverResult, ContextExtension}
+import sigmastate.interpreter.{ContextExtension, ProverResult}
 import sigmastate.serialization.SigmaSerializer
 import sigmastate.eval._
 import sigmastate.eval.Extensions._
@@ -45,6 +46,7 @@ class ErgoLikeTransactionSpec extends SigmaDslTesting {
       100,
       Coll(
         Digest32 @@ (ErgoAlgos.decodeUnsafe(token1)) -> 10000000L,
+        Digest32 @@ (ErgoAlgos.decodeUnsafe(token1)) -> 500L,
         Digest32 @@ (ErgoAlgos.decodeUnsafe(token2)) -> 500L
       )
     )
@@ -69,6 +71,7 @@ class ErgoLikeTransactionSpec extends SigmaDslTesting {
     b1.tokens shouldBe expectedTokens
     b1_clone.tokens shouldBe expectedTokens
     b3.tokens shouldBe expectedTokens
+    b2.tokens shouldBe Map[ModifierId, Long](ModifierId @@ token1 -> 10000500L, ModifierId @@ token2 -> 500L)
 
     assertResult(true)(b1.hashCode() == b1.hashCode())
     assertResult(true)(b1 == b1)
@@ -300,4 +303,12 @@ class ErgoLikeTransactionSpec extends SigmaDslTesting {
       }
     }
   }
+
+  property("Tuple in register test vector") {
+    val txId = "b5fd96c9f8c1ff436d609a225a12377c6873b890a145fc05f4099845b89315e5"
+    val txHex = "0278c93ee55eec066ec06290524dc01773440fbaff644bd181ab4334f97083863738644bf834de150b9abd077cf52ecb4892536ab6ad6ef8505fb932d9ca7b9fb211fc8294618723eb4909d4553f4d419762332f68865b1d703d00faae81c0e57683108c55647c5837a4fedfb52ffde04db74a8b4b77e37ab81afc3828c904862db6134e0f859ff5f62e4a5cb9da6b15ea1cd1a11dc932899d7b99e9073034ccd4e7c54fa9790d8ee356dce22ee151b044561c96000000038094ebdc031007040205c80105c8010500040004000e1f3faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf0d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a70856d605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057e8c7204010573019d9c72057e8c72040205730294e4c672020601720393e4c672020705720693e4c672020856720493e4c67202090ec5a7929c720672057207917207730395ef720393b1db630872027304d801d608b2db63087202730500ed938c7208017306938c7208027206d18f34000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405dceeaa0401010564860202660263c0edcea7090008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04d18f340000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304d18f340000"
+    val txBytes = Base16.decode(txHex).get
+    val tx = ErgoLikeTransactionSerializer.fromBytes(txBytes)
+    tx.id shouldBe txId
+  }
 }

sigmastate/src/test/scala/sigmastate/helpers/NegativeTesting.scala

@@ -50,7 +50,7 @@ trait NegativeTesting extends Matchers {
     */
   def exceptionLike[E <: Throwable : ClassTag]
       (msgParts: String*): Throwable => Boolean = {
-    case t: E => msgParts.forall(t.getMessage.contains(_))
+    case t: E => msgParts.forall(part => t.getMessage != null && t.getMessage.contains(part))
     case _ => false
   }