Release Candidate v5.0

.github/workflows/ci.yml

@@ -61,6 +61,7 @@ jobs:
           fail_ci_if_error: true
 
       - name: Publish a snapshot ${{ github.ref }}
+        if: env.HAS_SECRETS == 'true'
         run: sbt ++${{ matrix.scala }} publish
         env:
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}

README.md

@@ -1,3 +1,6 @@
+[![CI](https://github.com/ScorexFoundation/sigmastate-interpreter/actions/workflows/ci.yml/badge.svg)](https://github.com/ScorexFoundation/sigmastate-interpreter/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/ScorexFoundation/sigmastate-interpreter/branch/develop/graph/badge.svg?token=HNu2ZEOoV6)](https://codecov.io/gh/ScorexFoundation/sigmastate-interpreter)
+
 # ErgoScript compiler and ErgoTree interpreter 
 
 This repository contains implementations of ErgoScript compiler and ErgoTree

common/src/main/scala/scalan/util/CollectionUtil.scala

@@ -12,10 +12,11 @@ import scala.reflect.ClassTag
 
 object CollectionUtil {
 
+  /** @deprecated shouldn't be used other than for backwards compatibility with v3.x, v4.x. */
   def concatArrays[T](xs: Array[T], ys: Array[T]): Array[T] = {
     val len = xs.length + ys.length
     val result = (xs match {
-      case arr: Array[AnyRef] => new Array[AnyRef](len)
+      case arr: Array[AnyRef] => new Array[AnyRef](len) // creates an array with invalid type descriptor (i.e. when T == Tuple2)
       case arr: Array[Byte] => new Array[Byte](len)
       case arr: Array[Short] => new Array[Short](len)
       case arr: Array[Int] => new Array[Int](len)
@@ -30,6 +31,21 @@ object CollectionUtil {
     result
   }
 
+  /** Concatenates two arrays into a new resulting array.
+    * All items of both arrays are copied to the result using System.arraycopy.
+    * This method takes ClassTag to create proper resulting array.
+    * Can be used in v5.0 and above.
+    */
+  def concatArrays_v5[T:ClassTag](arr1: Array[T], arr2: Array[T]): Array[T] = {
+    val l1 = arr1.length
+    val l2 = arr2.length
+    val length: Int = l1 + l2
+    val result: Array[T] = new Array[T](length)
+    System.arraycopy(arr1, 0, result, 0, l1)
+    System.arraycopy(arr2, 0, result, l1, l2)
+    result
+  }
+
   def deepHashCode[T](arr: Array[T]): Int = arr match {
     case arr: Array[AnyRef] => util.Arrays.deepHashCode(arr)
     case arr: Array[Byte] => util.Arrays.hashCode(arr)
@@ -50,6 +66,10 @@ object CollectionUtil {
       }
     }
 
+  /** Group the given sequence of pairs by first values as keys.
+    * @param kvs sequence of values which is traversed once
+    * @return a multimap with ArrayBuffer of values for each key.
+    */
   def createMultiMap[K,V](kvs: GenIterable[(K,V)]): Map[K, ArrayBuffer[V]] = {
     val res = HashMap.empty[K, ArrayBuffer[V]]
     kvs.foreach { case (k, v) =>
@@ -62,12 +82,17 @@ object CollectionUtil {
     res.toMap
   }
 
-  // TODO optimize: using cfor and avoiding allocations
+  /** Perform relational inner join of two sequences using the given key projections. */
   def joinSeqs[O, I, K](outer: GenIterable[O], inner: GenIterable[I])(outKey: O=>K, inKey: I=>K): GenIterable[(O,I)] = {
     val kvs = createMultiMap(inner.map(i => (inKey(i), i)))
     val res = outer.flatMap(o => {
       val ko = outKey(o)
-      kvs(ko).map(i => (o,i))
+      kvs.get(ko) match {
+        case Some(inners) =>
+          inners.map(i => (o,i))
+        case None =>
+          Nil
+      }
     })
     res
   }
@@ -182,6 +207,21 @@ object CollectionUtil {
 
   implicit class TraversableOps[A, Source[X] <: GenIterable[X]](val xs: Source[A]) extends AnyVal {
 
+    /** Returns a copy of this collection where elements at `items(i)._1` are replaced
+      * with `items(i)._2` for each i.
+      */
+    def updateMany(items: Seq[(Int, A)])(implicit tA: ClassTag[A]): Seq[A] = {
+      val res = xs.toArray
+      val nItems = items.length
+      var i = 0
+      while (i < nItems) {
+        val item = items(i)
+        res(item._1) = item._2
+        i += 1
+      }
+      res
+    }
+
     def filterCast[B:ClassTag](implicit cbf: CanBuildFrom[Source[A], B, Source[B]]): Source[B] = {
       val b = cbf()
       for (x <- xs) {

common/src/main/scala/sigmastate/VersionContext.scala

@@ -0,0 +1,101 @@
+package sigmastate
+
+import sigmastate.VersionContext.JitActivationVersion
+
+import scala.util.DynamicVariable
+
+/** Represent currently activated protocol version and currently executed ErgoTree version.
+  *
+  * This parameters, once set in DynamicVariable can be accessed everywhere on the current
+  * thread.
+  *
+  * @param activatedVersion Currently activated script version == Block.headerVersion - 1
+  * @param ergoTreeVersion  version of the currently executed ErgoTree
+  *
+  * @see
+  */
+case class VersionContext(activatedVersion: Byte, ergoTreeVersion: Byte) {
+  require(ergoTreeVersion <= activatedVersion,
+    s"In a valid VersionContext ergoTreeVersion must never exceed activatedVersion: $this")
+
+  /** @return true, if the activated script version of Ergo protocol on the network is
+   * greater than v1. */
+  def isJitActivated: Boolean = activatedVersion >= JitActivationVersion
+
+  /** @return true, if the version of ErgoTree being executed greater than v1. */
+  def isErgoTreeVersionGreaterV1: Boolean =
+    ergoTreeVersion >= JitActivationVersion
+}
+
+object VersionContext {
+  /** Maximum version of ErgoTree supported by this interpreter release.
+    * See version bits in `ErgoTree.header` for more details.
+    * This value should be increased with each new protocol update via soft-fork.
+    * The following values are used for current and upcoming forks:
+    * - version 3.x this value must be 0
+    * - in v4.0 must be 1
+    * - in v5.x must be 2
+    * etc.
+    */
+  val MaxSupportedScriptVersion: Byte = 2 // supported versions 0, 1, 2
+
+  /** The first version of ErgoTree starting from which the JIT costing interpreter must be used.
+    * It must also be used for all subsequent versions (3, 4, etc).
+    */
+  val JitActivationVersion: Byte = 2
+
+  private val _defaultContext = VersionContext(
+    activatedVersion = 1/* v4.x */,
+    ergoTreeVersion = 1
+  )
+
+  /** Universally accessible version context which is used to version the code
+    * across the whole repository.
+    *
+    * The default value represent activated Ergo protocol and highest ErgoTree version.
+    */
+  private val _versionContext: DynamicVariable[VersionContext] =
+    new DynamicVariable[VersionContext](_defaultContext)
+
+  /** Returns the current VersionContext attached to the current thread.
+    * Each thread can have only one current version context at any time, which can be
+    * changed using `withVersions` method.
+    *
+    * @see withVersions()
+    */
+  def current: VersionContext = {
+    val ctx = _versionContext.value
+    if (ctx == null)
+      throw new IllegalStateException(
+        s"VersionContext is not specified on thread ${Thread.currentThread().getId}")
+    ctx
+  }
+
+  /** Executes the given block under the given version context attached to the current thread.
+    *
+    * The typical usage is to use `VersionContext.withVersions(activatedVersion,
+    * treeVersion) {...}` when the block of code needs to be executed with the given
+    * versions.
+    *
+    * For example, sigmastate.Interpreter uses it to execute operations according to the
+    * necessary versions of Ergo protocol and ErgoTree.
+    *
+    * @param activatedVersion Currently activated script version == Block.headerVersion - 1
+    * @param ergoTreeVersion  ErgoTree version to be set on the current thread
+    * @param block            block of code to execute
+    * @return result of block execution
+    */
+  def withVersions[T](activatedVersion: Byte, ergoTreeVersion: Byte)(block: => T): T =
+    _versionContext.withValue(VersionContext(activatedVersion, ergoTreeVersion))(block)
+
+  /** Checks the version context has the given versions*/
+  def checkVersions(activatedVersion: Byte, ergoTreeVersion: Byte) = {
+    val ctx = VersionContext.current
+    if (ctx.activatedVersion != activatedVersion || ctx.ergoTreeVersion != ergoTreeVersion) {
+      val expected = VersionContext(activatedVersion, ergoTreeVersion)
+      throw new IllegalStateException(
+        s"Global VersionContext.current = ${ctx} while expected $expected.")
+    }
+  }
+
+}

common/src/main/scala/sigmastate/util.scala

@@ -0,0 +1,31 @@
+package sigmastate
+
+import scalan.util.CollectionUtil
+
+import scala.reflect.ClassTag
+
+object util {
+  /** Maximum length of an allocatable array. */
+  val MaxArrayLength: Int = 100000
+
+  private def checkLength[A](len: Int)(implicit tA: ClassTag[A]) = {
+    if (len > MaxArrayLength)
+      throw new RuntimeException(
+        s"Cannot allocate array of $tA with $len items: max limit is $MaxArrayLength")
+  }
+
+  /** Allocates a new array with `len` items of type `A`.
+    * Should be used instead of `new Array[A](n)` or `Array.ofDim[A](n)`
+    */
+  final def safeNewArray[A](len: Int)(implicit tA: ClassTag[A]): Array[A] = {
+    checkLength[A](len)
+    new Array[A](len)
+  }
+
+  /** Concatenate two arrays checking length limit of the resulting array.
+    * Should be used in implementation of Coll operations of v5.0 and above. */
+  final def safeConcatArrays_v5[A](arr1: Array[A], arr2: Array[A])(implicit tA: ClassTag[A]): Array[A] = {
+    checkLength[A](arr1.length + arr2.length)
+    CollectionUtil.concatArrays_v5(arr1, arr2)
+  }
+}

common/src/test/scala/scalan/util/CollectionUtilTests.scala

@@ -9,16 +9,31 @@ class CollectionUtilTests extends BaseTests {
   import scalan.util.CollectionUtil._
   import java.lang.{Byte => JByte, Integer}
 
+  test("updateMany") {
+    val xs: Seq[Byte] = Array[Byte](1,2,3)
+    xs.updateMany(Seq.empty) shouldBe xs
+    xs.updateMany(Seq(0 -> 2)) shouldBe Seq(2, 2, 3)
+    xs.updateMany(Seq(0 -> 2, 2 -> 2)) shouldBe Seq(2, 2, 2)
+    an[IndexOutOfBoundsException] should be thrownBy {
+      xs.updateMany(Seq(3 -> 2))
+    }
+  }
+
   test("concatArrays") {
     val xs = Array[Byte](1,2,3)
     val ys = Array[Byte](4,5,6)
     val zs = concatArrays(xs, ys)
     assertResult(Array[Byte](1, 2, 3, 4, 5, 6))(zs)
 
-//    val jxs = Array[JByte](new JByte(1), new JByte(2), new JByte(3))
-//    val jys = Array[JByte](new JByte(4), new JByte(5), new JByte(6))
-//    val jzs = concatArrays(jxs, jys)
-//    assertResult(Array[Byte](1, 2, 3, 4, 5, 6))(jzs)
+    val pairs = xs.zip(ys)
+    // this reproduces the problem which takes place in v3.x, v4.x (ErgoTree v0, v1)
+    an[ClassCastException] should be thrownBy(concatArrays(pairs, pairs))
+
+    // and this is the fix in v5.0
+    concatArrays_v5(pairs, pairs) shouldBe Array((1, 4), (2, 5), (3, 6), (1, 4), (2, 5), (3, 6))
+
+    val xOpts = xs.map(Option(_))
+    concatArrays_v5(xOpts, xOpts) shouldBe Array(Some(1), Some(2), Some(3), Some(1), Some(2), Some(3))
   }
 
   def join(l: Map[Int,Int], r: Map[Int,Int]) =
@@ -28,6 +43,30 @@ class CollectionUtilTests extends BaseTests {
   def joinPairs(l: Seq[(String,Int)], r: Seq[(String,Int)]) =
     outerJoinSeqs(l, r)(l => l._1, r => r._1)((_,l) => l._2, (_,r) => r._2, (k,l,r) => l._2 + r._2)
 
+  test("joinSeqs") {
+    def key(p : (Int, String)): Int = p._1
+
+    {
+      val res = CollectionUtil.joinSeqs(
+        outer = Seq(1 -> "o1", 1 -> "o1"),
+        inner = Seq(1 -> "i1", 2 -> "i2"))(key, key)
+      res shouldBe Seq(
+        (1 -> "o1") -> (1 -> "i1"),
+        (1 -> "o1") -> (1 -> "i1")
+      )
+    }
+
+    { // same as above, but swapping inner and outer
+      val res = CollectionUtil.joinSeqs(
+        outer = Seq(1 -> "o1", 2 -> "o2"),
+        inner = Seq(1 -> "i1", 1 -> "i1"))(key, key)
+      res shouldBe Seq(
+        (1 -> "o1") -> (1 -> "i1"),
+        (1 -> "o1") -> (1 -> "i1")
+      )
+    }
+  }
+
   test("outerJoin maps") {
     val left = Map(1 -> 1, 2 -> 2, 3 -> 3)
     val right = Map(2 -> 2, 3 -> 3, 4 -> 4)

common/src/test/scala/sigmastate/VersionTesting.scala

@@ -0,0 +1,87 @@
+package sigmastate
+
+import spire.syntax.all.cfor
+
+import scala.util.DynamicVariable
+
+trait VersionTesting {
+
+  protected val activatedVersions: Seq[Byte] =
+    (0 to VersionContext.MaxSupportedScriptVersion).map(_.toByte).toArray[Byte]
+
+  private[sigmastate] val _currActivatedVersion = new DynamicVariable[Byte](0)
+  def activatedVersionInTests: Byte = _currActivatedVersion.value
+
+  /** Checks if the current activated script version used in tests corresponds to v4.x. */
+  def isActivatedVersion4: Boolean = activatedVersionInTests < VersionContext.JitActivationVersion
+
+  val ergoTreeVersions: Seq[Byte] =
+    (0 to VersionContext.MaxSupportedScriptVersion).map(_.toByte).toArray[Byte]
+
+  private[sigmastate] val _currErgoTreeVersion = new DynamicVariable[Byte](0)
+
+  /** Current ErgoTree version assigned dynamically. */
+  def ergoTreeVersionInTests: Byte = _currErgoTreeVersion.value
+
+  /** Executes the given block for each combination of _currActivatedVersion and
+    * _currErgoTreeVersion assigned to dynamic variables.
+    */
+  def forEachScriptAndErgoTreeVersion
+    (activatedVers: Seq[Byte], ergoTreeVers: Seq[Byte])
+    (block: => Unit): Unit = {
+    cfor(0)(_ < activatedVers.length, _ + 1) { i =>
+      val activatedVersion = activatedVers(i)
+      // setup each activated version
+      _currActivatedVersion.withValue(activatedVersion) {
+
+        cfor(0)(
+          i => i < ergoTreeVers.length && ergoTreeVers(i) <= activatedVersion,
+          _ + 1) { j =>
+          val treeVersion = ergoTreeVers(j)
+          // for each tree version up to currently activated, set it up and execute block
+          _currErgoTreeVersion.withValue(treeVersion)(block)
+        }
+
+      }
+    }
+  }
+
+  /** Helper method which executes the given `block` once for each `activatedVers`.
+    * The method sets the dynamic variable activatedVersionInTests with is then available
+    * in the block.
+    */
+  def forEachActivatedScriptVersion(activatedVers: Seq[Byte])(block: => Unit): Unit = {
+    cfor(0)(_ < activatedVers.length, _ + 1) { i =>
+      val activatedVersion = activatedVers(i)
+      _currActivatedVersion.withValue(activatedVersion)(block)
+    }
+  }
+
+  /** Helper method which executes the given `block` once for each `ergoTreeVers`.
+    * The method sets the dynamic variable ergoTreeVersionInTests with is then available
+    * in the block.
+    */
+  def forEachErgoTreeVersion(ergoTreeVers: Seq[Byte])(block: => Unit): Unit = {
+    cfor(0)(_ < ergoTreeVers.length, _ + 1) { i =>
+      val version = ergoTreeVers(i)
+      _currErgoTreeVersion.withValue(version)(block)
+    }
+  }
+
+  val printVersions: Boolean = false
+
+  protected def testFun_Run(testName: String, testFun: => Any): Unit = {
+    def msg = s"""property("$testName")(ActivatedVersion = $activatedVersionInTests; ErgoTree version = $ergoTreeVersionInTests)"""
+    if (printVersions) println(msg)
+    try testFun
+    catch {
+      case t: Throwable =>
+        if (!printVersions) {
+          // wasn't printed, print it now
+          println(msg)
+        }
+        throw t
+    }
+  }
+
+}