merging 6.0-deserialize

core/shared/src/main/scala/sigma/SigmaDsl.scala

@@ -647,7 +647,6 @@ trait Header {
     * @return result of header's proof-of-work validation
     */
   def checkPow: Boolean
-
 }
 
 /** Runtime representation of Context ErgoTree type.
@@ -935,6 +934,9 @@ trait SigmaDslBuilder {
   /** Returns a byte-wise XOR of the two collections of bytes. */
   def xor(l: Coll[Byte], r: Coll[Byte]): Coll[Byte]
 
+  /** Deserializes provided `bytes` into a value of type `T`. **/
+  def deserializeTo[T](bytes: Coll[Byte])(implicit cT: RType[T]): T
+
   /** Returns a number decoded from provided big-endian bytes array. */
   def fromBigEndianBytes[T](bytes: Coll[Byte])(implicit cT: RType[T]): T
 

core/shared/src/main/scala/sigma/reflection/ReflectionData.scala

@@ -501,6 +501,9 @@ object ReflectionData {
         mkMethod(clazz, "decodePoint", Array[Class[_]](cColl)) { (obj, args) =>
           obj.asInstanceOf[SigmaDslBuilder].decodePoint(args(0).asInstanceOf[Coll[Byte]])
         },
+        mkMethod(clazz, "deserializeTo", Array[Class[_]](cColl, classOf[RType[_]])) { (obj, args) =>
+          obj.asInstanceOf[SigmaDslBuilder].deserializeTo(args(0).asInstanceOf[Coll[Byte]])(args(1).asInstanceOf[RType[_]])
+        },
         mkMethod(clazz, "fromBigEndianBytes", Array[Class[_]](cColl, classOf[RType[_]])) { (obj, args) =>
           obj.asInstanceOf[SigmaDslBuilder].fromBigEndianBytes(args(0).asInstanceOf[Coll[Byte]])(args(1).asInstanceOf[RType[_]])
         },

core/shared/src/main/scala/sigma/serialization/CoreDataSerializer.scala

@@ -146,7 +146,7 @@ class CoreDataSerializer {
     res
   }
 
-  def deserializeColl[T <: SType](len: Int, tpeElem: T, r: CoreByteReader): Coll[T#WrappedType] =
+  private def deserializeColl[T <: SType](len: Int, tpeElem: T, r: CoreByteReader): Coll[T#WrappedType] =
     tpeElem match {
       case SBoolean =>
         Colls.fromArray(r.getBits(len)).asInstanceOf[Coll[T#WrappedType]]

data/shared/src/main/scala/sigma/SigmaDataReflection.scala

@@ -351,6 +351,11 @@ object SigmaDataReflection {
           obj.asInstanceOf[SGlobalMethods.type].serialize_eval(args(0).asInstanceOf[MethodCall],
             args(1).asInstanceOf[SigmaDslBuilder],
             args(2).asInstanceOf[SType#WrappedType])(args(3).asInstanceOf[ErgoTreeEvaluator])
+        },
+        mkMethod(clazz, "deserializeTo_eval", Array[Class[_]](classOf[MethodCall], classOf[SigmaDslBuilder], classOf[Coll[_]], classOf[ErgoTreeEvaluator])) { (obj, args) =>
+          obj.asInstanceOf[SGlobalMethods.type].deserializeTo_eval(args(0).asInstanceOf[MethodCall],
+            args(1).asInstanceOf[SigmaDslBuilder],
+            args(2).asInstanceOf[Coll[Byte]])(args(3).asInstanceOf[ErgoTreeEvaluator])
         }
       )
     )

data/shared/src/main/scala/sigma/ast/SigmaPredef.scala

@@ -448,6 +448,25 @@ object SigmaPredef {
       )
     )
 
+    val DeserializeToFunc = PredefinedFunc("deserializeTo",
+      Lambda(Seq(paramT), Array("bytes" -> SByteArray), tT, None),
+      irInfo = PredefFuncInfo(
+        irBuilder = { case (u, args) =>
+          val resType = u.opType.tRange.asInstanceOf[SFunc].tRange
+          MethodCall(
+            Global,
+            SGlobalMethods.deserializeToMethod.withConcreteTypes(Map(tT -> resType)),
+            args.toIndexedSeq,
+            Map(tT -> resType)
+          )
+        }),
+      docInfo = OperationInfo(MethodCall,
+        """Deserializes provided bytes into a value of given type using the default serialization format.
+        """.stripMargin,
+        Seq(ArgInfo("bytes", "bytes to deserialize"))
+      )
+    )
+
     val FromBigEndianBytesFunc = PredefinedFunc("fromBigEndianBytes",
       Lambda(Seq(paramT), Array("bytes" -> SByteArray), tT, None),
       irInfo = PredefFuncInfo(
@@ -498,7 +517,8 @@ object SigmaPredef {
       ExecuteFromSelfRegFunc,
       SerializeFunc,
       GetVarFromInputFunc,
-      FromBigEndianBytesFunc
+      FromBigEndianBytesFunc,
+      DeserializeToFunc
     ).map(f => f.name -> f).toMap
 
     def comparisonOp(symbolName: String, opDesc: ValueCompanion, desc: String, args: Seq[ArgInfo]) = {

data/shared/src/main/scala/sigma/ast/methods.scala

@@ -1901,6 +1901,7 @@ case object SHeaderMethods extends MonoTypeMethods {
   lazy val powDistanceMethod      = propertyCall("powDistance", SBigInt, 14, FixedCost(JitCost(10)))
   lazy val votesMethod            = propertyCall("votes", SByteArray, 15, FixedCost(JitCost(10)))
 
+  // methods added in 6.0 below
   // cost of checkPoW is 700 as about 2*32 hashes required, and 1 hash (id) over short data costs 10
   lazy val checkPowMethod = SMethod(
     this, "checkPow", SFunc(Array(SHeader), SBoolean), 16, FixedCost(JitCost(700)))
@@ -1971,6 +1972,16 @@ case object SGlobalMethods extends MonoTypeMethods {
     .withInfo(Xor, "Byte-wise XOR of two collections of bytes",
       ArgInfo("left", "left operand"), ArgInfo("right", "right operand"))
 
+  private val deserializeCostKind = PerItemCost(
+    baseCost = JitCost(20), perChunkCost = JitCost(7), chunkSize = 128)
+
+  lazy val deserializeToMethod = SMethod(
+    this, "deserializeTo", SFunc(Array(SGlobal, SByteArray), tT, Array(paramT)), 4, deserializeCostKind, Seq(tT))
+    .withIRInfo(MethodCallIrBuilder,
+      javaMethodOf[SigmaDslBuilder, Coll[Byte], RType[_]]("deserializeTo"))
+    .withInfo(MethodCall, "Deserialize provided bytes into an object of requested type",
+      ArgInfo("first", "Bytes to deserialize"))
+
   /** Implements evaluation of Global.xor method call ErgoTree node.
     * Called via reflection based on naming convention.
     * @see SMethod.evalMethod, Xor.eval, Xor.xorWithCosting
@@ -1992,6 +2003,15 @@ case object SGlobalMethods extends MonoTypeMethods {
       "Decode a number from big endian bytes.",
       ArgInfo("first", "Bytes which are big-endian encoded number."))
 
+  def deserializeTo_eval(mc: MethodCall, G: SigmaDslBuilder, bytes: Coll[Byte])
+                        (implicit E: ErgoTreeEvaluator): Any = {
+    val tpe = mc.tpe
+    val cT = stypeToRType(tpe)
+    E.addSeqCost(deserializeCostKind, bytes.length, deserializeToMethod.opDesc) { () =>
+      G.deserializeTo(bytes)(cT)
+    }
+  }
+
   lazy val serializeMethod = SMethod(this, "serialize",
     SFunc(Array(SGlobal, tT), SByteArray, Array(paramT)), 3, DynamicCost)
       .withIRInfo(MethodCallIrBuilder)
@@ -2042,6 +2062,7 @@ case object SGlobalMethods extends MonoTypeMethods {
         groupGeneratorMethod,
         xorMethod,
         serializeMethod,
+        deserializeToMethod,
         fromBigEndianBytesMethod,
         someMethod,
         noneMethod

data/shared/src/main/scala/sigma/data/CHeader.scala

@@ -71,7 +71,11 @@ class CHeader(val ergoHeader: ErgoHeader) extends Header with WrapperOf[ErgoHead
   }
 
   override def checkPow: Boolean = {
-    Autolykos2PowValidation.checkPoWForVersion2(this)
+    if (version == 1) {
+      throw new Exception("Autolykos v1 is not supported") //todo: more specific exception?
+    } else {
+      Autolykos2PowValidation.checkPoWForVersion2(this)
+    }
   }
 
   override def toString: String =

data/shared/src/main/scala/sigma/data/CSigmaDslBuilder.scala

@@ -1,15 +1,18 @@
 package sigma.data
 
 import debox.cfor
-import org.ergoplatform.ErgoBox
+import org.ergoplatform.{ErgoBox, ErgoHeader}
 import org.ergoplatform.validation.ValidationRules
 import scorex.crypto.hash.{Blake2b256, Sha256}
+import scorex.util.serialization.VLQByteBufferReader
 import scorex.utils.{Ints, Longs}
 import sigma.ast.{AtLeast, SBigInt, SubstConstants}
 import scorex.utils.Longs
+import sigma.Evaluation.rtypeToSType
 import sigma.ast.{AtLeast, SType, SubstConstants}
 import sigma.crypto.{CryptoConstants, EcPointType, Ecp}
 import sigma.eval.Extensions.EvalCollOps
+import sigma.serialization.{ConstantStore, DataSerializer, GroupElementSerializer, SigmaByteReader, SigmaSerializer}
 import sigma.serialization.{DataSerializer, GroupElementSerializer, SigmaSerializer}
 import sigma.serialization.{GroupElementSerializer, SerializerException, SigmaSerializer}
 import sigma.util.Extensions.BigIntegerOps
@@ -18,6 +21,7 @@ import sigma.{AvlTree, BigInt, Box, Coll, CollBuilder, Evaluation, GroupElement,
 import sigma.{AvlTree, BigInt, Box, Coll, CollBuilder, GroupElement, SigmaDslBuilder, SigmaProp, UnsignedBigInt, VersionContext}
 
 import java.math.BigInteger
+import java.nio.ByteBuffer
 
 /** A default implementation of [[SigmaDslBuilder]] interface.
   *
@@ -248,6 +252,13 @@ class CSigmaDslBuilder extends SigmaDslBuilder { dsl =>
     Colls.fromArray(w.toBytes)
   }
 
+  def deserializeTo[T](bytes: Coll[Byte])(implicit cT: RType[T]): T = {
+    val tpe = rtypeToSType(cT)
+    val reader = new SigmaByteReader(new VLQByteBufferReader(ByteBuffer.wrap(bytes.toArray)), new ConstantStore(), false)
+    val res = DataSerializer.deserialize(tpe, reader)
+    res.asInstanceOf[T]
+  }
+
   override def some[T](value: T)(implicit cT: RType[T]): Option[T] = {
     Some(value)
   }

docs/LangSpec.md

@@ -68,7 +68,7 @@ The following sections describe ErgoScript and its operations.
 #### Operations and constructs overview
 
 - Binary operations: `>, <, >=, <=, +, -, &&, ||, ==, !=, |, &, *, /, %, ^, ++`
-- predefined primitives: `serialize`, `blake2b256`, `byteArrayToBigInt`, `proveDlog` etc. 
+- predefined primitives: `deserializeTo`, `serialize`, `blake2b256`, `byteArrayToBigInt`, `proveDlog` etc. 
 - val declarations: `val h = blake2b256(pubkey)`
 - if-then-else clause: `if (x > 0) 1 else 0`
 - collection literals: `Coll(1, 2, 3, 4)`
@@ -903,6 +903,10 @@ def blake2b256(input: Coll[Byte]): Coll[Byte]
 /** Cryptographic hash function Sha256 (See scorex.crypto.hash.Sha256) */
 def sha256(input: Coll[Byte]): Coll[Byte]
 
+/** Create an instance of type T from bytes of its wrapped type. 
+See https://github.com/ScorexFoundation/sigmastate-interpreter/pull/979 for more details */
+def deserializeTo[T](input: Coll[Byte]): T
+
 /** Create BigInt from a collection of bytes. */
 def byteArrayToBigInt(input: Coll[Byte]): BigInt
 

interpreter/shared/src/main/scala/sigmastate/interpreter/Interpreter.scala

@@ -126,7 +126,7 @@ trait Interpreter {
     case _ => None
   }
 
-  /** Extracts proposition for ErgoTree handing soft-fork condition.
+  /** Extracts proposition for ErgoTree handling soft-fork condition.
     * @note soft-fork handler */
   protected def propositionFromErgoTree(ergoTree: ErgoTree, context: CTX): SigmaPropValue = {
     val validationSettings = context.validationSettings

interpreter/shared/src/test/scala/sigma/serialization/MethodCallSerializerSpecification.scala

@@ -2,6 +2,7 @@ package sigma.serialization
 
 import sigma.VersionContext
 import sigma.ast.SCollection.SByteArray
+import sigma.ast.SType.tT
 import sigma.ast._
 import sigma.validation.ValidationException
 
@@ -51,7 +52,7 @@ class MethodCallSerializerSpecification extends SerializationSpecification {
     def code = {
       val b = ByteArrayConstant(Array(1.toByte, 2.toByte, 3.toByte))
       val expr = MethodCall(Global,
-        SGlobalMethods.serializeMethod.withConcreteTypes(Map(STypeVar("T") -> SByteArray)),
+        SGlobalMethods.serializeMethod.withConcreteTypes(Map(tT -> SByteArray)),
         Vector(b),
         Map()
       )
@@ -62,6 +63,30 @@ class MethodCallSerializerSpecification extends SerializationSpecification {
       code
     }
 
+    an[ValidationException] should be thrownBy (
+      VersionContext.withVersions((VersionContext.V6SoftForkVersion - 1).toByte, 1) {
+        code
+      }
+    )
+  }
+
+  property("MethodCall deserialization round trip for Global.deserializeTo[]") {
+    def code = {
+      val h = HeaderConstant(headerGen.sample.get)
+      val expr = MethodCall(h,
+        SGlobalMethods.deserializeToMethod.withConcreteTypes(Map(tT -> SHeader)),
+        Array(ByteArrayConstant(Array(1.toByte, 2.toByte, 3.toByte))), // wrong header bytes but ok for test
+        Map(tT -> SHeader)
+      )
+      roundTripTest(expr)
+    }
+
+    println(SGlobalMethods.deserializeToMethod.hasExplicitTypeArgs)
+
+    VersionContext.withVersions(VersionContext.V6SoftForkVersion, 1) {
+      code
+    }
+
     an[Exception] should be thrownBy (
       VersionContext.withVersions((VersionContext.V6SoftForkVersion - 1).toByte, 1) {
         code

interpreter/shared/src/test/scala/special/sigma/SigmaTestingData.scala

@@ -1,6 +1,6 @@
 package sigma
 
-import org.ergoplatform.ErgoBox
+import org.ergoplatform.{ErgoBox, ErgoHeader}
 import org.ergoplatform.settings.ErgoAlgos
 import org.scalacheck.Arbitrary.arbitrary
 import org.scalacheck.Gen.containerOfN
@@ -267,7 +267,10 @@ trait SigmaTestingData extends TestingCommons with ObjectGenerators {
 
     val h1: Header = create_h1()
 
-    val h2: Header = new CHeader(h1.asInstanceOf[CHeader].wrappedValue.copy(height = 2))
+    val eh1 = h1.asInstanceOf[CHeader].ergoHeader
+    val h2: Header = new CHeader(new ErgoHeader(eh1.version, eh1.parentId, eh1.ADProofsRoot, eh1.stateRoot,
+                                eh1.transactionsRoot, eh1.timestamp, eh1.nBits, 2, eh1.extensionRoot,
+                                eh1.powSolution, eh1.votes, eh1.unparsedBytes, null))
 
     val dlog_instances = new CloneSet(1000, ProveDlog(
       SigmaDsl.toECPoint(create_ge1()).asInstanceOf[EcPointType]

sc/shared/src/main/scala/sigma/compiler/ir/Base.scala

@@ -169,7 +169,7 @@ abstract class Base { thisIR: IRContext =>
 
     /** Create a copy of this definition applying the given transformer to all `syms`. */
     def transform(t: Transformer): Def[T] =
-      !!!(s"Cannot transfrom definition using transform($this)", self)
+      !!!(s"Cannot transform definition using transform($this)", self)
 
     /** Clone this definition transforming all symbols using `t`.
       * If new Def[A] is created, it is added to the graph with collapsing and rewriting.

sc/shared/src/main/scala/sigma/compiler/ir/GraphBuilding.scala

@@ -2,6 +2,7 @@ package sigma.compiler.ir
 
 import org.ergoplatform._
 import sigma.ast.SType.tT
+import sigma.{SigmaException, VersionContext, ast}
 import sigma.Evaluation.stypeToRType
 import sigma.ast.SType.tT
 import sigma.ast.TypeCodes.LastConstantCode
@@ -21,11 +22,11 @@ import sigma.exceptions.GraphBuildingException
 import sigma.serialization.OpCodes
 import sigma.{SigmaException, ast}
 import sigma.util.Extensions.ByteOps
-import sigma.{SigmaException, VersionContext, ast}
 import sigmastate.interpreter.Interpreter.ScriptEnv
 
 import scala.collection.mutable.ArrayBuffer
 
+
 /** Perform translation of typed expression given by [[Value]] to a graph in IRContext.
   * Which be than be translated to [[ErgoTree]] by using [[TreeBuilding]].
   *
@@ -1217,6 +1218,10 @@ trait GraphBuilding extends Base with DefRewriting { IR: IRContext =>
               val c1 = asRep[Coll[Byte]](argsV(0))
               val c2 = asRep[Coll[Byte]](argsV(1))
               g.xor(c1, c2)
+            case SGlobalMethods.deserializeToMethod.name if VersionContext.current.isV6SoftForkActivated =>
+              val c1 = asRep[Coll[Byte]](argsV(0))
+              val c2 = stypeToElem(method.stype.tRange.withSubstTypes(typeSubst))
+              g.deserializeTo(c1)(c2)
             case SGlobalMethods.serializeMethod.name =>
               val value = asRep[Any](argsV(0))
               g.serialize(value)

sc/shared/src/main/scala/sigma/compiler/ir/GraphIRReflection.scala

@@ -1,5 +1,6 @@
 package sigma.compiler.ir
 
+import sigma.Coll
 import sigma.ast.SType
 import sigma.compiler.ir.primitives.Thunks
 import sigma.data.RType
@@ -539,6 +540,9 @@ object GraphIRReflection {
         },
         mkMethod(clazz, "none", Array[Class[_]](classOf[TypeDescs#Elem[_]])) { (obj, args) =>
           obj.asInstanceOf[ctx.SigmaDslBuilder].none()(args(0).asInstanceOf[ctx.Elem[SType]])
+        },
+        mkMethod(clazz, "deserializeTo", Array[Class[_]](classOf[Base#Ref[_]], classOf[TypeDescs#Elem[_]])) { (obj, args) =>
+          obj.asInstanceOf[ctx.SigmaDslBuilder].deserializeTo(args(0).asInstanceOf[ctx.Ref[ctx.Coll[Byte]]])(args(1).asInstanceOf[ctx.Elem[SType]])
         }
       )
     )

sc/shared/src/main/scala/sigma/compiler/ir/TreeBuilding.scala

@@ -6,9 +6,9 @@ import sigma.Evaluation.{rtypeToSType, stypeToRType}
 import sigma.ast.SType.tT
 import sigma.ast._
 import sigma.ast.syntax.{ValueOps, _}
-import sigma.data.{ProveDHTuple, ProveDlog}
-import sigma.serialization.ConstantStore
 import sigma.serialization.OpCodes._
+import sigma.serialization.ConstantStore
+import sigma.data.{ProveDHTuple, ProveDlog}
 import sigma.serialization.ValueCodes.OpCode
 
 import scala.collection.mutable.ArrayBuffer
@@ -279,6 +279,13 @@ trait TreeBuilding extends Base { IR: IRContext =>
         val tpe = elemToSType(eVar)
         mkGetVar(id, tpe)
 
+      case SDBM.deserializeTo(g, bytes, eVar) =>
+        val tpe = elemToSType(eVar)
+        val typeSubst = Map(tT -> tpe): STypeSubst
+        // method specialization done to avoid serialization roundtrip issues
+        val method = SGlobalMethods.deserializeToMethod.withConcreteTypes(typeSubst)
+        builder.mkMethodCall(recurse(g), method, IndexedSeq(recurse(bytes)), typeSubst)
+
       case BIM.subtract(In(x), In(y)) =>
         mkArith(x.asNumValue, y.asNumValue, MinusCode)
       case BIM.add(In(x), In(y)) =>

sc/shared/src/main/scala/sigma/compiler/ir/wrappers/sigma/SigmaDslUnit.scala

@@ -135,6 +135,7 @@ import scalan._
       def xor(l: Ref[Coll[Byte]], r: Ref[Coll[Byte]]): Ref[Coll[Byte]]
       def serialize[T](value: Ref[T]): Ref[Coll[Byte]]
       def fromBigEndianBytes[T](bytes: Ref[Coll[Byte]])(implicit cT: Elem[T]): Ref[T]
+      def deserializeTo[T](bytes: Ref[Coll[Byte]])(implicit cT: Elem[T]): Ref[T]
       def some[T](value: Ref[T])(implicit cT: Elem[T]): Ref[WOption[T]]
       def none[T]()(implicit cT: Elem[T]): Ref[WOption[T]]
     };