merging w. i1006 (numeric methods)

.gitignore

@@ -11,6 +11,7 @@
 
 yarn.lock
 *.log
+yarn.lock
 docs/spec/out/
 test-out/
 flamegraphs/

core/shared/src/main/scala/sigma/SigmaDsl.scala

@@ -153,6 +153,23 @@ trait BigInt {
   def or(that: BigInt): BigInt
   def |(that: BigInt): BigInt = or(that)
 
+  /**
+    * @return a big integer whose value is `this xor that`
+    */
+  def xor(that: BigInt): BigInt
+
+  /**
+    * @return a 256-bit signed integer whose value is (this << n). The shift distance, n, may be negative,
+    *         in which case this method performs a right shift. (Computes floor(this * 2n).)
+    */
+  def shiftLeft(n: Int): BigInt
+
+  /**
+    * @return a 256-bit signed integer whose value is (this >> n). Sign extension is performed. The shift distance, n,
+    *         may be negative, in which case this method performs a left shift. (Computes floor(this / 2n).)
+    */
+  def shiftRight(n: Int): BigInt
+
   def toUnsigned: UnsignedBigInt
 
   def toUnsignedMod(m: UnsignedBigInt): UnsignedBigInt
@@ -590,6 +607,23 @@ trait Header {
 
   /** Miner votes for changing system parameters. */
   def votes: Coll[Byte] //3 bytes
+
+  /** Bytes which are coming from future versions of the protocol, so
+    * their meaning is not known to current version of Sigma, but they
+    * are stored to get the same id as future version users.
+    */
+  def unparsedBytes: Coll[Byte]
+
+  /**
+    * @return header bytes without proof of work, a PoW is generated over them
+    */
+  def serializeWithoutPoW: Coll[Byte]
+
+  /**
+    * @return result of header's proof-of-work validation
+    */
+  def checkPow: Boolean
+
 }
 
 /** Runtime representation of Context ErgoTree type.

core/shared/src/main/scala/sigma/ast/SType.scala

@@ -113,27 +113,48 @@ object SType {
     * typeId this map contains a companion object which can be used to access the list of
     * corresponding methods.
     *
-    * NOTE: in the current implementation only monomorphic methods are supported (without
-    * type parameters)
+    * @note starting from v6.0 methods with type parameters are also supported.
     *
-    * NOTE2: in v3.x SNumericType.typeId is silently shadowed by SGlobal.typeId as part of
-    * `toMap` operation. As a result, the methods collected into SByte.methods cannot be
+    * @note on versioning:
+    * In v3.x-5.x SNumericType.typeId is silently shadowed by SGlobal.typeId as part of
+    * `toMap` operation. As a result, SNumericTypeMethods container cannot be resolved by
+    * typeId = 106, because SNumericType was being silently removed when `_types` map is
+    * constructed. See `property("SNumericType.typeId resolves to SGlobal")`.
+    * In addition, the methods associated with the concrete numeric types cannot be
     * resolved (using SMethod.fromIds()) for all numeric types (SByte, SShort, SInt,
-    * SLong, SBigInt). See the corresponding regression `property("MethodCall on numerics")`.
+    * SLong) because these types are not registered in the `_types` map.
+    * See the corresponding property("MethodCall on numerics")`.
     * However, this "shadowing" is not a problem since all casting methods are implemented
-    * via Downcast, Upcast opcodes and the remaining `toBytes`, `toBits` methods are not
-    * implemented at all.
-    * In order to allow MethodCalls on numeric types in future versions the SNumericType.typeId
-    * should be changed and SGlobal.typeId should be preserved. The regression tests in
-    * `property("MethodCall Codes")` should pass.
+    * via lowering to Downcast, Upcast opcodes and the remaining `toBytes`, `toBits`
+    * methods are not implemented at all.
+    *
+    * Starting from v6.0 the SNumericType.typeId is demoted as a receiver object of
+    * method calls and:
+    * 1) numeric type SByte, SShort, SInt, SLong are promoted as receivers and added to
+    * the _types map.
+    * 2) all methods from SNumericTypeMethods are copied to all the concrete numeric types
+    * (SByte, SShort, SInt, SLong, SBigInt) and the generic tNum type parameter is
+    * specialized accordingly.
+    *
+    * This difference in behaviour is tested by `property("MethodCall on numerics")`.
+    *
+    * The regression tests in `property("MethodCall Codes")` should pass.
     */
-  // TODO v6.0: should contain all numeric types (including also SNumericType)
-  //  to support method calls like 10.toByte which encoded as MethodCall with typeId = 4, methodId = 1
-  //  see https://github.com/ScorexFoundation/sigmastate-interpreter/issues/667
-  lazy val types: Map[Byte, STypeCompanion] = Seq(
-    SBoolean, SNumericType, SString, STuple, SGroupElement, SSigmaProp, SContext, SGlobal, SHeader, SPreHeader,
+  private val v5Types = Seq(
+    SBoolean, SString, STuple, SGroupElement, SSigmaProp, SContext, SGlobal, SHeader, SPreHeader,
     SAvlTree, SBox, SOption, SCollection, SBigInt
-  ).map { t => (t.typeId, t) }.toMap
+  )
+  private val v6Types = v5Types ++ Seq(SByte, SShort, SInt, SLong)
+
+  private val v5TypesMap = v5Types.map { t => (t.typeId, t) }.toMap
+
+  private val v6TypesMap = v6Types.map { t => (t.typeId, t) }.toMap
+
+  def types: Map[Byte, STypeCompanion] = if (VersionContext.current.isV6SoftForkActivated) {
+    v6TypesMap
+  } else {
+    v5TypesMap
+  }
 
   /** Checks that the type of the value corresponds to the descriptor `tpe`.
     * If the value has complex structure only root type constructor is checked.
@@ -625,7 +646,7 @@ case class SFunc(tDom: IndexedSeq[SType],  tRange: SType, tpeParams: Seq[STypePa
 }
 
 object SFunc {
-  final val FuncTypeCode: TypeCode = TypeCodes.FirstFuncType
+  final val FuncTypeCode: TypeCode = TypeCodes.FuncType
   def apply(tDom: SType, tRange: SType): SFunc = SFunc(Array(tDom), tRange) // HOTSPOT:
   val identity = { x: Any => x }
 }
@@ -692,7 +713,6 @@ object SOption extends STypeCompanion {
   def apply[T <: SType](implicit elemType: T, ov: Overloaded1): SOption[T] = SOption(elemType)
 }
 
-
 /** Base class for descriptors of `Coll[T]` ErgoTree type for some elemType T. */
 trait SCollection[T <: SType] extends SProduct with SGenericType {
   def elemType: T

core/shared/src/main/scala/sigma/ast/STypeParam.scala

@@ -2,18 +2,10 @@ package sigma.ast
 
 /** Represents a type parameter in a type system.
   *
-  * @param ident       The identifier for this type parameter.
-  * @param upperBound  The upper bound of this type parameter, if exists.
-  * @param lowerBound  The lower bound of this type parameter, if exists.
-  * @note Type parameters with bounds are currently not supported.
+  * @param ident       The identifier for this type parameter
   */
-case class STypeParam(
-    ident: STypeVar,
-    upperBound: Option[SType] = None,
-    lowerBound: Option[SType] = None) {
-  assert(upperBound.isEmpty && lowerBound.isEmpty, s"Type parameters with bounds are not supported, but found $this")
-
-  override def toString = ident.toString + upperBound.fold("")(u => s" <: $u") + lowerBound.fold("")(l => s" >: $l")
+case class STypeParam(ident: STypeVar) {
+  override def toString = ident.toString
 }
 
 object STypeParam {

core/shared/src/main/scala/sigma/ast/TypeCodes.scala

@@ -14,10 +14,8 @@ object TypeCodes {
 
   val LastDataType : TypeCode = TypeCode @@ 111.toByte
 
-  /** SFunc types occupy remaining space of byte values [FirstFuncType .. 255] */
-  val FirstFuncType: TypeCode = TypeCode @@ (LastDataType + 1).toByte
-
-  val LastFuncType : TypeCode = TypeCode @@ 255.toByte
+  /** SFunc type */
+  val FuncType: TypeCode = TypeCode @@ (LastDataType + 1).toByte
 
   /** We use optimized encoding of constant values to save space in serialization.
     * Since Box registers are stored as Constant nodes we save 1 byte for each register.

core/shared/src/main/scala/sigma/ast/package.scala

@@ -137,6 +137,12 @@ package object ast {
 
     def asNumType: SNumericType = tpe.asInstanceOf[SNumericType]
 
+    /** Cast this type to numeric type or else throws the given error. */
+    def asNumTypeOrElse(error: => Exception): SNumericType = tpe match {
+      case nt: SNumericType => nt
+      case _ => throw error
+    }
+
     def asFunc: SFunc = tpe.asInstanceOf[SFunc]
 
     def asProduct: SProduct = tpe.asInstanceOf[SProduct]

core/shared/src/main/scala/sigma/data/CBigInt.scala

@@ -50,6 +50,12 @@ case class CBigInt(override val wrappedValue: BigInteger) extends BigInt with Wr
 
   override def or(that: BigInt): BigInt = CBigInt(wrappedValue.or(that.asInstanceOf[CBigInt].wrappedValue))
 
+  override def xor(that: BigInt): BigInt = CBigInt(wrappedValue.xor(that.asInstanceOf[CBigInt].wrappedValue))
+
+  override def shiftLeft(n: Int): BigInt = CBigInt(wrappedValue.shiftLeft(n).to256BitValueExact)
+
+  override def shiftRight(n: Int): BigInt = CBigInt(wrappedValue.shiftRight(n).to256BitValueExact)
+
   def toUnsigned: UnsignedBigInt = {
     if(this.wrappedValue.compareTo(BigInteger.ZERO) < 0){
       throw new ArithmeticException("BigInteger argument for .toUnsigned is negative in");

core/shared/src/main/scala/sigma/reflection/ReflectionData.scala

@@ -350,6 +350,9 @@ object ReflectionData {
         },
         mkMethod(clazz, "powDistance", Array[Class[_]]()) { (obj, _) =>
           obj.asInstanceOf[Header].powDistance
+        },
+        mkMethod(clazz, "checkPow", Array[Class[_]]()) { (obj, _) =>
+          obj.asInstanceOf[Header].checkPow
         }
       )
     )

core/shared/src/main/scala/sigma/serialization/TypeSerializer.scala

@@ -1,9 +1,9 @@
 package sigma.serialization
 
 import debox.cfor
+import sigma.VersionContext
 import sigma.ast.SCollectionType.{CollectionTypeCode, NestedCollectionTypeCode}
 import sigma.ast._
-import sigma.serialization.{CoreByteReader, CoreByteWriter, InvalidTypePrefix}
 import sigma.util.safeNewArray
 import sigma.validation.ValidationRules.{CheckPrimitiveTypeCode, CheckTypeCode}
 
@@ -101,6 +101,17 @@ class TypeSerializer {
         // `Tuple` type with more than 4 items `(Int, Byte, Box, Boolean, Int)`
         serializeTuple(tup, w)
     }
+    case SFunc(tDom, tRange, tpeParams) =>
+      w.put(SFunc.FuncTypeCode)
+      w.putUByte(tDom.length)
+      tDom.foreach { st =>
+        serialize(st, w)
+      }
+      serialize(tRange, w)
+      w.putUByte(tpeParams.length)
+      tpeParams.foreach { tp =>
+        serialize(tp.ident, w)
+      }
     case typeIdent: STypeVar => {
       w.put(typeIdent.typeCode)
       val bytes = typeIdent.name.getBytes(StandardCharsets.UTF_8)
@@ -189,7 +200,23 @@ class TypeSerializer {
         case SHeader.typeCode => SHeader
         case SPreHeader.typeCode => SPreHeader
         case SGlobal.typeCode => SGlobal
+        case SFunc.FuncTypeCode if VersionContext.current.isV6SoftForkActivated =>
+          val tdLength = r.getUByte()
+
+          val tDom = (1 to tdLength).map { _ =>
+            deserialize(r)
+          }
+          val tRange = deserialize(r)
+          val tpeParamsLength = r.getUByte()
+          val tpeParams = (1 to tpeParamsLength).map { _ =>
+            val ident = deserialize(r)
+            require(ident.isInstanceOf[STypeVar])
+            STypeParam(ident.asInstanceOf[STypeVar])
+          }
+          SFunc(tDom, tRange, tpeParams)
         case _ =>
+          // todo: 6.0: replace 1008 check with identical behavior but other opcode, to activate
+          //  ReplacedRule(1008 -> new opcode) during 6.0 activation
           CheckTypeCode(c.toByte)
           NoType
       }

core/shared/src/main/scala/sigma/util/NBitsUtils.scala

@@ -0,0 +1,84 @@
+package sigma.util
+
+import java.math.BigInteger
+
+object NBitsUtils {
+
+  /**
+    * <p>The "compact" format is a representation of a whole number N using an unsigned 32 bit number similar to a
+    * floating point format. The most significant 8 bits are the unsigned exponent of base 256. This exponent can
+    * be thought of as "number of bytes of N". The lower 23 bits are the mantissa. Bit number 24 (0x800000) represents
+    * the sign of N. Therefore, N = (-1^sign) * mantissa * 256^(exponent-3).</p>
+    *
+    * <p>Satoshi's original implementation used BN_bn2mpi() and BN_mpi2bn(). MPI uses the most significant bit of the
+    * first byte as sign. Thus 0x1234560000 is compact 0x05123456 and 0xc0de000000 is compact 0x0600c0de. Compact
+    * 0x05c0de00 would be -0x40de000000.</p>
+    *
+    * <p>Bitcoin only uses this "compact" format for encoding difficulty targets, which are unsigned 256bit quantities.
+    * Thus, all the complexities of the sign bit and using base 256 are probably an implementation accident.</p>
+    */
+  def decodeCompactBits(compact: Long): BigInt = {
+    val size: Int = (compact >> 24).toInt & 0xFF
+    val bytes: Array[Byte] = new Array[Byte](4 + size)
+    bytes(3) = size.toByte
+    if (size >= 1) bytes(4) = ((compact >> 16) & 0xFF).toByte
+    if (size >= 2) bytes(5) = ((compact >> 8) & 0xFF).toByte
+    if (size >= 3) bytes(6) = (compact & 0xFF).toByte
+    decodeMPI(bytes)
+  }
+
+  /**
+    * @see Utils#decodeCompactBits(long)
+    */
+  def encodeCompactBits(requiredDifficulty: BigInt): Long = {
+    val value = requiredDifficulty.bigInteger
+    var result: Long = 0L
+    var size: Int = value.toByteArray.length
+    if (size <= 3) {
+      result = value.longValue << 8 * (3 - size)
+    } else {
+      result = value.shiftRight(8 * (size - 3)).longValue
+    }
+    // The 0x00800000 bit denotes the sign.
+    // Thus, if it is already set, divide the mantissa by 256 and increase the exponent.
+    if ((result & 0x00800000L) != 0) {
+      result >>= 8
+      size += 1
+    }
+    result |= size << 24
+    val a: Int = if (value.signum == -1) 0x00800000 else 0
+    result |= a
+    result
+  }
+
+
+  /** Parse 4 bytes from the byte array (starting at the offset) as unsigned 32-bit integer in big endian format. */
+  def readUint32BE(bytes: Array[Byte]): Long = ((bytes(0) & 0xffL) << 24) | ((bytes(1) & 0xffL) << 16) | ((bytes(2) & 0xffL) << 8) | (bytes(3) & 0xffL)
+
+  /**
+    * MPI encoded numbers are produced by the OpenSSL BN_bn2mpi function. They consist of
+    * a 4 byte big endian length field, followed by the stated number of bytes representing
+    * the number in big endian format (with a sign bit).
+    *
+    */
+  private def decodeMPI(mpi: Array[Byte]): BigInteger = {
+
+    val length: Int = readUint32BE(mpi).toInt
+    val buf = new Array[Byte](length)
+    System.arraycopy(mpi, 4, buf, 0, length)
+
+    if (buf.length == 0) {
+      BigInteger.ZERO
+    } else {
+      val isNegative: Boolean = (buf(0) & 0x80) == 0x80
+      if (isNegative) buf(0) = (buf(0) & 0x7f).toByte
+      val result: BigInteger = new BigInteger(buf)
+      if (isNegative) {
+        result.negate
+      } else {
+        result
+      }
+    }
+  }
+
+}