Merge pull request #972 from ScorexFoundation/i969

[6.0.0] Support Header (de)serialization

core/shared/src/main/scala/sigma/SigmaDsl.scala

@@ -459,6 +459,18 @@ trait Header {
 
   /** Miner votes for changing system parameters. */
   def votes: Coll[Byte] //3 bytes
+
+  /** Bytes which are coming from future versions of the protocol, so
+    * their meaning is not known to current version of Sigma, but they
+    * are stored to get the same id as future version users.
+    */
+  def unparsedBytes: Coll[Byte]
+
+  /**
+    * @return header bytes without proof of work, a PoW is generated over them
+    */
+  def serializeWithoutPoW: Coll[Byte]
+
 }
 
 /** Runtime representation of Context ErgoTree type.

data/shared/src/main/scala/org/ergoplatform/ErgoHeader.scala

@@ -0,0 +1,182 @@
+package org.ergoplatform
+
+import scorex.crypto.authds.ADDigest
+import scorex.crypto.hash.{Blake2b256, Digest32}
+import scorex.util.ModifierId
+import sigma.Colls
+import sigma.crypto.{BigIntegers, CryptoConstants, EcPointType}
+import sigma.serialization.{GroupElementSerializer, SigmaByteReader, SigmaByteWriter, SigmaSerializer}
+
+import scala.runtime.ScalaRunTime
+import scala.util.hashing.MurmurHash3
+
+
+
+/**
+  * Solution for an Autolykos PoW puzzle.
+  *
+  * In Autolykos v.1 all the four fields are used, in Autolykos v.2 only pk and n fields are used.
+  *
+  * @param pk - miner public key. Should be used to collect block rewards
+  * @param w  - one-time public key. Prevents revealing of miners secret
+  * @param n  - nonce (8 bytes)
+  * @param d  - distance between pseudo-random number, corresponding to nonce `n` and a secret,
+  *           corresponding to `pk`. The lower `d` is, the harder it was to find this solution.
+  */
+class AutolykosSolution(val pk: EcPointType,
+                        val w: EcPointType,
+                        val n: Array[Byte],
+                        val d: BigInt) {
+
+    val encodedPk: Array[Byte] = GroupElementSerializer.toBytes(pk)
+
+    override def hashCode(): Int = {
+        var h = pk.hashCode()
+        h = h * 31 + w.hashCode()
+        h = h * 31 + MurmurHash3.arrayHash(n)
+        h = h * 31 + d.hashCode()
+        h
+    }
+
+    override def equals(obj: Any): Boolean = {
+        obj match {
+            case other: AutolykosSolution =>
+                this.pk == other.pk &&
+                  this.n.sameElements(other.n) &&
+                  this.w == other.w &&
+                  this.d == other.d
+
+            case _ => false
+        }
+    }
+}
+
+
+object AutolykosSolution {
+    // "pk", "w" and "d" values for Autolykos v2 solution, where they not passed from outside
+    val pkForV2: EcPointType = CryptoConstants.dlogGroup.identity
+    val wForV2: EcPointType = CryptoConstants.dlogGroup.generator
+    val dForV2: BigInt = 0
+
+    object sigmaSerializerV1 extends SigmaSerializer[AutolykosSolution, AutolykosSolution] {
+        override def serialize(s: AutolykosSolution, w: SigmaByteWriter): Unit = {
+            GroupElementSerializer.serialize(s.pk, w)
+            GroupElementSerializer.serialize(s.w, w)
+            require(s.n.length == 8) // non-consensus check on prover side
+            w.putBytes(s.n)
+            val dBytes = BigIntegers.asUnsignedByteArray(s.d.bigInteger)
+            w.putUByte(dBytes.length)
+            w.putBytes(dBytes)
+        }
+
+        override def parse(r: SigmaByteReader): AutolykosSolution = {
+            val pk = GroupElementSerializer.parse(r)
+            val w = GroupElementSerializer.parse(r)
+            val nonce = r.getBytes(8)
+            val dBytesLength = r.getUByte()
+            val d = BigInt(BigIntegers.fromUnsignedByteArray(r.getBytes(dBytesLength)))
+            new AutolykosSolution(pk, w, nonce, d)
+        }
+    }
+
+    object sigmaSerializerV2 extends SigmaSerializer[AutolykosSolution, AutolykosSolution] {
+        override def serialize(s: AutolykosSolution, w: SigmaByteWriter): Unit = {
+            GroupElementSerializer.serialize(s.pk, w)
+            require(s.n.length == 8) // non-consensus check on prover side
+            w.putBytes(s.n)
+        }
+
+        override def parse(r: SigmaByteReader): AutolykosSolution = {
+            val pk = GroupElementSerializer.parse(r)
+            val nonce = r.getBytes(8)
+            new AutolykosSolution(pk, wForV2, nonce, dForV2)
+        }
+    }
+}
+
+/**
+  * Header of a block. It authenticates link to a previous block, other block sections
+  * (transactions, UTXO set transformation proofs, extension), UTXO set, votes for parameters
+  * to be changed and proof-of-work related data.
+  *
+  * @param version - protocol version
+  * @param parentId - id of a parent block header
+  * @param ADProofsRoot - digest of UTXO set transformation proofs
+  * @param stateRoot - AVL+ tree digest of UTXO set (after the block)
+  * @param transactionsRoot - Merkle tree digest of transactions in the block (BlockTransactions section)
+  * @param timestamp - block generation time reported by a miner
+  * @param nBits - difficulty encoded
+  * @param height - height of the block (genesis block height == 1)
+  * @param extensionRoot - Merkle tree digest of the extension section of the block
+  * @param powSolution - solution for the proof-of-work puzzle
+  * @param votes - votes for changing system parameters
+  * @param unparsedBytes - bytes from future versions of the protocol our version can't parse
+  * @param _bytes - serialized bytes of the header when not `null`
+  */
+case class ErgoHeader(override val version: ErgoHeader.Version,
+                      override val parentId: ModifierId,
+                      override val ADProofsRoot: Digest32,
+                      override val stateRoot: ADDigest, //33 bytes! extra byte with tree height here!
+                      override val transactionsRoot: Digest32,
+                      override val timestamp: ErgoHeader.Timestamp,
+                      override val nBits: Long, //actually it is unsigned int
+                      override val height: Int,
+                      override val extensionRoot: Digest32,
+                      powSolution: AutolykosSolution,
+                      override val votes: Array[Byte], //3 bytes
+                      override val unparsedBytes: Array[Byte],
+                      _bytes: Array[Byte]) extends
+  HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp,
+    nBits, height, extensionRoot, votes, unparsedBytes) {
+
+    lazy val bytes = if(_bytes != null) {
+        _bytes
+    } else {
+        ErgoHeader.sigmaSerializer.toBytes(this)
+    }
+
+    lazy val serializedId: Array[Byte] = Blake2b256.hash(bytes)
+
+    lazy val id = Colls.fromArray(serializedId)
+
+    override def hashCode(): Int = id.hashCode()
+
+    override def equals(other: Any): Boolean = other match {
+        case h: ErgoHeader => h.id == this.id
+        case _ => false
+    }
+}
+
+
+object ErgoHeader {
+
+    type Timestamp = Long
+
+    type Version = Byte
+
+    object sigmaSerializer extends SigmaSerializer[ErgoHeader, ErgoHeader] {
+        override def serialize(hdr: ErgoHeader, w: SigmaByteWriter): Unit = {
+            HeaderWithoutPowSerializer.serialize(hdr, w)
+            if (hdr.version == 1) {
+                AutolykosSolution.sigmaSerializerV1.serialize(hdr.powSolution, w)
+            } else {
+                AutolykosSolution.sigmaSerializerV2.serialize(hdr.powSolution, w)
+            }
+        }
+
+        override def parse(r: SigmaByteReader): ErgoHeader = {
+            val start = r.position
+            val headerWithoutPow = HeaderWithoutPowSerializer.parse(r)
+            val powSolution = if (headerWithoutPow.version == 1) {
+                AutolykosSolution.sigmaSerializerV1.parse(r)
+            } else {
+                AutolykosSolution.sigmaSerializerV2.parse(r)
+            }
+            val end = r.position
+            val len = end - start
+            r.position = start
+            val headerBytes = r.getBytes(len) // also moves position back to end
+            headerWithoutPow.toHeader(powSolution, headerBytes)
+        }
+    }
+}

data/shared/src/main/scala/org/ergoplatform/HeaderWithoutPow.scala

@@ -0,0 +1,139 @@
+package org.ergoplatform
+
+import scorex.crypto.authds.ADDigest
+import scorex.crypto.hash.Digest32
+import scorex.util.{ModifierId, bytesToId, idToBytes}
+import sigma.serialization.{SigmaByteReader, SigmaByteWriter, SigmaSerializer}
+import scorex.util.Extensions._
+
+/**
+  * Header without proof-of-work puzzle solution, see Header class description for details.
+  */
+class HeaderWithoutPow(val version:  Byte, // 1 byte
+                       val parentId: ModifierId, // 32 bytes
+                       val ADProofsRoot: Digest32, // 32 bytes
+                       val stateRoot: ADDigest, //33 bytes! extra byte with tree height here!
+                       val transactionsRoot: Digest32, // 32 bytes
+                       val timestamp: Long,
+                       val nBits: Long, //actually it is unsigned int
+                       val height: Int,
+                       val extensionRoot: Digest32,
+                       val votes: Array[Byte], //3 bytes
+                       val unparsedBytes: Array[Byte]) {
+  def toHeader(powSolution: AutolykosSolution, bytes: Array[Byte]): ErgoHeader =
+    ErgoHeader(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp,
+      nBits, height, extensionRoot, powSolution, votes, unparsedBytes, bytes)
+}
+
+object HeaderWithoutPow {
+
+  def apply(version: Byte, parentId: ModifierId, ADProofsRoot: Digest32, stateRoot: ADDigest,
+            transactionsRoot: Digest32, timestamp: Long, nBits: Long, height: Int,
+            extensionRoot: Digest32, votes: Array[Byte], unparsedBytes: Array[Byte]): HeaderWithoutPow = {
+    new HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp,
+      nBits, height, extensionRoot, votes, unparsedBytes)
+  }
+
+}
+
+object HeaderWithoutPowSerializer extends SigmaSerializer[HeaderWithoutPow, HeaderWithoutPow] {
+
+  override def serialize(h: HeaderWithoutPow, w: SigmaByteWriter): Unit = {
+    w.put(h.version)
+    w.putBytes(idToBytes(h.parentId))
+    w.putBytes(h.ADProofsRoot)
+    w.putBytes(h.transactionsRoot)
+    w.putBytes(h.stateRoot)
+    w.putULong(h.timestamp)
+    w.putBytes(h.extensionRoot)
+    DifficultySerializer.serialize(h.nBits, w)
+    w.putUInt(h.height.toLong)
+    w.putBytes(h.votes)
+
+    // For block version >= 2, this new byte encodes length of possible new fields.
+    // Set to 0 for now, so no new fields.
+    if (h.version > HeaderVersion.InitialVersion) {
+      w.putUByte(h.unparsedBytes.length)
+      w.putBytes(h.unparsedBytes)
+    }
+  }
+
+  override def parse(r: SigmaByteReader): HeaderWithoutPow = {
+    val version = r.getByte()
+    val parentId = bytesToId(r.getBytes(32))
+    val ADProofsRoot = Digest32 @@ r.getBytes(32)
+    val transactionsRoot = Digest32 @@ r.getBytes(32)
+    val stateRoot = ADDigest @@ r.getBytes(33)
+    val timestamp = r.getULong()
+    val extensionHash = Digest32 @@ r.getBytes(32)
+    val nBits = DifficultySerializer.parse(r)
+    val height = r.getUInt().toIntExact
+    val votes = r.getBytes(3)
+
+    // For block version >= 2, a new byte encodes length of possible new fields.
+    // If this byte > 0, we read new fields but do nothing, as semantics of the fields is not known.
+    val unparsedBytes = if (version > HeaderVersion.InitialVersion) {
+      val newFieldsSize = r.getUByte()
+      if (newFieldsSize > 0 && version > HeaderVersion.Interpreter60Version) {
+        // new bytes could be added only for block version >= 5
+        r.getBytes(newFieldsSize)
+      } else {
+        Array.emptyByteArray
+      }
+    } else {
+      Array.emptyByteArray
+    }
+
+    HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp,
+      nBits, height, extensionHash, votes, unparsedBytes)
+  }
+
+}
+
+
+object DifficultySerializer extends SigmaSerializer[Long, Long] {
+
+  /** Parse 4 bytes from the byte array (starting at the offset) as unsigned 32-bit integer in big endian format. */
+  def readUint32BE(bytes: Array[Byte]): Long = ((bytes(0) & 0xffL) << 24) | ((bytes(1) & 0xffL) << 16) | ((bytes(2) & 0xffL) << 8) | (bytes(3) & 0xffL)
+
+  def uint32ToByteArrayBE(value: Long): Array[Byte] = {
+    Array(0xFF & (value >> 24), 0xFF & (value >> 16), 0xFF & (value >> 8), 0xFF & value).map(_.toByte)
+  }
+
+  override def serialize(obj: Long, w: SigmaByteWriter): Unit = {
+    w.putBytes(uint32ToByteArrayBE(obj))
+  }
+
+  override def parse(r: SigmaByteReader): Long = {
+    readUint32BE(r.getBytes(4))
+  }
+
+}
+
+object HeaderVersion {
+  type Value = Byte
+
+  /**
+    * Block version during mainnet launch
+    */
+  val InitialVersion: Value = 1.toByte
+
+  /**
+    * Block version after the Hardening hard-fork
+    * Autolykos v2 PoW, witnesses in transactions Merkle tree
+    */
+  val HardeningVersion: Value = 2.toByte
+
+  /**
+    * Block version after the 5.0 soft-fork
+    * 5.0 interpreter with JITC, monotonic height rule (EIP-39)
+    */
+  val Interpreter50Version: Value = 3.toByte
+
+  /**
+    * Block version after the 6.0 soft-fork
+    * 6.0 interpreter (EIP-50)
+    */
+  val Interpreter60Version: Value = 4.toByte
+
+}