[6.0] Fix SubstConstants: serialize ErgoTree size

core/shared/src/main/scala/sigma/SigmaDsl.scala

@@ -727,6 +727,9 @@ trait SigmaDslBuilder {
   /** Construct a new authenticated dictionary with given parameters and tree root digest. */
   def avlTree(operationFlags: Byte, digest: Coll[Byte], keyLength: Int, valueLengthOpt: Option[Int]): AvlTree
 
+  /** Serializes the given `value` into bytes using the default serialization format. */
+  def serialize[T](value: T)(implicit cT: RType[T]): Coll[Byte]
+
   /** Returns a byte-wise XOR of the two collections of bytes. */
   def xor(l: Coll[Byte], r: Coll[Byte]): Coll[Byte]
 }

core/shared/src/main/scala/sigma/reflection/ReflectionData.scala

@@ -441,6 +441,10 @@ object ReflectionData {
         mkMethod(clazz, "sha256", Array[Class[_]](cColl)) { (obj, args) =>
           obj.asInstanceOf[SigmaDslBuilder].sha256(args(0).asInstanceOf[Coll[Byte]])
         },
+        mkMethod(clazz, "serialize", Array[Class[_]](classOf[Object], classOf[RType[_]])) { (obj, args) =>
+          obj.asInstanceOf[SigmaDslBuilder].serialize[Any](
+            args(0).asInstanceOf[Any])(args(1).asInstanceOf[RType[Any]])
+        },
         mkMethod(clazz, "decodePoint", Array[Class[_]](cColl)) { (obj, args) =>
           obj.asInstanceOf[SigmaDslBuilder].decodePoint(args(0).asInstanceOf[Coll[Byte]])
         }

data/shared/src/main/scala/org/ergoplatform/validation/ValidationRules.scala

@@ -155,6 +155,20 @@ object ValidationRules {
     override protected lazy val settings: SigmaValidationSettings = currentSettings
   }
 
+  object CheckMinimalErgoTreeVersion extends ValidationRule(1016,
+    "ErgoTree should have at least required version") with SoftForkWhenReplaced {
+    override protected lazy val settings: SigmaValidationSettings = currentSettings
+
+    final def apply(currentVersion: Byte, minVersion: Byte): Unit = {
+      checkRule()
+      if (currentVersion < minVersion) {
+        throwValidationException(
+          new SigmaException(s"ErgoTree should have at least $minVersion version, but was $currentVersion"),
+          Array(currentVersion, minVersion))
+      }
+    }
+  }
+
   val ruleSpecs: Seq[ValidationRule] = Seq(
     CheckDeserializedScriptType,
     CheckDeserializedScriptIsSigmaProp,
@@ -171,7 +185,8 @@ object ValidationRules {
     CheckHeaderSizeBit,
     CheckCostFuncOperation,
     CheckPositionLimit,
-    CheckLoopLevelInCostFunction
+    CheckLoopLevelInCostFunction,
+    CheckMinimalErgoTreeVersion
   )
 
   /** Validation settings that correspond to the current version of the ErgoScript implementation.

data/shared/src/main/scala/sigma/SigmaDataReflection.scala

@@ -322,6 +322,11 @@ object SigmaDataReflection {
             args(1).asInstanceOf[SigmaDslBuilder],
             args(2).asInstanceOf[Coll[Byte]],
             args(3).asInstanceOf[Coll[Byte]])(args(4).asInstanceOf[ErgoTreeEvaluator])
+        },
+        mkMethod(clazz, "serialize_eval", Array[Class[_]](classOf[MethodCall], classOf[SigmaDslBuilder], classOf[Object], classOf[ErgoTreeEvaluator])) { (obj, args) =>
+          obj.asInstanceOf[SGlobalMethods.type].serialize_eval(args(0).asInstanceOf[MethodCall],
+            args(1).asInstanceOf[SigmaDslBuilder],
+            args(2).asInstanceOf[SType#WrappedType])(args(3).asInstanceOf[ErgoTreeEvaluator])
         }
       )
     )

data/shared/src/main/scala/sigma/ast/ErgoTree.scala

@@ -228,6 +228,17 @@ object ErgoTree {
 
   type HeaderType = HeaderType.Type
 
+  /** Convenience methods for working with ErgoTree headers. */
+  implicit class HeaderTypeOps(val header: HeaderType) extends AnyVal {
+    /** Update the version bits of the given header byte with the given version value,
+      * leaving all other bits unchanged.
+      */
+    def withVersion(version: Byte): HeaderType = ErgoTree.headerWithVersion(header, version)
+
+    /**  Sets the constant segregation bit in the given header. */
+    def withConstantSegregation: HeaderType = ErgoTree.setConstantSegregation(header)
+  }
+
   /** Current version of ErgoTree serialization format (aka bite-code language version) */
   val VersionFlag: Byte = VersionContext.MaxSupportedScriptVersion
 

data/shared/src/main/scala/sigma/ast/SMethod.scala

@@ -50,31 +50,37 @@ case class MethodIRInfo(
 
 /** Represents method descriptor.
   *
-  * @param objType type or type constructor descriptor
-  * @param name    method name
-  * @param stype   method signature type,
-  *                where `stype.tDom`` - argument type and
-  *                `stype.tRange` - method result type.
-  * @param methodId method code, it should be unique among methods of the same objType.
-  * @param costKind cost descriptor for this method
-  * @param irInfo  meta information connecting SMethod with ErgoTree (see [[MethodIRInfo]])
-  * @param docInfo optional human readable method description data
-  * @param costFunc optional specification of how the cost should be computed for the
-  *                 given method call (See ErgoTreeEvaluator.calcCost method).
+  * @param objType         type or type constructor descriptor
+  * @param name            method name
+  * @param stype           method signature type,
+  *                        where `stype.tDom`` - argument type and
+  *                        `stype.tRange` - method result type.
+  * @param methodId        method code, it should be unique among methods of the same objType.
+  * @param costKind        cost descriptor for this method
+  * @param explicitTypeArgs list of type parameters which require explicit
+  *                        serialization in [[MethodCall]]s (i.e for deserialize[T], getVar[T], getReg[T])
+  * @param irInfo          meta information connecting SMethod with ErgoTree (see [[MethodIRInfo]])
+  * @param docInfo         optional human readable method description data
+  * @param costFunc        optional specification of how the cost should be computed for the
+  *                        given method call (See ErgoTreeEvaluator.calcCost method).
   */
 case class SMethod(
     objType: MethodsContainer,
     name: String,
     stype: SFunc,
     methodId: Byte,
     costKind: CostKind,
+    explicitTypeArgs: Seq[STypeVar],
     irInfo: MethodIRInfo,
     docInfo: Option[OperationInfo],
     costFunc: Option[MethodCostFunc]) {
 
   /** Operation descriptor of this method. */
   lazy val opDesc = MethodDesc(this)
 
+  /** Return true if this method has runtime type parameters */
+  def hasExplicitTypeArgs: Boolean = explicitTypeArgs.nonEmpty
+
   /** Finds and keeps the [[RMethod]] instance which corresponds to this method descriptor.
     * The lazy value is forced only if irInfo.javaMethod == None
     */
@@ -284,9 +290,11 @@ object SMethod {
   /** Convenience factory method. */
   def apply(objType: MethodsContainer, name: String, stype: SFunc,
       methodId: Byte,
-      costKind: CostKind): SMethod = {
+      costKind: CostKind,
+      explicitTypeArgs: Seq[STypeVar] = Nil
+  ): SMethod = {
     SMethod(
-      objType, name, stype, methodId, costKind,
+      objType, name, stype, methodId, costKind, explicitTypeArgs,
       MethodIRInfo(None, None, None), None, None)
   }
 

data/shared/src/main/scala/sigma/ast/SigmaPredef.scala

@@ -403,6 +403,24 @@ object SigmaPredef {
           ArgInfo("default", "optional default value, if register is not available")))
     )
 
+    val SerializeFunc = PredefinedFunc("serialize",
+      Lambda(Seq(paramT), Array("value" -> tT), SByteArray, None),
+      PredefFuncInfo(
+        { case (_, args @ Seq(value)) =>
+          MethodCall.typed[Value[SCollection[SByte.type]]](
+            Global,
+            SGlobalMethods.serializeMethod.withConcreteTypes(Map(tT -> value.tpe)),
+            args.toIndexedSeq,
+            Map()
+          )
+        }),
+      OperationInfo(MethodCall,
+        """
+        """.stripMargin,
+        Seq(ArgInfo("value", ""))
+      )
+    )
+
     val globalFuncs: Map[String, PredefinedFunc] = Seq(
       AllOfFunc,
       AnyOfFunc,
@@ -430,7 +448,8 @@ object SigmaPredef {
       AvlTreeFunc,
       SubstConstantsFunc,
       ExecuteFromVarFunc,
-      ExecuteFromSelfRegFunc
+      ExecuteFromSelfRegFunc,
+      SerializeFunc
     ).map(f => f.name -> f).toMap
 
     def comparisonOp(symbolName: String, opDesc: ValueCompanion, desc: String, args: Seq[ArgInfo]) = {
@@ -544,7 +563,7 @@ object SigmaPredef {
 
     val funcs: Map[String, PredefinedFunc] = globalFuncs ++ infixFuncs ++ unaryFuncs
 
-    /** WARNING: This operations are not used in frontend, and should be be used.
+    /** WARNING: This operations are not used in frontend, and should not be used.
       * They are used in SpecGen only the source of metadata for the corresponding ErgoTree nodes.
       */
     val specialFuncs: Map[String, PredefinedFunc] = Seq(

data/shared/src/main/scala/sigma/ast/methods.scala

@@ -1,11 +1,12 @@
 package sigma.ast
 
 import org.ergoplatform._
+import org.ergoplatform.validation.ValidationRules.CheckMinimalErgoTreeVersion
 import org.ergoplatform.validation._
 import sigma._
 import sigma.ast.SCollection.{SBooleanArray, SBoxArray, SByteArray, SByteArray2, SHeaderArray}
 import sigma.ast.SMethod.{MethodCallIrBuilder, MethodCostFunc, javaMethodOf}
-import sigma.ast.SType.TypeCode
+import sigma.ast.SType.{TypeCode, paramT, tT}
 import sigma.ast.syntax.{SValue, ValueOps}
 import sigma.data.OverloadHack.Overloaded1
 import sigma.data.{DataValueComparer, KeyValueColl, Nullable, RType, SigmaConstants}
@@ -1519,9 +1520,38 @@ case object SGlobalMethods extends MonoTypeMethods {
     Xor.xorWithCosting(ls, rs)
   }
 
-  protected override def getMethods() = super.getMethods() ++ Seq(
-    groupGeneratorMethod,
-    xorMethod
-  )
+  lazy val serializeMethod = SMethod(this, "serialize",
+    SFunc(Array(SGlobal, tT), SByteArray, Array(paramT)), 3, DynamicCost)
+      .withIRInfo(MethodCallIrBuilder)
+      .withInfo(MethodCall, "",
+        ArgInfo("value", "value to be serialized"))
+
+
+  /** Implements evaluation of Global.serialize method call ErgoTree node.
+    * Called via reflection based on naming convention.
+    * @see SMethod.evalMethod
+    */
+  def serialize_eval(mc: MethodCall, G: SigmaDslBuilder, value: SType#WrappedType)
+      (implicit E: ErgoTreeEvaluator): Coll[Byte] = {
+    // TODO v6.0: accumulate cost
+    CheckMinimalErgoTreeVersion(E.context.currentErgoTreeVersion, VersionContext.V6SoftForkVersion)
+    val t = Evaluation.stypeToRType(mc.args(0).tpe)
+    G.serialize(value)(t)
+  }
+
+  protected override def getMethods() = super.getMethods() ++ {
+    if (VersionContext.current.isV6SoftForkActivated) {
+      Seq(
+        groupGeneratorMethod,
+        xorMethod,
+        serializeMethod
+      )
+    } else {
+      Seq(
+        groupGeneratorMethod,
+        xorMethod
+      )
+    }
+  }
 }
 

data/shared/src/main/scala/sigma/ast/values.scala

@@ -1298,6 +1298,8 @@ case class MethodCall(
     method: SMethod,
     args: IndexedSeq[Value[SType]],
     typeSubst: Map[STypeVar, SType]) extends Value[SType] {
+  require(method.explicitTypeArgs.forall(tyArg => typeSubst.contains(tyArg)),
+    s"Runtime Generic method call should have concrete type for each runtime type parameter, but was: $this")
   override def companion = if (args.isEmpty) PropertyCall else MethodCall
 
   override def opType: SFunc = SFunc(obj.tpe +: args.map(_.tpe), tpe)

data/shared/src/main/scala/sigma/data/CSigmaDslBuilder.scala

@@ -5,13 +5,13 @@ import org.ergoplatform.ErgoBox
 import org.ergoplatform.validation.ValidationRules
 import scorex.crypto.hash.{Blake2b256, Sha256}
 import scorex.utils.Longs
-import sigma.ast.{AtLeast, SubstConstants}
+import sigma.ast.{AtLeast, SType, SubstConstants}
 import sigma.crypto.{CryptoConstants, EcPointType, Ecp}
 import sigma.eval.Extensions.EvalCollOps
-import sigma.serialization.{GroupElementSerializer, SigmaSerializer}
+import sigma.serialization.{DataSerializer, GroupElementSerializer, SigmaSerializer}
 import sigma.util.Extensions.BigIntegerOps
 import sigma.validation.SigmaValidationSettings
-import sigma.{AvlTree, BigInt, Box, Coll, CollBuilder, GroupElement, SigmaDslBuilder, SigmaProp, VersionContext}
+import sigma.{AvlTree, BigInt, Box, Coll, CollBuilder, Evaluation, GroupElement, SigmaDslBuilder, SigmaProp, VersionContext}
 
 import java.math.BigInteger
 
@@ -200,6 +200,14 @@ class CSigmaDslBuilder extends SigmaDslBuilder { dsl =>
     val p = GroupElementSerializer.parse(r)
     this.GroupElement(p)
   }
+
+  /** Serializes the given `value` into bytes using the default serialization format. */
+  override def serialize[T](value: T)(implicit cT: RType[T]): Coll[Byte] = {
+    val tpe = Evaluation.rtypeToSType(cT)
+    val w = SigmaSerializer.startWriter()
+    DataSerializer.serialize(value.asInstanceOf[SType#WrappedType], tpe, w)
+    Colls.fromArray(w.toBytes)
+  }
 }
 
 /** Default singleton instance of Global object, which implements global ErgoTree functions. */

data/shared/src/main/scala/sigma/serialization/ErgoTreeSerializer.scala

@@ -287,6 +287,9 @@ class ErgoTreeSerializer {
     * allow to use serialized scripts as pre-defined templates.
     * See [[SubstConstants]] for details.
     *
+    * Note, this operation doesn't require (de)serialization of ErgoTree expression,
+    * thus it is more efficient than serialization roundtrip.
+    *
     * @param scriptBytes serialized ErgoTree with ConstantSegregationFlag set to 1.
     * @param positions   zero based indexes in ErgoTree.constants array which
     *                    should be replaced with new values
@@ -304,39 +307,62 @@ class ErgoTreeSerializer {
       s"expected positions and newVals to have the same length, got: positions: ${positions.toSeq},\n newVals: ${newVals.toSeq}")
     val r = SigmaSerializer.startReader(scriptBytes)
     val (header, _, constants, treeBytes) = deserializeHeaderWithTreeBytes(r)
-    val w = SigmaSerializer.startWriter()
-    w.put(header)
+    val nConstants = constants.length
+
+    val resBytes = if (VersionContext.current.isJitActivated) {
+      // need to measure the serialized size of the new constants
+      // by serializing them into a separate writer
+      val constW = SigmaSerializer.startWriter()
 
-    if (VersionContext.current.isJitActivated) {
       // The following `constants.length` should not be serialized when segregation is off
       // in the `header`, because in this case there is no `constants` section in the
       // ErgoTree serialization format. Thus, applying this `substituteConstants` for
       // non-segregated trees will return non-parsable ErgoTree bytes (when
       // `constants.length` is put in `w`).
       if (ErgoTree.isConstantSegregation(header)) {
-        w.putUInt(constants.length)
+        constW.putUInt(constants.length)
       }
 
       // The following is optimized O(nConstants + position.length) implementation
-      val nConstants = constants.length
       if (nConstants > 0) {
         val backrefs = getPositionsBackref(positions, nConstants)
         cfor(0)(_ < nConstants, _ + 1) { i =>
           val c = constants(i)
           val iPos = backrefs(i) // index to `positions`
           if (iPos == -1) {
             // no position => no substitution, serialize original constant
-            constantSerializer.serialize(c, w)
+            constantSerializer.serialize(c, constW)
           } else {
-            assert(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
+            require(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
             val newConst = newVals(iPos)
             require(c.tpe == newConst.tpe,
               s"expected new constant to have the same ${c.tpe} tpe, got ${newConst.tpe}")
-            constantSerializer.serialize(newConst, w)
+            constantSerializer.serialize(newConst, constW)
           }
         }
       }
+
+      val constBytes = constW.toBytes // nConstants + serialized new constants
+
+      // start composing the resulting tree bytes
+      val w = SigmaSerializer.startWriter()
+      w.put(header) // header byte
+
+      if (VersionContext.current.isV6SoftForkActivated) {
+        // fix in v6.0 to save tree size to respect size bit of the original tree
+        if (ErgoTree.hasSize(header)) {
+          val size = constBytes.length + treeBytes.length
+          w.putUInt(size)     // tree size
+        }
+      }
+
+      w.putBytes(constBytes)  // constants section
+      w.putBytes(treeBytes)   // tree section
+      w.toBytes
     } else {
+      val w = SigmaSerializer.startWriter()
+      w.put(header)
+
       // for v4.x compatibility we save constants.length here (see the above comment to
       // understand the consequences)
       w.putUInt(constants.length)
@@ -357,10 +383,12 @@ class ErgoTreeSerializer {
         case (c, _) =>
           constantSerializer.serialize(c, w)
       }
+
+      w.putBytes(treeBytes)
+      w.toBytes
     }
 
-    w.putBytes(treeBytes)
-    (w.toBytes, constants.length)
+    (resBytes, nConstants)
   }
 
 }