Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[6.0] Lazy default for Option.getOrElse and Coll.getOrElse #1008

Merged
merged 10 commits into from
Oct 21, 2024
Merged

[6.0] Lazy default for Option.getOrElse and Coll.getOrElse #1008

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
dd62bd7
failing tests
kushti Jun 11, 2024
aaef77e
close #906: lazy evaluation of default in Option.getOrElse and Coll.g…
kushti Jun 11, 2024
8f9b549
Merge branch 'v6.0.0' of github.com:ScorexFoundation/sigmastate-inter…
kushti Jul 30, 2024
4c63a60
LSV5 & LSV6 tests
kushti Jul 30, 2024
61f90cd
assertEXceptionThrown
kushti Jul 31, 2024
26809c9
passing but improper changedFeature test
kushti Aug 2, 2024
57ba522
merging w. 6.0.0
kushti Aug 8, 2024
8da3575
fixed tests in LSV6
kushti Aug 9, 2024
8f45909
merging w 6.0.0
kushti Sep 27, 2024
f151e1d
merging w. 6.0.0
kushti Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
*.fdb_latexmk
*.gz


yarn.lock
*.log
yarn.lock
Expand Down
43 changes: 33 additions & 10 deletions data/shared/src/main/scala/sigma/ast/transformers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import sigma.eval.ErgoTreeEvaluator.DataEnv
import sigma.serialization.CoreByteWriter.ArgInfo
import sigma.serialization.OpCodes
import sigma.serialization.ValueCodes.OpCode
import sigma.{Box, Coll, Evaluation}
import sigma.{Box, Coll, Evaluation, VersionContext}

// TODO refactor: remove this trait as it doesn't have semantic meaning

Expand Down Expand Up @@ -258,10 +258,22 @@ case class ByIndex[V <: SType](input: Value[SCollection[V]],
val indexV = index.evalTo[Int](env)
default match {
case Some(d) =>
val dV = d.evalTo[V#WrappedType](env)
Value.checkType(d, dV) // necessary because cast to V#WrappedType is erased
addCost(ByIndex.costKind)
inputV.getOrElse(indexV, dV)
if (VersionContext.current.isV6SoftForkActivated) {
// lazy evaluation of default in 6.0
addCost(ByIndex.costKind)
if (inputV.isDefinedAt(indexV)) {
inputV.apply(indexV)
} else {
val dV = d.evalTo[V#WrappedType](env)
Value.checkType(d, dV) // necessary because cast to V#WrappedType is erased
inputV.getOrElse(indexV, dV)
}
} else {
val dV = d.evalTo[V#WrappedType](env)
Value.checkType(d, dV) // necessary because cast to V#WrappedType is erased
addCost(ByIndex.costKind)
inputV.getOrElse(indexV, dV)
}
case _ =>
addCost(ByIndex.costKind)
inputV.apply(indexV)
Expand Down Expand Up @@ -613,11 +625,22 @@ case class OptionGetOrElse[V <: SType](input: Value[SOption[V]], default: Value[
override val opType = SFunc(IndexedSeq(input.tpe, tpe), tpe)
override def tpe: V = input.tpe.elemType
protected final override def eval(env: DataEnv)(implicit E: ErgoTreeEvaluator): Any = {
val inputV = input.evalTo[Option[V#WrappedType]](env)
val dV = default.evalTo[V#WrappedType](env) // TODO v6.0: execute lazily (see https://github.com/ScorexFoundation/sigmastate-interpreter/issues/906)
Value.checkType(default, dV) // necessary because cast to V#WrappedType is erased
addCost(OptionGetOrElse.costKind)
inputV.getOrElse(dV)
if(VersionContext.current.isV6SoftForkActivated) {
// lazy evaluation of default in 6.0
val inputV = input.evalTo[Option[V#WrappedType]](env)
addCost(OptionGetOrElse.costKind)
inputV.getOrElse {
val dV = default.evalTo[V#WrappedType](env)
Value.checkType(default, dV) // necessary because cast to V#WrappedType is erased
dV
}
} else {
val inputV = input.evalTo[Option[V#WrappedType]](env)
val dV = default.evalTo[V#WrappedType](env)
Value.checkType(default, dV) // necessary because cast to V#WrappedType is erased
addCost(OptionGetOrElse.costKind)
inputV.getOrElse(dV)
}
}
}
object OptionGetOrElse extends ValueCompanion with FixedCostValueCompanion {
Expand Down
19 changes: 11 additions & 8 deletions sc/shared/src/test/scala/sigma/LanguageSpecificationV5.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8059,6 +8059,7 @@ class LanguageSpecificationV5 extends LanguageSpecificationBase { suite =>
)
)
)
if(!VersionContext.current.isV6SoftForkActivated) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Protocol change like this can be tested using changeFeature (see other tests) in combination with newVersionedResults (also see code). In this case both old and new behaviour will be tested.

In principle this test case can be moved to LSV6.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changedFeature is not enough here, you need for versioned verifyCases, as in 6.0 costing trace would be different in some cases due to difference in default evaluation (at the same time, it wouldnt provide higher cost, so the change is secure costing-wise).

Seems like *Feature machinery is overly complex, but still not enough to describe protocol changes, thus I would recommend to decide what it is actually testing (or should test), and then follow simpler approaches

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please see other examples where changedFeature is used, there are test cases where cost trace depend on the version and you can pass different traces to the constructor of Expected.
The perceived complexity is due to learning curve.

Copy link
Member Author

@kushti kushti Aug 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changedFeature is tied to V4/V5 switch, see checkExpected in ChangedFeature / checkVerify functions in SigmaDslTesting. So it cant be used in V6 testing without modifications it seems. So it is definitely not about learning curve, changedFeature is just not ready for V6

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aslesarenko see #1024 for example

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

verifyCases(
// (coll, (index, default))
{
Expand Down Expand Up @@ -8129,7 +8130,7 @@ class LanguageSpecificationV5 extends LanguageSpecificationBase { suite =>
)
)
)
))
))}
}

property("Tuple size method equivalence") {
Expand Down Expand Up @@ -8696,13 +8697,15 @@ class LanguageSpecificationV5 extends LanguageSpecificationBase { suite =>
"{ (x: Option[Long]) => x.isDefined }",
FuncValue(Vector((1, SOption(SLong))), OptionIsDefined(ValUse(1, SOption(SLong))))))

verifyCases(
Seq(
(None -> Expected(Success(1L), 1766, costDetails3, 1766, Seq.fill(4)(2006))),
(Some(10L) -> Expected(Success(10L), 1766, costDetails3, 1766, Seq.fill(4)(2006)))),
existingFeature({ (x: Option[Long]) => x.getOrElse(1L) },
"{ (x: Option[Long]) => x.getOrElse(1L) }",
FuncValue(Vector((1, SOption(SLong))), OptionGetOrElse(ValUse(1, SOption(SLong)), LongConstant(1L)))))
if (!VersionContext.current.isV6SoftForkActivated) {
verifyCases(
Seq(
(None -> Expected(Success(1L), 1766, costDetails3, 1766, Seq.fill(4)(2006))),
(Some(10L) -> Expected(Success(10L), 1766, costDetails3, 1766, Seq.fill(4)(2006)))),
existingFeature({ (x: Option[Long]) => x.getOrElse(1L) },
"{ (x: Option[Long]) => x.getOrElse(1L) }",
FuncValue(Vector((1, SOption(SLong))), OptionGetOrElse(ValUse(1, SOption(SLong)), LongConstant(1L)))))
}

verifyCases(
Seq(
Expand Down
96 changes: 95 additions & 1 deletion sc/shared/src/test/scala/sigma/LanguageSpecificationV6.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ import sigma.ast.{SInt, _}
import sigma.data.{CBigInt, CBox, CHeader, CSigmaDslBuilder, ExactNumeric, PairOfCols, RType}
import sigma.eval.{CostDetails, SigmaDsl, TracedCost}
import sigma.serialization.ValueCodes.OpCode
import sigma.util.Extensions.{BooleanOps, IntOps}
import sigma.data.{RType}
import sigma.serialization.ValueCodes.OpCode
import sigma.util.Extensions.{BooleanOps, ByteOps, IntOps, LongOps}
import sigmastate.exceptions.MethodNotFound
import sigmastate.utils.Extensions.ByteOpsForSigma
import sigmastate.utils.Helpers
Expand Down Expand Up @@ -1523,6 +1525,98 @@ class LanguageSpecificationV6 extends LanguageSpecificationBase { suite =>
}



property("Option.getOrElse with lazy default") {

val trace = TracedCost(
Array(
FixedCostItem(Apply),
FixedCostItem(FuncValue),
FixedCostItem(GetVar),
FixedCostItem(OptionGet),
FixedCostItem(FuncValue.AddToEnvironmentDesc, FixedCost(JitCost(5))),
FixedCostItem(ValUse),
FixedCostItem(OptionGetOrElse)
)
)

verifyCases(
Seq(
Some(2L) -> Expected(Failure(new java.lang.ArithmeticException("/ by zero")), 6, trace, 1793,
newVersionedResults = {
expectedSuccessForAllTreeVersions(2L, 2015, trace)
} ),
None -> Expected(Failure(new java.lang.ArithmeticException("/ by zero")), 6, trace, 1793)
),
changedFeature(
changedInVersion = VersionContext.V6SoftForkVersion,
{ (x: Option[Long]) => val default = 1 / 0L; x.getOrElse(default) },
{ (x: Option[Long]) => if (VersionContext.current.isV6SoftForkActivated) {x.getOrElse(1 / 0L)} else {val default = 1 / 0L; x.getOrElse(default)} },
"{ (x: Option[Long]) => x.getOrElse(1 / 0L) }",
FuncValue(
Array((1, SOption(SLong))),
OptionGetOrElse(
ValUse(1, SOption(SLong)),
ArithOp(LongConstant(1L), LongConstant(0L), OpCode @@ (-99.toByte))
)
),
allowNewToSucceed = true
)
)
}

property("Coll getOrElse with lazy default") {

val trace = TracedCost(
Array(
FixedCostItem(Apply),
FixedCostItem(FuncValue),
FixedCostItem(GetVar),
FixedCostItem(OptionGet),
FixedCostItem(FuncValue.AddToEnvironmentDesc, FixedCost(JitCost(5))),
FixedCostItem(ValUse),
FixedCostItem(Constant),
FixedCostItem(ByIndex)
)
)

def scalaFuncNew(x: Coll[Int]) = {
if (VersionContext.current.isV6SoftForkActivated) {
x.toArray.toIndexedSeq.headOption.getOrElse(1 / 0)
} else scalaFuncOld(x)
}

def scalaFuncOld(x: Coll[Int]) = {
x.getOrElse(0, 1 / 0)
}

verifyCases(
Seq(
Coll(1) -> Expected(Failure(new java.lang.ArithmeticException("/ by zero")), 6, trace, 1793,
newVersionedResults = {
expectedSuccessForAllTreeVersions(1, 2029, trace)
} ),
Coll[Int]() -> Expected(Failure(new java.lang.ArithmeticException("/ by zero")), 6, trace, 1793)
),
changedFeature(
changedInVersion = VersionContext.V6SoftForkVersion,
scalaFuncOld,
scalaFuncNew,
"{ (x: Coll[Int]) => x.getOrElse(0, 1 / 0) }",
FuncValue(
Array((1, SCollectionType(SInt))),
ByIndex(
ValUse(1, SCollectionType(SInt)),
IntConstant(0),
Some(ArithOp(IntConstant(1), IntConstant(0), OpCode @@ (-99.toByte)))
)
),
allowNewToSucceed = true
)
)
}


property("Global - fromBigEndianBytes") {
import sigma.data.OrderingOps.BigIntOrdering

Expand Down
2 changes: 1 addition & 1 deletion sc/shared/src/test/scala/sigma/SigmaDslTesting.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -771,7 +771,7 @@ class SigmaDslTesting extends AnyPropSpec
override def checkExpected(input: A, expected: Expected[B]): Unit = {
// check the new implementation with Scala semantic function
val newRes = VersionContext.withVersions(activatedVersionInTests, ergoTreeVersionInTests) {
checkEq(scalaFuncNew)(newF)(input)
checkEq(scalaFuncNew)(newF)(input)
}

if (VersionContext.current.activatedVersion < changedInVersion) {
Expand Down
37 changes: 37 additions & 0 deletions sc/shared/src/test/scala/sigmastate/utxo/BasicOpsSpecification.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1061,6 +1061,43 @@ class BasicOpsSpecification extends CompilerTestingCommons
}
}

property("Lazy evaluation of default in Option.getOrElse") {
val customExt = Map (
1.toByte -> IntConstant(5)
).toSeq
def optTest() = test("getOrElse", env, customExt,
"""{
| getVar[Int](1).getOrElse(getVar[Int](44).get) > 0
|}
|""".stripMargin,
null
)

if (VersionContext.current.isV6SoftForkActivated) {
optTest()
} else {
assertExceptionThrown(optTest(), _.isInstanceOf[NoSuchElementException])
}
}

property("Lazy evaluation of default in Coll.getOrElse") {
def optTest() = test("getOrElse", env, ext,
"""{
| val c = Coll[Int](1)
| c.getOrElse(0, getVar[Int](44).get) > 0 &&
| c.getOrElse(1, c.getOrElse(0, getVar[Int](44).get)) > 0
|}
|""".stripMargin,
null
)

if(VersionContext.current.isV6SoftForkActivated) {
optTest()
} else {
an[Exception] shouldBe thrownBy(optTest())
}
}

property("Relation operations") {
test("R1", env, ext,
"{ allOf(Coll(getVar[Boolean](trueVar).get, true, true)) }",
Expand Down
Loading