diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test b/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test index 184d41b5aeb..6998d4af7e7 100644 --- a/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test @@ -54,4 +54,8 @@ metadata { "outputs.biscayne_new_engine_functions.bigIntFloatComparison": 10.0 "outputs.biscayne_new_engine_functions.minMaxIntFloatComposition": 1.0 "outputs.biscayne_new_engine_functions.maxIntVsMaxFloat": 1.79769313E+308 + + "outputs.biscayne_new_engine_functions.with_suffixes.0": "aaaS" + "outputs.biscayne_new_engine_functions.with_suffixes.1": "bbbS" + "outputs.biscayne_new_engine_functions.with_suffixes.2": "cccS" } diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl index 220fcb535e5..f7b3d0de7fd 100644 --- a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl @@ -4,7 +4,7 @@ workflow biscayne_new_engine_functions { meta { description: "This test makes sure that these functions work in a real workflow" - functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key" ] + functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "suffix" ] } Map[String, Int] x_map_in = {"a": 1, "b": 2, "c": 3} @@ -15,6 +15,8 @@ workflow biscayne_new_engine_functions { Array[Pair[String,Int]] z_pairs_in = [("a", 1), ("b", 2), ("a", 3)] + Array[String] some_strings = ["aaa", "bbb", "ccc"] + Int smallestInt = 1 Float smallFloat = 2.718 Float bigFloat = 3.141 @@ -48,6 +50,10 @@ workflow biscayne_new_engine_functions { Float bigIntFloatComparison = max(bigFloat, biggestInt) # 10.0 Float minMaxIntFloatComposition = min(max(biggestInt, smallFloat), smallestInt) # 1.0 Float maxIntVsMaxFloat = max(maxInt, maxFloat) + + # suffix(): + # ================================================= + Array[String] with_suffixes = suffix("S", some_strings) } } diff --git a/wdl/model/draft3/src/main/scala/wdl/model/draft3/elements/ExpressionElement.scala b/wdl/model/draft3/src/main/scala/wdl/model/draft3/elements/ExpressionElement.scala index a2fc6202193..91f34d064ae 100644 --- a/wdl/model/draft3/src/main/scala/wdl/model/draft3/elements/ExpressionElement.scala +++ b/wdl/model/draft3/src/main/scala/wdl/model/draft3/elements/ExpressionElement.scala @@ -164,6 +164,12 @@ object ExpressionElement { override def arg1: ExpressionElement = prefix override def arg2: ExpressionElement = array } + + final case class Suffix(suffix: ExpressionElement, array: ExpressionElement) extends TwoParamFunctionCallElement { + override def arg1: ExpressionElement = suffix + override def arg2: ExpressionElement = array + } + final case class Min(arg1: ExpressionElement, arg2: ExpressionElement) extends TwoParamFunctionCallElement final case class Max(arg1: ExpressionElement, arg2: ExpressionElement) extends TwoParamFunctionCallElement final case class Sep(arg1: ExpressionElement, arg2: ExpressionElement) extends TwoParamFunctionCallElement diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/AstToNewExpressionElements.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/AstToNewExpressionElements.scala index 52ac86b10a6..5c0c01859c2 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/AstToNewExpressionElements.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/AstToNewExpressionElements.scala @@ -3,7 +3,7 @@ package wdl.transforms.biscayne.ast2wdlom import cats.syntax.validated._ import common.validation.ErrorOr.ErrorOr import wdl.model.draft3.elements.ExpressionElement -import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep} +import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep, Suffix} import wdl.transforms.base.ast2wdlom.AstNodeToExpressionElement object AstToNewExpressionElements { @@ -15,6 +15,7 @@ object AstToNewExpressionElements { "min" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Min, "min"), "max" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Max, "max"), "sep" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Sep, "sep"), + "suffix" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Suffix, "suffix"), "read_object" -> (_ => "read_object is no longer available in this WDL version. Consider using read_json instead".invalidNel ), diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumers.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumers.scala index ad168067177..3c73f8b55cd 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumers.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumers.scala @@ -58,6 +58,14 @@ object BiscayneExpressionValueConsumers { expressionValueConsumer.expressionConsumedValueHooks(a.arg2)(expressionValueConsumer) } + implicit val suffixExpressionValueConsumer: ExpressionValueConsumer[Suffix] = new ExpressionValueConsumer[Suffix] { + override def expressionConsumedValueHooks(a: Suffix)(implicit + expressionValueConsumer: ExpressionValueConsumer[ExpressionElement] + ): Set[UnlinkedConsumedValueHook] = + expressionValueConsumer.expressionConsumedValueHooks(a.arg1)(expressionValueConsumer) ++ + expressionValueConsumer.expressionConsumedValueHooks(a.arg2)(expressionValueConsumer) + } + implicit val noneLiteralExpressionValueConsumer: ExpressionValueConsumer[NoneLiteralElement.type] = new ExpressionValueConsumer[NoneLiteralElement.type] { override def expressionConsumedValueHooks(a: NoneLiteralElement.type)(implicit diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/consumed.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/consumed.scala index f70c3603b4a..e1121e1d549 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/consumed.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/consumed.scala @@ -81,6 +81,7 @@ package object consumed { case a: Length => a.expressionConsumedValueHooks(expressionValueConsumer) case a: Flatten => a.expressionConsumedValueHooks(expressionValueConsumer) case a: Prefix => a.expressionConsumedValueHooks(expressionValueConsumer) + case a: Suffix => a.expressionConsumedValueHooks(expressionValueConsumer) case a: SelectFirst => a.expressionConsumedValueHooks(expressionValueConsumer) case a: SelectAll => a.expressionConsumedValueHooks(expressionValueConsumer) case a: Defined => a.expressionConsumedValueHooks(expressionValueConsumer) diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluators.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluators.scala index fa968525bbc..6a79d2b3948 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluators.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluators.scala @@ -1,6 +1,6 @@ package wdl.transforms.biscayne.linking.expression.files -import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep} +import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep, Suffix} import wdl.model.draft3.graph.expression.FileEvaluator import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators.twoParameterFunctionPassthroughFileEvaluator @@ -16,6 +16,7 @@ object BiscayneFileEvaluators { EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator implicit val sepFunctionEvaluator: FileEvaluator[Sep] = twoParameterFunctionPassthroughFileEvaluator[Sep] + implicit val suffixFunctionEvaluator: FileEvaluator[Suffix] = twoParameterFunctionPassthroughFileEvaluator[Suffix] implicit val minFunctionEvaluator: FileEvaluator[Min] = twoParameterFunctionPassthroughFileEvaluator[Min] implicit val maxFunctionEvaluator: FileEvaluator[Max] = twoParameterFunctionPassthroughFileEvaluator[Max] diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/files.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/files.scala index d143e948272..9dd95e9a0ec 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/files.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/files.scala @@ -135,6 +135,7 @@ package object files { case a: Flatten => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) case a: Prefix => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) + case a: Suffix => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) case a: SelectFirst => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) case a: SelectAll => diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluators.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluators.scala index 9a993f6b5b8..1ff5bf71ba4 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluators.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluators.scala @@ -1,5 +1,6 @@ package wdl.transforms.biscayne.linking.expression.types +import cats.implicits.catsSyntaxTuple2Semigroupal import cats.syntax.validated._ import common.validation.ErrorOr._ import wdl.model.draft3.elements.ExpressionElement @@ -100,4 +101,13 @@ object BiscayneTypeEvaluators { } } + + implicit val suffixFunctionEvaluator: TypeEvaluator[Suffix] = new TypeEvaluator[Suffix] { + override def evaluateType(a: Suffix, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit + expressionTypeEvaluator: TypeEvaluator[ExpressionElement] + ): ErrorOr[WomType] = + (validateParamType(a.suffix, linkedValues, WomStringType), + validateParamType(a.array, linkedValues, WomArrayType(WomStringType)) + ) mapN { (_, _) => WomArrayType(WomStringType) } + } } diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/types.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/types.scala index 0b61f7fd71c..dec2deeb24f 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/types.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/types.scala @@ -83,6 +83,7 @@ package object types { case a: Length => a.evaluateType(linkedValues)(typeEvaluator) case a: Flatten => a.evaluateType(linkedValues)(typeEvaluator) case a: Prefix => a.evaluateType(linkedValues)(typeEvaluator) + case a: Suffix => a.evaluateType(linkedValues)(typeEvaluator) case a: SelectFirst => a.evaluateType(linkedValues)(typeEvaluator) case a: SelectAll => a.evaluateType(linkedValues)(typeEvaluator) case a: Defined => a.evaluateType(linkedValues)(typeEvaluator) diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluators.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluators.scala index bdb3516f787..0c4805f0e68 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluators.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluators.scala @@ -217,4 +217,22 @@ object BiscayneValueEvaluators { EvaluatedValue(WomString(arr.value.map(v => v.valueString).mkString(sepvalue.value)), Seq.empty).validNel } } + + implicit val suffixFunctionEvaluator: ValueEvaluator[Suffix] = new ValueEvaluator[Suffix] { + override def evaluateValue(a: Suffix, + inputs: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + forCommandInstantiationOptions: Option[ForCommandInstantiationOptions] + )(implicit expressionValueEvaluator: ValueEvaluator[ExpressionElement]): ErrorOr[EvaluatedValue[WomArray]] = + processTwoValidatedValues[WomString, WomArray, WomArray]( + expressionValueEvaluator.evaluateValue(a.arg1, inputs, ioFunctionSet, forCommandInstantiationOptions)( + expressionValueEvaluator + ), + expressionValueEvaluator.evaluateValue(a.arg2, inputs, ioFunctionSet, forCommandInstantiationOptions)( + expressionValueEvaluator + ) + ) { (suffix, arr) => + EvaluatedValue(WomArray(arr.value.map(v => WomString(v.valueString + suffix.value))), Seq.empty).validNel + } + } } diff --git a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/values.scala b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/values.scala index 338c8676906..3106ceba428 100644 --- a/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/values.scala +++ b/wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/values.scala @@ -142,6 +142,8 @@ package object values { a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) case a: Prefix => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) + case a: Suffix => + a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) case a: SelectFirst => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) case a: SelectAll => diff --git a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/Ast2WdlomSpec.scala b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/Ast2WdlomSpec.scala index 4fde60b1efd..a0953cd8397 100644 --- a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/Ast2WdlomSpec.scala +++ b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/Ast2WdlomSpec.scala @@ -97,4 +97,10 @@ class Ast2WdlomSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { val expr = fromString[ExpressionElement](str, parser.parse_e) expr shouldBeValid NoneLiteralElement } + + it should "parse the new suffix function" in { + val str = "suffix(some_str, some_arr)" + val expr = fromString[ExpressionElement](str, parser.parse_e) + expr shouldBeValid (Suffix(IdentifierLookup("some_str"), IdentifierLookup("some_arr"))) + } } diff --git a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumersSpec.scala b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumersSpec.scala index 11ca72b3a49..76f4a0338e7 100644 --- a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumersSpec.scala +++ b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumersSpec.scala @@ -57,4 +57,22 @@ class BiscayneExpressionValueConsumersSpec extends AnyFlatSpec with CromwellTime e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_separator"), UnlinkedIdentifierHook("c"))) } } + + it should "discover the variable lookups within a suffix() call" in { + val str = """ suffix(my_suffix, ["a", "b", c]) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_suffix"), UnlinkedIdentifierHook("c"))) + } + } + + it should "discover an array variable lookup within a suffix() call" in { + val str = """ suffix("SFX", my_array) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_array"))) + } + } } diff --git a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluatorSpec.scala b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluatorSpec.scala index 59ebfd18dc4..4a3ed6b63b9 100644 --- a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluatorSpec.scala +++ b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluatorSpec.scala @@ -45,4 +45,15 @@ class BiscayneFileEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit ) } } + + it should "discover the file which would be required to evaluate a suffix() function" in { + val str = """ suffix(' # what a line', read_lines("foo.txt")) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.predictFilesNeededToEvaluate(Map.empty, NoIoFunctionSet, WomStringType) shouldBeValid Set( + WomSingleFile("foo.txt") + ) + } + } } diff --git a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluatorSpec.scala b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluatorSpec.scala index 0593605bb2b..c8372d36cd1 100644 --- a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluatorSpec.scala +++ b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluatorSpec.scala @@ -56,4 +56,12 @@ class BiscayneTypeEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit } } + it should "evaluate the type of a suffix() function as Array[String]" in { + val str = """ suffix('S', ["a", "b", "c"]) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomStringType) + } + } } diff --git a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluatorSpec.scala b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluatorSpec.scala index 53660a407d9..1487543a838 100644 --- a/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluatorSpec.scala +++ b/wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluatorSpec.scala @@ -200,4 +200,21 @@ class BiscayneValueEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wi e.evaluateValue(Map.empty, NoIoFunctionSet, None) shouldBeValid EvaluatedValue(expectedString, Seq.empty) } } + + it should "evaluate a suffix expression correctly" in { + val str = """ suffix("S", ["a", "b", "c"]) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + val expectedArray: WomArray = WomArray( + Seq( + WomString("aS"), + WomString("bS"), + WomString("cS") + ) + ) + + expr.shouldBeValidPF { case e => + e.evaluateValue(Map.empty, NoIoFunctionSet, None) shouldBeValid EvaluatedValue(expectedArray, Seq.empty) + } + } } diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/ast2wdlom/AstToNewExpressionElements.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/ast2wdlom/AstToNewExpressionElements.scala index ba331383e45..46cd8cb2e60 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/ast2wdlom/AstToNewExpressionElements.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/ast2wdlom/AstToNewExpressionElements.scala @@ -3,7 +3,7 @@ package wdl.transforms.cascades.ast2wdlom import cats.syntax.validated._ import common.validation.ErrorOr.ErrorOr import wdl.model.draft3.elements.ExpressionElement -import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep} +import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep, Suffix} import wdl.transforms.base.ast2wdlom.AstNodeToExpressionElement object AstToNewExpressionElements { @@ -15,6 +15,7 @@ object AstToNewExpressionElements { "min" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Min, "min"), "max" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Max, "max"), "sep" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Sep, "sep"), + "suffix" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Suffix, "suffix"), "read_object" -> (_ => "read_object is no longer available in this WDL version. Consider using read_json instead".invalidNel ), diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumers.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumers.scala index b4e6c5be1d6..ebc6d48fba1 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumers.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumers.scala @@ -58,6 +58,14 @@ object cascadesExpressionValueConsumers { expressionValueConsumer.expressionConsumedValueHooks(a.arg2)(expressionValueConsumer) } + implicit val suffixExpressionValueConsumer: ExpressionValueConsumer[Suffix] = new ExpressionValueConsumer[Suffix] { + override def expressionConsumedValueHooks(a: Suffix)(implicit + expressionValueConsumer: ExpressionValueConsumer[ExpressionElement] + ): Set[UnlinkedConsumedValueHook] = + expressionValueConsumer.expressionConsumedValueHooks(a.arg1)(expressionValueConsumer) ++ + expressionValueConsumer.expressionConsumedValueHooks(a.arg2)(expressionValueConsumer) + } + implicit val noneLiteralExpressionValueConsumer: ExpressionValueConsumer[NoneLiteralElement.type] = new ExpressionValueConsumer[NoneLiteralElement.type] { override def expressionConsumedValueHooks(a: NoneLiteralElement.type)(implicit diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/consumed.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/consumed.scala index af54bbbd6a6..2026aaec010 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/consumed.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/consumed/consumed.scala @@ -81,6 +81,7 @@ package object consumed { case a: Length => a.expressionConsumedValueHooks(expressionValueConsumer) case a: Flatten => a.expressionConsumedValueHooks(expressionValueConsumer) case a: Prefix => a.expressionConsumedValueHooks(expressionValueConsumer) + case a: Suffix => a.expressionConsumedValueHooks(expressionValueConsumer) case a: SelectFirst => a.expressionConsumedValueHooks(expressionValueConsumer) case a: SelectAll => a.expressionConsumedValueHooks(expressionValueConsumer) case a: Defined => a.expressionConsumedValueHooks(expressionValueConsumer) diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluators.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluators.scala index 3b7b299a090..32ea447b584 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluators.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluators.scala @@ -1,6 +1,6 @@ package wdl.transforms.cascades.linking.expression.files -import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep} +import wdl.model.draft3.elements.ExpressionElement.{AsMap, AsPairs, CollectByKey, Keys, Max, Min, Sep, Suffix} import wdl.model.draft3.graph.expression.FileEvaluator import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators.twoParameterFunctionPassthroughFileEvaluator @@ -16,6 +16,7 @@ object cascadesFileEvaluators { EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator implicit val sepFunctionEvaluator: FileEvaluator[Sep] = twoParameterFunctionPassthroughFileEvaluator[Sep] + implicit val suffixFunctionEvaluator: FileEvaluator[Suffix] = twoParameterFunctionPassthroughFileEvaluator[Suffix] implicit val minFunctionEvaluator: FileEvaluator[Min] = twoParameterFunctionPassthroughFileEvaluator[Min] implicit val maxFunctionEvaluator: FileEvaluator[Max] = twoParameterFunctionPassthroughFileEvaluator[Max] diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/files.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/files.scala index 9693cbfaba6..a53c85b9b89 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/files.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/files/files.scala @@ -135,6 +135,7 @@ package object files { case a: Flatten => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) case a: Prefix => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) + case a: Suffix => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) case a: SelectFirst => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator) case a: SelectAll => diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluators.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluators.scala index e169a561d53..180af9eb0d8 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluators.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluators.scala @@ -1,5 +1,6 @@ package wdl.transforms.biscayne.linking.expression.types +import cats.implicits.catsSyntaxTuple2Semigroupal import cats.syntax.validated._ import common.validation.ErrorOr._ import wdl.model.draft3.elements.ExpressionElement @@ -100,4 +101,13 @@ object cascadesTypeEvaluators { } } + + implicit val suffixFunctionEvaluator: TypeEvaluator[Suffix] = new TypeEvaluator[Suffix] { + override def evaluateType(a: Suffix, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit + expressionTypeEvaluator: TypeEvaluator[ExpressionElement] + ): ErrorOr[WomType] = + (validateParamType(a.suffix, linkedValues, WomStringType), + validateParamType(a.array, linkedValues, WomArrayType(WomStringType)) + ) mapN { (_, _) => WomArrayType(WomStringType) } + } } diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/types.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/types.scala index 70e6e61a180..4dd7c528e8b 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/types.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/types/types.scala @@ -83,6 +83,7 @@ package object types { case a: Length => a.evaluateType(linkedValues)(typeEvaluator) case a: Flatten => a.evaluateType(linkedValues)(typeEvaluator) case a: Prefix => a.evaluateType(linkedValues)(typeEvaluator) + case a: Suffix => a.evaluateType(linkedValues)(typeEvaluator) case a: SelectFirst => a.evaluateType(linkedValues)(typeEvaluator) case a: SelectAll => a.evaluateType(linkedValues)(typeEvaluator) case a: Defined => a.evaluateType(linkedValues)(typeEvaluator) diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluators.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluators.scala index 788f114e181..db9df85bf06 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluators.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluators.scala @@ -217,4 +217,22 @@ object cascadesValueEvaluators { EvaluatedValue(WomString(arr.value.map(v => v.valueString).mkString(sepvalue.value)), Seq.empty).validNel } } + + implicit val suffixFunctionEvaluator: ValueEvaluator[Suffix] = new ValueEvaluator[Suffix] { + override def evaluateValue(a: Suffix, + inputs: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + forCommandInstantiationOptions: Option[ForCommandInstantiationOptions] + )(implicit expressionValueEvaluator: ValueEvaluator[ExpressionElement]): ErrorOr[EvaluatedValue[WomArray]] = + processTwoValidatedValues[WomString, WomArray, WomArray]( + expressionValueEvaluator.evaluateValue(a.arg1, inputs, ioFunctionSet, forCommandInstantiationOptions)( + expressionValueEvaluator + ), + expressionValueEvaluator.evaluateValue(a.arg2, inputs, ioFunctionSet, forCommandInstantiationOptions)( + expressionValueEvaluator + ) + ) { (suffix, arr) => + EvaluatedValue(WomArray(arr.value.map(v => WomString(v.valueString + suffix.value))), Seq.empty).validNel + } + } } diff --git a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/values.scala b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/values.scala index 08eb74ac19c..8850d4371df 100644 --- a/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/values.scala +++ b/wdl/transforms/cascades/src/main/scala/wdl/transforms/cascades/linking/expression/values/values.scala @@ -142,6 +142,8 @@ package object values { a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) case a: Prefix => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) + case a: Suffix => + a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) case a: SelectFirst => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator) case a: SelectAll => diff --git a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/Ast2WdlomSpec.scala b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/Ast2WdlomSpec.scala index a3a9cd4fb65..88b7aff6578 100644 --- a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/Ast2WdlomSpec.scala +++ b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/Ast2WdlomSpec.scala @@ -97,4 +97,10 @@ class Ast2WdlomSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { val expr = fromString[ExpressionElement](str, parser.parse_e) expr shouldBeValid NoneLiteralElement } + + it should "parse the new suffix function" in { + val str = "suffix(some_str, some_arr)" + val expr = fromString[ExpressionElement](str, parser.parse_e) + expr shouldBeValid (Suffix(IdentifierLookup("some_str"), IdentifierLookup("some_arr"))) + } } diff --git a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumersSpec.scala b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumersSpec.scala index 65a03812c25..57beb7a12a1 100644 --- a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumersSpec.scala +++ b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/consumed/CascadesExpressionValueConsumersSpec.scala @@ -57,4 +57,22 @@ class CascadesExpressionValueConsumersSpec extends AnyFlatSpec with CromwellTime e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_separator"), UnlinkedIdentifierHook("c"))) } } + + it should "discover the variable lookups within a suffix() call" in { + val str = """ suffix(my_suffix, ["a", "b", c]) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_suffix"), UnlinkedIdentifierHook("c"))) + } + } + + it should "discover an array variable lookup within a suffix() call" in { + val str = """ suffix("SFX", my_array) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_array"))) + } + } } diff --git a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluatorSpec.scala b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluatorSpec.scala index 6e0a828855e..5335f7d5f29 100644 --- a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluatorSpec.scala +++ b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/files/CascadesFileEvaluatorSpec.scala @@ -45,4 +45,15 @@ class CascadesFileEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit ) } } + + it should "discover the file which would be required to evaluate a suffix() function" in { + val str = """ suffix(' # what a line', read_lines("foo.txt")) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.predictFilesNeededToEvaluate(Map.empty, NoIoFunctionSet, WomStringType) shouldBeValid Set( + WomSingleFile("foo.txt") + ) + } + } } diff --git a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluatorSpec.scala b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluatorSpec.scala index 96c3c51e937..954ef0f52f0 100644 --- a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluatorSpec.scala +++ b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/types/CascadesTypeEvaluatorSpec.scala @@ -56,4 +56,12 @@ class CascadesTypeEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit } } + it should "evaluate the type of a suffix() function as Array[String]" in { + val str = """ suffix('S', ["a", "b", "c"]) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + expr.shouldBeValidPF { case e => + e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomStringType) + } + } } diff --git a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluatorSpec.scala b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluatorSpec.scala index 6dbad32d7d0..bd47a20ca68 100644 --- a/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluatorSpec.scala +++ b/wdl/transforms/cascades/src/test/scala/wdl/transforms/cascades/linking/expression/values/CascadesValueEvaluatorSpec.scala @@ -200,4 +200,21 @@ class CascadesValueEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wi e.evaluateValue(Map.empty, NoIoFunctionSet, None) shouldBeValid EvaluatedValue(expectedString, Seq.empty) } } + + it should "evaluate a suffix expression correctly" in { + val str = """ suffix("S", ["a", "b", "c"]) """ + val expr = fromString[ExpressionElement](str, parser.parse_e) + + val expectedArray: WomArray = WomArray( + Seq( + WomString("aS"), + WomString("bS"), + WomString("cS") + ) + ) + + expr.shouldBeValidPF { case e => + e.evaluateValue(Map.empty, NoIoFunctionSet, None) shouldBeValid EvaluatedValue(expectedArray, Seq.empty) + } + } } diff --git a/wdl/transforms/draft3/src/test/scala/wdl/draft3/transforms/ast2wdlom/Ast2WdlomSpec.scala b/wdl/transforms/draft3/src/test/scala/wdl/draft3/transforms/ast2wdlom/Ast2WdlomSpec.scala index 7765cac73bc..664696ee86d 100644 --- a/wdl/transforms/draft3/src/test/scala/wdl/draft3/transforms/ast2wdlom/Ast2WdlomSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/wdl/draft3/transforms/ast2wdlom/Ast2WdlomSpec.scala @@ -52,6 +52,12 @@ class Ast2WdlomSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { expr shouldBeInvalid "Failed to parse expression (reason 1 of 1): Unknown engine function: 'collect_by_key'" } + it should "not parse the new suffix function" in { + val str = "suffix(aStr, anArray)" + val expr = fromString[ExpressionElement](str, parser.parse_e) + expr shouldBeInvalid "Failed to parse expression (reason 1 of 1): Unknown engine function: 'suffix'" + } + it should "parse the (biscayne) None keyword as a plain old identifier" in { val str = "None" val expr = fromString[ExpressionElement](str, parser.parse_e) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wdl/WdlWriterImpl.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wdl/WdlWriterImpl.scala index 70cc56b5a65..a85f3c34420 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wdl/WdlWriterImpl.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/wdlom2wdl/WdlWriterImpl.scala @@ -465,6 +465,7 @@ object WdlWriterImpl { case _: Zip => functionCall("zip") case _: Cross => functionCall("cross") case _: Prefix => functionCall("prefix") + case _: Suffix => functionCall("suffix") case _: Min => functionCall("min") case _: Max => functionCall("max") case _: Sep => functionCall("sep")