feat(core): YAML input

Fixes kestra-io#4126
yuri1969 · Sep 2, 2024 · aa0c12b · aa0c12b
1 parent 749a3aa
commit aa0c12b
Show file tree

Hide file tree

Showing 11 changed files with 89 additions and 14 deletions.
diff --git a/core/src/main/java/io/kestra/core/models/flows/Input.java b/core/src/main/java/io/kestra/core/models/flows/Input.java
@@ -37,7 +37,8 @@
     @JsonSubTypes.Type(value = SelectInput.class, name = "SELECT"),
     @JsonSubTypes.Type(value = TimeInput.class, name = "TIME"),
     @JsonSubTypes.Type(value = URIInput.class, name = "URI"),
-    @JsonSubTypes.Type(value = MultiselectInput.class, name = "MULTISELECT")
+    @JsonSubTypes.Type(value = MultiselectInput.class, name = "MULTISELECT"),
+    @JsonSubTypes.Type(value = YamlInput.class, name = "YAML")
 })
 @JsonInclude(JsonInclude.Include.NON_DEFAULT)
 public abstract class Input<T> implements Data {

diff --git a/core/src/main/java/io/kestra/core/models/flows/Type.java b/core/src/main/java/io/kestra/core/models/flows/Type.java
@@ -23,7 +23,8 @@ public enum Type {
     URI(URIInput.class.getName()),
     SECRET(SecretInput.class.getName()),
     ARRAY(ArrayInput.class.getName()),
-    MULTISELECT(MultiselectInput.class.getName());
+    MULTISELECT(MultiselectInput.class.getName()),
+    YAML(YamlInput.class.getName());
 
     private final String clsName;
 

diff --git a/core/src/main/java/io/kestra/core/models/flows/input/YamlInput.java b/core/src/main/java/io/kestra/core/models/flows/input/YamlInput.java
@@ -0,0 +1,17 @@
+package io.kestra.core.models.flows.input;
+
+import io.kestra.core.models.flows.Input;
+import jakarta.validation.ConstraintViolationException;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+@SuperBuilder
+@Getter
+@NoArgsConstructor
+public class YamlInput  extends Input<Object> {
+    @Override
+    public void validate(Object input) throws ConstraintViolationException {
+        // no validation yet
+    }
+}
diff --git a/core/src/main/java/io/kestra/core/runners/FlowInputOutput.java b/core/src/main/java/io/kestra/core/runners/FlowInputOutput.java
@@ -1,5 +1,6 @@
 package io.kestra.core.runners;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
 import io.kestra.core.encryption.EncryptionService;
 import io.kestra.core.models.executions.Execution;
@@ -44,6 +45,7 @@
 @Singleton
 public class FlowInputOutput {
     public static final Pattern URI_PATTERN = Pattern.compile("^[a-z]+:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$");
+    public static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml();
 
     private final StorageInterface storageInterface;
     private final String secretKey;
@@ -281,6 +283,7 @@ private Object parseType(Execution execution, Type type, String id, Type element
                     }
                 }
                 case JSON -> JacksonMapper.toObject(((String) current));
+                case YAML -> YAML_MAPPER.readValue((String) current, JacksonMapper.OBJECT_TYPE_REFERENCE);
                 case URI -> {
                     Matcher matcher = URI_PATTERN.matcher((String) current);
                     if (matcher.matches()) {

diff --git a/core/src/test/java/io/kestra/core/runners/InputsTest.java b/core/src/test/java/io/kestra/core/runners/InputsTest.java
@@ -56,6 +56,11 @@ public class InputsTest extends AbstractMemoryRunnerTest {
         .put("validatedTime", "11:27:49")
         .put("secret", "secret")
         .put("array", "[1, 2, 3]")
+        .put("yaml", """
+            some: property
+            alist:
+            - of
+            - values""")
         .build();
 
     @Inject
@@ -135,6 +140,9 @@ void allValidInputs() throws URISyntaxException, IOException {
         assertThat(typeds.get("array"), instanceOf(List.class));
         assertThat((List<String>)typeds.get("array"), hasSize(3));
         assertThat((List<String>)typeds.get("array"), contains(1, 2, 3));
+        assertThat(typeds.get("yaml"), is(Map.of(
+            "some", "property",
+            "alist", List.of("of", "values"))));
     }
 
     @Test
@@ -161,7 +169,7 @@ void inputFlow() throws TimeoutException {
             (flow, execution1) -> flowIO.typedInputs(flow, execution1, inputs)
         );
 
-        assertThat(execution.getTaskRunList(), hasSize(13));
+        assertThat(execution.getTaskRunList(), hasSize(14));
         assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS));
         assertThat(
             (String) execution.findTaskRunsByTaskId("file").getFirst().getOutputs().get("value"),
@@ -333,7 +341,7 @@ void inputEmptyJsonFlow() throws TimeoutException {
             (flow, execution1) -> flowIO.typedInputs(flow, execution1, map)
         );
 
-        assertThat(execution.getTaskRunList(), hasSize(13));
+        assertThat(execution.getTaskRunList(), hasSize(14));
         assertThat(execution.getState().getCurrent(), is(State.Type.SUCCESS));
 
         assertThat(execution.getInputs().get("json"), instanceOf(Map.class));

diff --git a/core/src/test/java/io/kestra/core/serializers/YamlFlowParserTest.java b/core/src/test/java/io/kestra/core/serializers/YamlFlowParserTest.java
@@ -123,10 +123,10 @@ void inputsFailed() {
     void inputs() {
         Flow flow = this.parse("flows/valids/inputs.yaml");
 
-        assertThat(flow.getInputs().size(), is(27));
-        assertThat(flow.getInputs().stream().filter(Input::getRequired).count(), is(9L));
+        assertThat(flow.getInputs().size(), is(28));
+        assertThat(flow.getInputs().stream().filter(Input::getRequired).count(), is(10L));
         assertThat(flow.getInputs().stream().filter(r -> !r.getRequired()).count(), is(18L));
-        assertThat(flow.getInputs().stream().filter(r -> r.getDefaults() != null).count(), is(1L));
+        assertThat(flow.getInputs().stream().filter(r -> r.getDefaults() != null).count(), is(2L));
         assertThat(flow.getInputs().stream().filter(r -> r instanceof StringInput && ((StringInput)r).getValidator() != null).count(), is(1L));
     }
 

diff --git a/core/src/test/resources/flows/valids/inputs.yaml b/core/src/test/resources/flows/valids/inputs.yaml
@@ -95,6 +95,15 @@ inputs:
 - name: array
   type: ARRAY
   itemType: INT
+- name: yaml
+  type: YAML
+  defaults:
+    property: something
+    list:
+    - key: key1
+      value: value1
+    - key: key2
+      value: value2
 
 tasks:
 - id: string
@@ -131,4 +140,7 @@ tasks:
   format: "{{inputs.json}}"
 - id: jsonOutput
   type: io.kestra.plugin.core.debug.Return
-  format: "{{outputs.json.value}}"
+  format: "{{outputs.json.value}}"
+- id: yamlOutput
+  type: io.kestra.plugin.core.debug.Return
+  format: "{{inputs.yaml}}"
diff --git a/ui/src/components/inputs/InputsForm.vue b/ui/src/components/inputs/InputsForm.vue
@@ -129,6 +129,15 @@
                 lang="json"
                 v-model="inputs[input.id]"
             />
+            <editor
+                :full-height="false"
+                :input="true"
+                :navbar="false"
+                v-if="input.type === 'YAML'"
+                lang="yaml"
+                :model-value="inputs[input.id]"
+                @change="onYamlChange(input, $event)"
+            />
 
             <markdown v-if="input.description" class="markdown-tooltip text-muted" :source="input.description" font-size-var="font-size-xs" />
         </el-form-item>
@@ -142,8 +151,14 @@
     import Editor from "../../components/inputs/Editor.vue";
     import Markdown from "../layout/Markdown.vue";
     import Inputs from "../../utils/inputs";
+    import YamlUtils from "../../utils/yamlUtils.js";
 
     export default {
+        computed: {
+            YamlUtils() {
+                return YamlUtils
+            }
+        },
         components: {Editor, Markdown},
         props: {
             modelValue: {
@@ -214,6 +229,10 @@
                 this.inputs[input.id] = e.target.files[0];
                 this.onChange();
             },
+            onYamlChange(input, e) {
+                this.inputs[input.id] = e.target.value;
+                this.onChange();
+            },
             numberHint(input){
                 const {min, max} = input;
 

diff --git a/ui/src/utils/inputs.js b/ui/src/utils/inputs.js
@@ -1,4 +1,5 @@
 import moment from "moment/moment";
+import YamlUtils from "./yamlUtils.js";
 
 export default class Inputs {
     static normalize(type, value) {
@@ -11,7 +12,13 @@ export default class Inputs {
         } else if (type === "DURATION" || type === "TIME") {
             res = moment().startOf("day").add(res, "seconds").toString()
         } else if (type === "ARRAY" || type === "MULTISELECT" || type === "JSON") {
-            if(typeof res !== "string") res = JSON.stringify(res).toString();
+            if(typeof res !== "string") {
+                res = JSON.stringify(res).toString();
+            }
+        } else if (type === "YAML") {
+            if(typeof res !== "string") {
+                res = YamlUtils.stringify(res).toString();
+            }
         } else if (type === "BOOLEAN" && type === undefined){
             res = "undefined";
         } else if (type === "STRING" && Array.isArray(res)){

diff --git a/webserver/src/test/java/io/kestra/webserver/controllers/api/ExecutionControllerTest.java b/webserver/src/test/java/io/kestra/webserver/controllers/api/ExecutionControllerTest.java
@@ -110,6 +110,11 @@ class ExecutionControllerTest extends JdbcH2ControllerTest {
         .put("secret", "secret")
         .put("array", "[1, 2, 3]")
         .put("json", "{}")
+        .put("yaml", """
+            some: property
+            alist:
+            - of
+            - values""")
         .build();
 
     @Test
@@ -152,6 +157,7 @@ private MultipartBody createInputsFlowBody() {
             .addPart("secret", "secret")
             .addPart("array", "[1, 2, 3]")
             .addPart("json", "{}")
+            .addPart("yaml", "{}")
             .build();
     }
 
@@ -212,7 +218,7 @@ void triggerAndWait() {
         Execution result = triggerInputsFlowExecution(true);
 
         assertThat(result.getState().getCurrent(), is(State.Type.SUCCESS));
-        assertThat(result.getTaskRunList().size(), is(13));
+        assertThat(result.getTaskRunList().size(), is(14));
     }
 
     @Test
@@ -571,7 +577,7 @@ void restartFromLastFailedWithPause() throws TimeoutException {
     @Test
     void downloadFile() throws TimeoutException {
         Execution execution = runnerUtils.runOne(null, TESTS_FLOW_NS, "inputs", null, (flow, execution1) -> flowIO.typedInputs(flow, execution1, inputs));
-        assertThat(execution.getTaskRunList(), hasSize(13));
+        assertThat(execution.getTaskRunList(), hasSize(14));
 
         String path = (String) execution.getInputs().get("file");
 
@@ -608,7 +614,7 @@ void downloadFile() throws TimeoutException {
     @Test
     void filePreview() throws TimeoutException {
         Execution defaultExecution = runnerUtils.runOne(null, TESTS_FLOW_NS, "inputs", null, (flow, execution1) -> flowIO.typedInputs(flow, execution1, inputs));
-        assertThat(defaultExecution.getTaskRunList(), hasSize(13));
+        assertThat(defaultExecution.getTaskRunList(), hasSize(14));
 
         String defaultPath = (String) defaultExecution.getInputs().get("file");
 
@@ -630,10 +636,11 @@ void filePreview() throws TimeoutException {
             .put("secret", "secret")
             .put("array", "[1, 2, 3]")
             .put("json", "{}")
+            .put("yaml", "{}")
             .build();
 
         Execution latin1Execution = runnerUtils.runOne(null, TESTS_FLOW_NS, "inputs", null, (flow, execution1) -> flowIO.typedInputs(flow, execution1, latin1FileInputs));
-        assertThat(latin1Execution.getTaskRunList(), hasSize(13));
+        assertThat(latin1Execution.getTaskRunList(), hasSize(14));
 
         String latin1Path = (String) latin1Execution.getInputs().get("file");
 

diff --git a/webserver/src/test/java/io/kestra/webserver/controllers/api/PluginControllerTest.java b/webserver/src/test/java/io/kestra/webserver/controllers/api/PluginControllerTest.java
@@ -213,7 +213,7 @@ void inputs() throws URISyntaxException {
                 Argument.listOf(InputType.class)
             );
 
-            assertThat(doc.size(), is(16));
+            assertThat(doc.size(), is(17));
         });
     }