feat(ui): introduce autocomplete on the editor (#913)

Co-authored-by: Ludovic DEHON <tchiot.ludo@gmail.com>

cli/src/main/resources/application.yml

@@ -22,6 +22,9 @@ micronaut:
     idle-timeout: 60m
     netty:
       max-chunk-size: 10MB
+  caches:
+    default:
+      maximum-weight: 10485760
 
 jackson:
   serialization:

cli/src/test/java/io/kestra/cli/commands/plugins/PluginListCommandTest.java

@@ -36,8 +36,8 @@ void run() throws IOException, URISyntaxException {
 
         FileUtils.copyFile(
             new File(Objects.requireNonNull(PluginListCommandTest.class.getClassLoader()
-                .getResource("plugins/plugin-template-test-0.2.0-SNAPSHOT.jar")).toURI()),
-            new File(URI.create("file://" + pluginsPath.toAbsolutePath() + "/plugin-template-test-0.2.0-SNAPSHOT.jar"))
+                .getResource("plugins/plugin-template-test-0.6.0-SNAPSHOT.jar")).toURI()),
+            new File(URI.create("file://" + pluginsPath.toAbsolutePath() + "/plugin-template-test-0.6.0-SNAPSHOT.jar"))
         );
 
         ByteArrayOutputStream out = new ByteArrayOutputStream();

core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java

@@ -1,7 +1,9 @@
 package io.kestra.core.docs;
 
 import com.fasterxml.classmate.ResolvedType;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
 import com.github.victools.jsonschema.generator.*;
@@ -14,23 +16,113 @@
 import com.google.common.collect.ImmutableMap;
 import io.kestra.core.models.annotations.Plugin;
 import io.kestra.core.models.annotations.PluginProperty;
+import io.kestra.core.models.conditions.Condition;
+import io.kestra.core.models.conditions.ScheduleCondition;
+import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.tasks.Output;
 import io.kestra.core.models.tasks.Task;
+import io.kestra.core.models.triggers.AbstractTrigger;
 import io.kestra.core.serializers.JacksonMapper;
+import io.kestra.core.services.PluginService;
 import io.micronaut.core.annotation.Nullable;
 import io.swagger.v3.oas.annotations.media.Schema;
 
 import java.lang.reflect.*;
 import java.time.Duration;
 import java.util.*;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 
 @Singleton
 public class JsonSchemaGenerator {
+    @Inject
+    private PluginService pluginService;
+
     Map<Class<?>, Object> defaultInstances = new HashMap<>();
 
+    public <T> Map<String, Object> schemas(Class<? extends T> cls) {
+        SchemaGeneratorConfigBuilder builder = new SchemaGeneratorConfigBuilder(
+            SchemaVersion.DRAFT_7,
+            OptionPreset.PLAIN_JSON
+        );
+
+        this.build(builder, cls);
+
+        SchemaGeneratorConfig schemaGeneratorConfig = builder.build();
+
+        SchemaGenerator generator = new SchemaGenerator(schemaGeneratorConfig);
+        try {
+            ObjectNode objectNode = generator.generateSchema(cls);
+
+            Map<String, Object> map = JacksonMapper.toMap(objectNode);
+
+            // hack
+            if (cls == Task.class) {
+                fixTask(map);
+            } else if (cls == Flow.class) {
+                fixFlow(map);
+            }
+
+
+            return map;
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Unable to generate jsonschema for '" + cls.getName() + "'", e);
+        }
+    }
+
+    private void mutateDescription(ObjectNode collectedTypeAttributes) {
+        if (collectedTypeAttributes.has("description")) {
+            collectedTypeAttributes.set("markdownDescription", collectedTypeAttributes.get("description"));
+            collectedTypeAttributes.remove("description");
+        }
+
+        if (collectedTypeAttributes.has("description")) {
+            collectedTypeAttributes.set("markdownDescription", collectedTypeAttributes.get("description"));
+            collectedTypeAttributes.remove("description");
+        }
+
+        if (collectedTypeAttributes.has("default")) {
+            StringBuilder sb = new StringBuilder();
+            if (collectedTypeAttributes.has("markdownDescription")) {
+                sb.append(collectedTypeAttributes.get("markdownDescription").asText());
+                sb.append("\n\n");
+            }
+
+            try {
+                sb.append("Default value is : `")
+                    .append(JacksonMapper.ofYaml().writeValueAsString(collectedTypeAttributes.get("default")).trim())
+                    .append("`");
+            } catch (JsonProcessingException ignored) {
+
+            }
+
+            collectedTypeAttributes.set("markdownDescription", new TextNode(sb.toString()));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void fixTask(Map<String, Object> map) {
+        var definitions = (Map<String, Map<String, Object>>) map.get("definitions");
+        var task = (Map<String, Object>) definitions.get("io.kestra.core.models.tasks.Task-2");
+        var allOf = (List<Object>) task.get("allOf");
+        allOf.remove(1);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void fixFlow(Map<String, Object> map) {
+        var definitions = (Map<String, Map<String, Object>>) map.get("definitions");
+        var flow = (Map<String, Object>) definitions.get("io.kestra.core.models.flows.Flow");
+
+        var requireds = (List<String>) flow.get("required");
+        requireds.remove("deleted");
+
+        var properties = (Map<String, Object>) flow.get("properties");
+        properties.remove("deleted");
+    }
+
     public <T> Map<String, Object> properties(Class<T> base, Class<? extends T> cls) {
         return this.generate(cls, base);
     }
@@ -61,6 +153,7 @@ public <T> Map<String, Object> outputs(Class<T> base, Class<? extends T> cls) {
     }
 
     protected <T> void build(SchemaGeneratorConfigBuilder builder, Class<? extends T> cls) {
+
         builder
             .with(new JacksonModule())
             .with(new JavaxValidationModule(
@@ -156,6 +249,109 @@ public CustomDefinition provideCustomSchemaDefinition(ResolvedType javaType, Sch
 
             return Object.class;
         });
+        if(builder.build().getSchemaVersion() != SchemaVersion.DRAFT_2019_09) {
+            builder.forTypesInGeneral()
+                .withSubtypeResolver((declaredType, context) -> {
+                    TypeContext typeContext = context.getTypeContext();
+
+                    if (declaredType.getErasedType() == Task.class) {
+                        return pluginService
+                            .allPlugins()
+                            .stream()
+                            .flatMap(registeredPlugin -> registeredPlugin.getTasks().stream())
+                            .map(clz -> typeContext.resolveSubtype(declaredType, clz))
+                            .collect(Collectors.toList());
+                    } else if (declaredType.getErasedType() == AbstractTrigger.class) {
+                        return pluginService
+                            .allPlugins()
+                            .stream()
+                            .flatMap(registeredPlugin -> registeredPlugin.getTriggers().stream())
+                            .map(clz -> typeContext.resolveSubtype(declaredType, clz))
+                            .collect(Collectors.toList());
+                    } else if (declaredType.getErasedType() == Condition.class) {
+                        return pluginService
+                            .allPlugins()
+                            .stream()
+                            .flatMap(registeredPlugin -> registeredPlugin.getConditions().stream())
+                            .map(clz -> typeContext.resolveSubtype(declaredType, clz))
+                            .collect(Collectors.toList());
+                    } else if (declaredType.getErasedType() == ScheduleCondition.class) {
+                        return pluginService
+                            .allPlugins()
+                            .stream()
+                            .flatMap(registeredPlugin -> registeredPlugin.getConditions().stream())
+                            .filter(ScheduleCondition.class::isAssignableFrom)
+                            .map(clz -> typeContext.resolveSubtype(declaredType, clz))
+                            .collect(Collectors.toList());
+                    }
+
+                    return null;
+                });
+            // description as Markdown
+            builder.forTypesInGeneral().withTypeAttributeOverride((collectedTypeAttributes, scope, context) -> {
+                this.mutateDescription(collectedTypeAttributes);
+            });
+
+            builder.forFields().withInstanceAttributeOverride((collectedTypeAttributes, scope, context) -> {
+                this.mutateDescription(collectedTypeAttributes);
+            });
+
+            // default is no more required
+            builder.forTypesInGeneral().withTypeAttributeOverride((collectedTypeAttributes, scope, context) -> {
+                if (collectedTypeAttributes.has("required") && collectedTypeAttributes.get("required") instanceof ArrayNode) {
+                    ArrayNode required = context.getGeneratorConfig().createArrayNode();
+
+                    collectedTypeAttributes.get("required").forEach(jsonNode -> {
+                        if (!collectedTypeAttributes.get("properties").get(jsonNode.asText()).has("default")) {
+                            required.add(jsonNode.asText());
+                        }
+                    });
+
+                    collectedTypeAttributes.set("required", required);
+                }
+            });
+
+            // invalid regexp for jsonschema
+            builder.forFields().withInstanceAttributeOverride((collectedTypeAttributes, scope, context) -> {
+                if (collectedTypeAttributes.has("pattern") && collectedTypeAttributes.get("pattern").asText().contains("javaJavaIdentifier")) {
+                    collectedTypeAttributes.remove("pattern");
+                }
+            });
+
+            // examples in description
+            builder.forTypesInGeneral().withTypeAttributeOverride((collectedTypeAttributes, scope, context) -> {
+                if (collectedTypeAttributes.has("$examples")) {
+                    ArrayNode examples = (ArrayNode) collectedTypeAttributes.get("$examples");
+
+                    String doc = StreamSupport.stream(examples.spliterator(), true)
+                        .map(jsonNode -> {
+                            String description = "";
+                            if (jsonNode.has("title")) {
+                                description += "> " + jsonNode.get("title").asText() + "\n";
+                            }
+
+                            description += "```" +
+                                (jsonNode.has("lang") ? jsonNode.get("lang").asText() : "yaml")
+                                + "\n" +
+                                jsonNode.get("code").asText() +
+                                "\n```";
+
+                            return description;
+                        })
+                        .collect(Collectors.joining("\n\n"));
+
+                    String description = collectedTypeAttributes.has("markdownDescription") ?
+                        collectedTypeAttributes.get("markdownDescription").asText() :
+                        "";
+
+                    description += "##### Examples\n" + doc;
+
+                    collectedTypeAttributes.set("markdownDescription", new TextNode(description));
+
+                    collectedTypeAttributes.remove("$examples");
+                }
+            });
+        }
     }
 
     protected <T> Map<String, Object> generate(Class<? extends T> cls, @Nullable Class<T> base) {

core/src/test/java/io/kestra/core/docs/JsonSchemaGeneratorTest.java

@@ -1,11 +1,14 @@
 package io.kestra.core.docs;
 
-import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
-import org.junit.jupiter.api.Test;
+import io.kestra.core.Helpers;
+import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.tasks.Task;
 import io.kestra.core.plugins.PluginScanner;
 import io.kestra.core.plugins.RegisteredPlugin;
 import io.kestra.core.tasks.scripts.Bash;
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
 
 import java.net.URISyntaxException;
 import java.nio.file.Path;
@@ -14,8 +17,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import jakarta.inject.Inject;
-
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
@@ -49,16 +50,63 @@ void tasks() throws URISyntaxException {
 
     @SuppressWarnings("unchecked")
     @Test
-    void bash() {
-        Map<String, Object> generate = jsonSchemaGenerator.properties(Task.class, Bash.class);
+    void flow() throws URISyntaxException {
+        Helpers.runApplicationContext((applicationContext) -> {
+            JsonSchemaGenerator jsonSchemaGenerator = applicationContext.getBean(JsonSchemaGenerator.class);
+
+            Map<String, Object> generate = jsonSchemaGenerator.schemas(Flow.class);
+
+            var definitions = (Map<String, Map<String, Object>>) generate.get("definitions");
+
+            var flow = definitions.get("io.kestra.core.models.flows.Flow");
+            assertThat((List<String>) flow.get("required"), not(contains("deleted")));
+            assertThat((List<String>) flow.get("required"), hasItems("id", "namespace", "tasks"));
+
+            var bash = definitions.get("io.kestra.core.tasks.scripts.Bash-1");
+            assertThat((List<String>) bash.get("required"), not(contains("exitOnFailed")));
+            assertThat((String) ((Map<String, Map<String, Object>>) bash.get("properties")).get("exitOnFailed").get("markdownDescription"), containsString("Default value is : `true`"));
+            assertThat(((String) ((Map<String, Map<String, Object>>) bash.get("properties")).get("exitOnFailed").get("markdownDescription")).startsWith("This tells bash that"), is(true));
+            assertThat(((Map<String, Map<String, Object>>) bash.get("properties")).get("type").containsKey("pattern"), is(false));
+            assertThat((String) bash.get("markdownDescription"), containsString("Bash with some inputs files"));
+            assertThat((String) bash.get("markdownDescription"), containsString("outputFiles.first"));
+
+            var bashType = definitions.get("io.kestra.core.tasks.scripts.Bash-2");
+
+            var python = definitions.get("io.kestra.core.tasks.scripts.Python-1");
+            assertThat((List<String>) python.get("required"), not(contains("exitOnFailed")));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    void task() throws URISyntaxException {
+        Helpers.runApplicationContext((applicationContext) -> {
+            JsonSchemaGenerator jsonSchemaGenerator = applicationContext.getBean(JsonSchemaGenerator.class);
+
+            Map<String, Object> generate = jsonSchemaGenerator.schemas(Task.class);
+
+            var definitions = (Map<String, Map<String, Object>>) generate.get("definitions");
+            var task = (Map<String, Object>) definitions.get("io.kestra.core.models.tasks.Task-2");
+            var allOf = (List<Object>) task.get("allOf");
+
+            assertThat(allOf.size(), is(1));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    void bash() throws URISyntaxException {
+        Helpers.runApplicationContext((applicationContext) -> {
+            JsonSchemaGenerator jsonSchemaGenerator = applicationContext.getBean(JsonSchemaGenerator.class);
 
-        assertThat(generate, is(not(nullValue())));
-        assertThat(((Map<String, Map<String, Object>>) generate.get("properties")).size(), is(13));
+            Map<String, Object> generate = jsonSchemaGenerator.schemas(Bash.class);
 
-        generate = jsonSchemaGenerator.outputs(Task.class, Bash.class);
+            var definitions = (Map<String, Map<String, Object>>) generate.get("definitions");
 
-        assertThat(generate, is(not(nullValue())));
-        assertThat(((Map<String, Map<String, Object>>) generate.get("properties")).size(), is(6));
+            var bash = definitions.get("io.kestra.core.tasks.scripts.Bash-1");
+            assertThat((List<String>) bash.get("required"), not(contains("exitOnFailed")));
+            assertThat((List<String>) bash.get("required"), not(contains("interpreter")));
+        });
     }
 
     @SuppressWarnings("unchecked")

ui/src/components/flows/FlowEdit.vue

@@ -1,6 +1,6 @@
 <template>
     <div>
-        <flow-editor @save="save" v-model="content" lang="yaml" @update:model-value="onChange($event)" />
+        <editor @save="save" v-model="content" schemaType="flow" lang="yaml" @update:model-value="onChange($event)" />
         <bottom-line v-if="canSave || canDelete || canExecute">
             <ul>
                 <li>

ui/src/components/flows/FlowSource.vue

@@ -4,7 +4,7 @@
 
 <script>
     import {mapGetters} from "vuex";
-    import FlowEdit from "override/components/flows/FlowEdit.vue";
+    import FlowEdit from "./FlowEdit.vue";
     import RouteContext from "../../mixins/routeContext";
 
     export default {