fix(core): unable to include another file in the flow (#1105)

also add an expand command to expand a flow containing helper

close #1088

cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java

@@ -0,0 +1,37 @@
+package io.kestra.cli.commands.flows;
+
+import io.kestra.cli.AbstractCommand;
+import io.kestra.core.models.flows.Flow;
+import io.kestra.core.models.validations.ModelValidator;
+import io.kestra.core.serializers.YamlFlowParser;
+import jakarta.inject.Inject;
+import picocli.CommandLine;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@CommandLine.Command(
+    name = "expand",
+    description = "expand a flow"
+)
+public class FlowExpandCommand extends AbstractCommand {
+
+    @CommandLine.Parameters(index = "0", description = "the flow file to expand")
+    private Path file;
+
+    @Inject
+    private YamlFlowParser yamlFlowParser;
+
+    @Inject
+    private ModelValidator modelValidator;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+        String content = IncludeHelperExpander.expand(Files.readString(file), file.getParent());
+        Flow flow = yamlFlowParser.parse(content, Flow.class);
+        modelValidator.validate(flow);
+        stdOut(content);
+        return 0;
+    }
+}

cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java

@@ -0,0 +1,39 @@
+package io.kestra.cli.commands.flows;
+
+import com.google.common.io.Files;
+import lombok.SneakyThrows;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public abstract class IncludeHelperExpander {
+
+    public static String expand(String value, Path directory) throws IOException {
+        return value.lines()
+            .map(line -> line.contains("[[>") && line.contains("]]") ? expandLine(line, directory) : line)
+            .collect(Collectors.joining("\n"));
+    }
+
+    @SneakyThrows
+    private static String expandLine(String line, Path directory) {
+        String prefix = line.substring(0, line.indexOf("[[>"));
+        String suffix = line.substring(line.indexOf("]]") + 2, line.length());
+        String file = line.substring(line.indexOf("[[>") + 3 , line.indexOf("]]")).strip();
+        Path includePath = directory.resolve(file);
+        List<String> include = Files.readLines(includePath.toFile(), Charset.defaultCharset());
+
+        // handle single line directly with the suffix (should be between quotes or double-quotes
+        if(include.size() == 1) {
+            String singleInclude = include.get(0);
+            return prefix + singleInclude + suffix;
+        }
+
+        // multi-line will be expanded with the prefix but no suffix
+        return include.stream()
+            .map(includeLine -> prefix + includeLine)
+            .collect(Collectors.joining("\n"));
+    }
+}

cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java

@@ -2,7 +2,7 @@
 
 import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
 import io.kestra.cli.commands.flows.FlowValidateCommand;
-import io.kestra.core.models.flows.FlowWithSource;
+import io.kestra.cli.commands.flows.IncludeHelperExpander;
 import io.kestra.core.serializers.YamlFlowParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
@@ -42,7 +42,7 @@ public Integer call() throws Exception {
                 .filter(YamlFlowParser::isValidExtension)
                 .map(path -> {
                     try {
-                        return Files.readString(path, Charset.defaultCharset());
+                        return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent());
                     } catch (IOException e) {
                         throw new RuntimeException(e);
                     }

cli/src/test/java/io/kestra/cli/commands/flows/FlowExpandCommandTest.java

@@ -0,0 +1,49 @@
+package io.kestra.cli.commands.flows;
+
+import io.micronaut.configuration.picocli.PicocliRunner;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.env.Environment;
+import io.micronaut.runtime.server.EmbeddedServer;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.StringContains.containsString;
+
+class FlowExpandCommandTest {
+    @Test
+    void run() {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            String[] args = {
+                "src/test/resources/helper/flow.yaml"
+            };
+            Integer call = PicocliRunner.call(FlowExpandCommand.class, ctx, args);
+
+            assertThat(call, is(0));
+            assertThat(out.toString(), is(
+                "id: include\n" +
+                    "namespace: io.kestra.cli\n" +
+                    "\n" +
+                    "# The list of tasks\n" +
+                    "tasks:\n" +
+                    "- id: t1\n" +
+                    "  type: io.kestra.core.tasks.debugs.Return\n" +
+                    "  format: \"Lorem ipsum dolor sit amet\"\n" +
+                    "- id: t2\n" +
+                    "  type: io.kestra.core.tasks.debugs.Return\n" +
+                    "  format: |\n" +
+                    "    Lorem ipsum dolor sit amet\n" +
+                    "    Lorem ipsum dolor sit amet\n"
+            ));
+        }
+    }
+}

cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java

@@ -0,0 +1,36 @@
+package io.kestra.cli.commands.flows;
+
+import io.micronaut.configuration.picocli.PicocliRunner;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.env.Environment;
+import io.micronaut.runtime.server.EmbeddedServer;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.StringContains.containsString;
+
+class FlowValidateCommandTest {
+    @Test
+    void run() {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            String[] args = {
+                "--local",
+                "src/test/resources/helper/flow.yaml"
+            };
+            Integer call = PicocliRunner.call(FlowValidateCommand.class, ctx, args);
+
+            assertThat(call, is(0));
+            assertThat(out.toString(), containsString("✓ - io.kestra.cli / include"));
+        }
+    }
+}

cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommandTest.java

@@ -109,4 +109,31 @@ void runNoDelete()  {
             assertThat(out.toString(), containsString("1 flow(s)"));
         }
     }
+
+    @Test
+    void helper()  {
+        URL directory = FlowNamespaceUpdateCommandTest.class.getClassLoader().getResource("helper");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            String[] args = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.cli",
+                directory.getPath(),
+
+            };
+            Integer call = PicocliRunner.call(FlowNamespaceUpdateCommand.class, ctx, args);
+
+            assertThat(call, is(0));
+            assertThat(out.toString(), containsString("1 flow(s)"));
+        }
+    }
 }

cli/src/test/resources/helper/flow.yaml

@@ -0,0 +1,12 @@
+id: include
+namespace: io.kestra.cli
+
+# The list of tasks
+tasks:
+- id: t1
+  type: io.kestra.core.tasks.debugs.Return
+  format: "[[> lorem.txt]]"
+- id: t2
+  type: io.kestra.core.tasks.debugs.Return
+  format: |
+    [[> lorem-multiple.txt]]

cli/src/test/resources/helper/lorem-multiple.txt

@@ -0,0 +1,2 @@
+Lorem ipsum dolor sit amet
+Lorem ipsum dolor sit amet

cli/src/test/resources/helper/lorem.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet