From eb9ef947e1cb3007f61b9cd11b8363b556fee303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Mathieu?= Date: Tue, 28 Mar 2023 09:03:32 +0200 Subject: [PATCH] fix(core): unable to include another file in the flow (#1105) also add an expand command to expand a flow containing helper close #1088 --- .../cli/commands/flows/FlowExpandCommand.java | 37 ++++++++++++++ .../commands/flows/IncludeHelperExpander.java | 39 +++++++++++++++ .../FlowNamespaceUpdateCommand.java | 4 +- .../commands/flows/FlowExpandCommandTest.java | 49 +++++++++++++++++++ .../flows/FlowValidateCommandTest.java | 36 ++++++++++++++ .../FlowNamespaceUpdateCommandTest.java | 27 ++++++++++ cli/src/test/resources/helper/flow.yaml | 12 +++++ .../test/resources/helper/lorem-multiple.txt | 2 + cli/src/test/resources/helper/lorem.txt | 1 + 9 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java create mode 100644 cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java create mode 100644 cli/src/test/java/io/kestra/cli/commands/flows/FlowExpandCommandTest.java create mode 100644 cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java create mode 100644 cli/src/test/resources/helper/flow.yaml create mode 100644 cli/src/test/resources/helper/lorem-multiple.txt create mode 100644 cli/src/test/resources/helper/lorem.txt diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java new file mode 100644 index 00000000000..bb6e0ac27da --- /dev/null +++ b/cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java @@ -0,0 +1,37 @@ +package io.kestra.cli.commands.flows; + +import io.kestra.cli.AbstractCommand; +import io.kestra.core.models.flows.Flow; +import io.kestra.core.models.validations.ModelValidator; +import io.kestra.core.serializers.YamlFlowParser; +import jakarta.inject.Inject; +import picocli.CommandLine; + +import java.nio.file.Files; +import java.nio.file.Path; + +@CommandLine.Command( + name = "expand", + description = "expand a flow" +) +public class FlowExpandCommand extends AbstractCommand { + + @CommandLine.Parameters(index = "0", description = "the flow file to expand") + private Path file; + + @Inject + private YamlFlowParser yamlFlowParser; + + @Inject + private ModelValidator modelValidator; + + @Override + public Integer call() throws Exception { + super.call(); + String content = IncludeHelperExpander.expand(Files.readString(file), file.getParent()); + Flow flow = yamlFlowParser.parse(content, Flow.class); + modelValidator.validate(flow); + stdOut(content); + return 0; + } +} diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java b/cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java new file mode 100644 index 00000000000..2959ca0cace --- /dev/null +++ b/cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java @@ -0,0 +1,39 @@ +package io.kestra.cli.commands.flows; + +import com.google.common.io.Files; +import lombok.SneakyThrows; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +public abstract class IncludeHelperExpander { + + public static String expand(String value, Path directory) throws IOException { + return value.lines() + .map(line -> line.contains("[[>") && line.contains("]]") ? expandLine(line, directory) : line) + .collect(Collectors.joining("\n")); + } + + @SneakyThrows + private static String expandLine(String line, Path directory) { + String prefix = line.substring(0, line.indexOf("[[>")); + String suffix = line.substring(line.indexOf("]]") + 2, line.length()); + String file = line.substring(line.indexOf("[[>") + 3 , line.indexOf("]]")).strip(); + Path includePath = directory.resolve(file); + List include = Files.readLines(includePath.toFile(), Charset.defaultCharset()); + + // handle single line directly with the suffix (should be between quotes or double-quotes + if(include.size() == 1) { + String singleInclude = include.get(0); + return prefix + singleInclude + suffix; + } + + // multi-line will be expanded with the prefix but no suffix + return include.stream() + .map(includeLine -> prefix + includeLine) + .collect(Collectors.joining("\n")); + } +} diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java index 3a095926513..410c3f19cb9 100644 --- a/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java @@ -2,7 +2,7 @@ import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand; import io.kestra.cli.commands.flows.FlowValidateCommand; -import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.cli.commands.flows.IncludeHelperExpander; import io.kestra.core.serializers.YamlFlowParser; import io.micronaut.core.type.Argument; import io.micronaut.http.HttpRequest; @@ -42,7 +42,7 @@ public Integer call() throws Exception { .filter(YamlFlowParser::isValidExtension) .map(path -> { try { - return Files.readString(path, Charset.defaultCharset()); + return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/cli/src/test/java/io/kestra/cli/commands/flows/FlowExpandCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/flows/FlowExpandCommandTest.java new file mode 100644 index 00000000000..adfa2b6066e --- /dev/null +++ b/cli/src/test/java/io/kestra/cli/commands/flows/FlowExpandCommandTest.java @@ -0,0 +1,49 @@ +package io.kestra.cli.commands.flows; + +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.env.Environment; +import io.micronaut.runtime.server.EmbeddedServer; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringContains.containsString; + +class FlowExpandCommandTest { + @Test + void run() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); + embeddedServer.start(); + + String[] args = { + "src/test/resources/helper/flow.yaml" + }; + Integer call = PicocliRunner.call(FlowExpandCommand.class, ctx, args); + + assertThat(call, is(0)); + assertThat(out.toString(), is( + "id: include\n" + + "namespace: io.kestra.cli\n" + + "\n" + + "# The list of tasks\n" + + "tasks:\n" + + "- id: t1\n" + + " type: io.kestra.core.tasks.debugs.Return\n" + + " format: \"Lorem ipsum dolor sit amet\"\n" + + "- id: t2\n" + + " type: io.kestra.core.tasks.debugs.Return\n" + + " format: |\n" + + " Lorem ipsum dolor sit amet\n" + + " Lorem ipsum dolor sit amet\n" + )); + } + } +} \ No newline at end of file diff --git a/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java new file mode 100644 index 00000000000..0a40b056cc0 --- /dev/null +++ b/cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java @@ -0,0 +1,36 @@ +package io.kestra.cli.commands.flows; + +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.env.Environment; +import io.micronaut.runtime.server.EmbeddedServer; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringContains.containsString; + +class FlowValidateCommandTest { + @Test + void run() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); + embeddedServer.start(); + + String[] args = { + "--local", + "src/test/resources/helper/flow.yaml" + }; + Integer call = PicocliRunner.call(FlowValidateCommand.class, ctx, args); + + assertThat(call, is(0)); + assertThat(out.toString(), containsString("✓ - io.kestra.cli / include")); + } + } +} \ No newline at end of file diff --git a/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommandTest.java index a0524dd2db7..24f93271921 100644 --- a/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommandTest.java @@ -109,4 +109,31 @@ void runNoDelete() { assertThat(out.toString(), containsString("1 flow(s)")); } } + + @Test + void helper() { + URL directory = FlowNamespaceUpdateCommandTest.class.getClassLoader().getResource("helper"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + + EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); + embeddedServer.start(); + + String[] args = { + "--server", + embeddedServer.getURL().toString(), + "--user", + "myuser:pass:word", + "io.kestra.cli", + directory.getPath(), + + }; + Integer call = PicocliRunner.call(FlowNamespaceUpdateCommand.class, ctx, args); + + assertThat(call, is(0)); + assertThat(out.toString(), containsString("1 flow(s)")); + } + } } diff --git a/cli/src/test/resources/helper/flow.yaml b/cli/src/test/resources/helper/flow.yaml new file mode 100644 index 00000000000..13d03a8c512 --- /dev/null +++ b/cli/src/test/resources/helper/flow.yaml @@ -0,0 +1,12 @@ +id: include +namespace: io.kestra.cli + +# The list of tasks +tasks: +- id: t1 + type: io.kestra.core.tasks.debugs.Return + format: "[[> lorem.txt]]" +- id: t2 + type: io.kestra.core.tasks.debugs.Return + format: | + [[> lorem-multiple.txt]] \ No newline at end of file diff --git a/cli/src/test/resources/helper/lorem-multiple.txt b/cli/src/test/resources/helper/lorem-multiple.txt new file mode 100644 index 00000000000..4d8f63e1e1c --- /dev/null +++ b/cli/src/test/resources/helper/lorem-multiple.txt @@ -0,0 +1,2 @@ +Lorem ipsum dolor sit amet +Lorem ipsum dolor sit amet \ No newline at end of file diff --git a/cli/src/test/resources/helper/lorem.txt b/cli/src/test/resources/helper/lorem.txt new file mode 100644 index 00000000000..d781006b2d0 --- /dev/null +++ b/cli/src/test/resources/helper/lorem.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet \ No newline at end of file