feat: extract flows and templates

cli/src/main/java/io/kestra/cli/commands/flows/FlowCommand.java

@@ -16,7 +16,8 @@
         FlowValidateCommand.class,
         FlowTestCommand.class,
         FlowNamespaceCommand.class,
-        FlowDotCommand.class
+        FlowDotCommand.class,
+        FlowExportCommand.class,
     }
 )
 @Slf4j

cli/src/main/java/io/kestra/cli/commands/flows/FlowExportCommand.java

@@ -0,0 +1,54 @@
+package io.kestra.cli.commands.flows;
+
+import io.kestra.cli.AbstractApiCommand;
+import io.micronaut.http.HttpRequest;
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.MutableHttpRequest;
+import io.micronaut.http.client.exceptions.HttpClientResponseException;
+import io.micronaut.http.client.netty.DefaultHttpClient;
+import lombok.extern.slf4j.Slf4j;
+import picocli.CommandLine;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@CommandLine.Command(
+    name = "export",
+    description = "export flows to a zip file",
+    mixinStandardHelpOptions = true
+)
+@Slf4j
+public class FlowExportCommand extends AbstractApiCommand {
+    private static final String DEFAULT_FILE_NAME = "flows.zip";
+
+    @CommandLine.Option(names = {"--namespace"}, description = "the namespace of flows to export")
+    public String namespace;
+
+    @CommandLine.Parameters(index = "0", description = "the directory to export the file to")
+    public Path directory;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+
+        try(DefaultHttpClient client = client()) {
+            MutableHttpRequest<Object> request = HttpRequest
+                .GET("/api/v1/flows/export/by-query" + (namespace != null ? "?namespace=" + namespace : ""))
+                .accept(MediaType.APPLICATION_OCTET_STREAM);
+
+            HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
+            Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME);
+            zipFile.toFile().createNewFile();
+            Files.write(zipFile, response.body());
+
+            stdOut("Exporting flow(s) for namespace '" + namespace + "' successfully done !");
+        } catch (HttpClientResponseException e) {
+            FlowValidateCommand.handleHttpException(e, "flow");
+            return 1;
+        }
+
+        return 0;
+    }
+
+}

cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java

@@ -14,7 +14,8 @@
     mixinStandardHelpOptions = true,
     subcommands = {
         TemplateNamespaceCommand.class,
-        TemplateValidateCommand.class
+        TemplateValidateCommand.class,
+        TemplateExportCommand.class,
     }
 )
 @Slf4j

cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java

@@ -0,0 +1,53 @@
+package io.kestra.cli.commands.templates;
+
+import io.kestra.cli.AbstractApiCommand;
+import io.micronaut.http.HttpRequest;
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.MutableHttpRequest;
+import io.micronaut.http.client.exceptions.HttpClientResponseException;
+import io.micronaut.http.client.netty.DefaultHttpClient;
+import lombok.extern.slf4j.Slf4j;
+import picocli.CommandLine;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@CommandLine.Command(
+    name = "export",
+    description = "export templates to a zip file",
+    mixinStandardHelpOptions = true
+)
+@Slf4j
+public class TemplateExportCommand extends AbstractApiCommand {
+    private static final String DEFAULT_FILE_NAME = "templates.zip";
+
+    @CommandLine.Option(names = {"--namespace"}, description = "the namespace of templates to export")
+    public String namespace;
+
+    @CommandLine.Parameters(index = "0", description = "the directory to export the file to")
+    public Path directory;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+
+        try(DefaultHttpClient client = client()) {
+            MutableHttpRequest<Object> request = HttpRequest
+                .GET("/api/v1/templates/export/by-query" + (namespace != null ? "?namespace=" + namespace : ""))
+                .accept(MediaType.APPLICATION_OCTET_STREAM);
+
+            HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
+            Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME);
+            zipFile.toFile().createNewFile();
+            Files.write(zipFile, response.body());
+
+            stdOut("Exporting template(s) for namespace '" + namespace + "' successfully done !");
+        } catch (HttpClientResponseException e) {
+            TemplateValidateCommand.handleHttpException(e, "template");
+            return 1;
+        }
+
+        return 0;
+    }
+}

cli/src/test/java/io/kestra/cli/commands/flows/FlowExportCommandTest.java

@@ -0,0 +1,63 @@
+package io.kestra.cli.commands.flows;
+
+import io.kestra.cli.commands.flows.namespaces.FlowNamespaceUpdateCommand;
+import io.micronaut.configuration.picocli.PicocliRunner;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.env.Environment;
+import io.micronaut.runtime.server.EmbeddedServer;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.zip.ZipFile;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.StringContains.containsString;
+
+class FlowExportCommandTest {
+    @Test
+    void run() throws IOException {
+        URL directory = FlowExportCommandTest.class.getClassLoader().getResource("flows");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            // we use the update command to add flows to extract
+            String[] updateArgs = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.cli",
+                directory.getPath(),
+            };
+            PicocliRunner.call(FlowNamespaceUpdateCommand.class, ctx, updateArgs);
+            assertThat(out.toString(), containsString("3 flow(s)"));
+
+            // then we export them
+            String[] exportArgs = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "--namespace",
+                "io.kestra.cli",
+                "/tmp",
+            };
+            PicocliRunner.call(FlowExportCommand.class, ctx, exportArgs);
+            File file = new File("/tmp/flows.zip");
+            assertThat(file.exists(), is(true));
+            ZipFile zipFile = new ZipFile(file);
+            assertThat(zipFile.stream().count(), is(3L));
+
+            file.delete();
+        }
+    }
+}

cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java

@@ -0,0 +1,66 @@
+package io.kestra.cli.commands.templates;
+
+import io.kestra.cli.commands.templates.namespaces.TemplateNamespaceUpdateCommand;
+import io.micronaut.configuration.picocli.PicocliRunner;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.env.Environment;
+import io.micronaut.runtime.server.EmbeddedServer;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.zip.ZipFile;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.hamcrest.core.Is.is;
+
+class TemplateExportCommandTest {
+    @Test
+    void run() throws IOException {
+        URL directory = TemplateExportCommandTest.class.getClassLoader().getResource("templates");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            // we use the update command to add templates to extract
+            String[] args = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.tests",
+                directory.getPath(),
+
+            };
+            PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args);
+            assertThat(out.toString(), containsString("3 template(s)"));
+
+            // then we export them
+            String[] exportArgs = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "--namespace",
+                "io.kestra.tests",
+                "/tmp",
+            };
+            PicocliRunner.call(TemplateExportCommand.class, ctx, exportArgs);
+            File file = new File("/tmp/templates.zip");
+            assertThat(file.exists(), is(true));
+            ZipFile zipFile = new ZipFile(file);
+            assertThat(zipFile.stream().count(), is(3L));
+
+            file.delete();
+        }
+    }
+
+}

core/src/main/java/io/kestra/core/models/templates/Template.java

@@ -1,9 +1,15 @@
 package io.kestra.core.models.templates;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
+import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
 import io.kestra.core.models.DeletedInterface;
 import io.kestra.core.models.tasks.Task;
 import io.kestra.core.models.validations.ManualConstraintViolation;
+import io.kestra.core.serializers.JacksonMapper;
 import io.micronaut.core.annotation.Introspected;
 import lombok.*;
 import lombok.experimental.SuperBuilder;
@@ -25,6 +31,16 @@
 @ToString
 @EqualsAndHashCode
 public class Template implements DeletedInterface {
+    private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml().copy()
+        .setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
+            @Override
+            public boolean hasIgnoreMarker(final AnnotatedMember m) {
+                List<String> exclusions = Arrays.asList("revision", "deleted", "source");
+                return exclusions.contains(m.getName()) || super.hasIgnoreMarker(m);
+            }
+        })
+        .setSerializationInclusion(JsonInclude.Include.NON_DEFAULT);
+
     @NotNull
     @NotBlank
     @Pattern(regexp = "[a-zA-Z0-9._-]+")
@@ -93,6 +109,14 @@ public Optional<ConstraintViolationException> validateUpdate(Template updated) {
         }
     }
 
+    public String generateSource() {
+        try {
+            return YAML_MAPPER.writeValueAsString(this);
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public Template toDeleted() {
         return new Template(
             this.id,

core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java

@@ -55,6 +55,12 @@ ArrayListTotal<Flow> find(
         @Nullable Map<String, String> labels
     );
 
+    List<FlowWithSource> findWithSource(
+        @Nullable String query,
+        @Nullable String namespace,
+        @Nullable Map<String, String> labels
+    );
+
     ArrayListTotal<SearchResult<Flow>> findSourceCode(Pageable pageable, @Nullable String query, @Nullable String namespace);
 
     List<String> findDistinctNamespace();

core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java

@@ -18,6 +18,12 @@ ArrayListTotal<Template> find(
         @Nullable String namespace
     );
 
+    // Should normally be TemplateWithSource but it didn't exist yet
+    List<Template> find(
+        @Nullable String query,
+        @Nullable String namespace
+    );
+
     List<Template> findByNamespace(String namespace);
 
     Template create(Template template);

core/src/test/java/io/kestra/core/repositories/AbstractFlowRepositoryTest.java

@@ -19,6 +19,7 @@
 import io.kestra.core.utils.IdUtils;
 import io.kestra.core.utils.TestsUtils;
 import io.micronaut.context.event.ApplicationEventListener;
+import io.micronaut.data.model.Pageable;
 import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
@@ -216,6 +217,24 @@ void findByNamespace() {
         assertThat((long) save.size(), is(1L));
     }
 
+    @Test
+    void find() {
+        List<Flow> save = flowRepository.find(Pageable.from(1, 10),null, "io.kestra.tests", Collections.emptyMap());
+        assertThat((long) save.size(), is(10L));
+
+        save = flowRepository.find(Pageable.from(1),null, "io.kestra.tests.minimal.bis", Collections.emptyMap());
+        assertThat((long) save.size(), is(1L));
+    }
+
+    @Test
+    void findWithSource() {
+        List<FlowWithSource> save = flowRepository.findWithSource(null, "io.kestra.tests", Collections.emptyMap());
+        assertThat((long) save.size(), is(Helpers.FLOWS_COUNT));
+
+        save = flowRepository.findWithSource(null, "io.kestra.tests.minimal.bis", Collections.emptyMap());
+        assertThat((long) save.size(), is(1L));
+    }
+
     @Test
     void delete() {
         Flow flow = builder().build();