feat: extract flows and templates

cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommand.java

@@ -0,0 +1,54 @@
+package io.kestra.cli.commands.flows.namespaces;
+
+import io.kestra.cli.AbstractApiCommand;
+import io.kestra.cli.commands.flows.FlowValidateCommand;
+import io.micronaut.http.HttpRequest;
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.MutableHttpRequest;
+import io.micronaut.http.client.exceptions.HttpClientResponseException;
+import io.micronaut.http.client.netty.DefaultHttpClient;
+import lombok.extern.slf4j.Slf4j;
+import picocli.CommandLine;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@CommandLine.Command(
+    name = "extract",
+    description = "extract namespace flows",
+    mixinStandardHelpOptions = true
+)
+@Slf4j
+public class FlowNamespaceExtractCommand extends AbstractApiCommand {
+    private static final String DEFAULT_FILE_NAME = "flows.zip";
+
+    @CommandLine.Parameters(index = "0", description = "the namespace of templates to extract")
+    public String namespace;
+
+    @CommandLine.Parameters(index = "1", description = "the directory to extract the file to")
+    public Path directory;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+
+        try(DefaultHttpClient client = client()) {
+            MutableHttpRequest<Object> request = HttpRequest
+                .GET("/api/v1/flows/extract/by_query?namespace=" + namespace).accept(MediaType.APPLICATION_OCTET_STREAM);
+
+            HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
+            Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME);
+            zipFile.toFile().createNewFile();
+            Files.write(zipFile, response.body());
+
+            stdOut("Extracted flow(s) for namespace '" + namespace + "' successfully done !");
+        } catch (HttpClientResponseException e) {
+            FlowValidateCommand.handleHttpException(e, "flow");
+            return 1;
+        }
+
+        return 0;
+    }
+
+}

cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommand.java

@@ -0,0 +1,63 @@
+package io.kestra.cli.commands.templates.namespaces;
+
+import io.kestra.cli.AbstractApiCommand;
+import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
+import io.kestra.cli.commands.flows.FlowValidateCommand;
+import io.kestra.cli.commands.templates.TemplateValidateCommand;
+import io.kestra.core.models.templates.Template;
+import io.kestra.core.serializers.YamlFlowParser;
+import io.micronaut.core.type.Argument;
+import io.micronaut.http.HttpRequest;
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.MutableHttpRequest;
+import io.micronaut.http.client.exceptions.HttpClientResponseException;
+import io.micronaut.http.client.netty.DefaultHttpClient;
+import jakarta.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+import picocli.CommandLine;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import javax.validation.ConstraintViolationException;
+
+@CommandLine.Command(
+    name = "extract",
+    description = "extract namespace templates",
+    mixinStandardHelpOptions = true
+)
+@Slf4j
+public class TemplateNamespaceExtractCommand extends AbstractApiCommand {
+    private static final String DEFAULT_FILE_NAME = "templates.zip";
+
+    @CommandLine.Parameters(index = "0", description = "the namespace of templates to extract")
+    public String namespace;
+
+    @CommandLine.Parameters(index = "1", description = "the directory to extract the file to")
+    public Path directory;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+
+        try(DefaultHttpClient client = client()) {
+            MutableHttpRequest<Object> request = HttpRequest
+                .GET("/api/v1/templates/extract/by_query?namespace=" + namespace).accept(MediaType.APPLICATION_OCTET_STREAM);
+
+            HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
+            Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME);
+            zipFile.toFile().createNewFile();
+            Files.write(zipFile, response.body());
+
+            stdOut("Extracted template(s) for namespace '" + namespace + "' successfully done !");
+        } catch (HttpClientResponseException e) {
+            TemplateValidateCommand.handleHttpException(e, "template");
+            return 1;
+        }
+
+        return 0;
+    }
+
+}

cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommandTest.java

@@ -0,0 +1,63 @@
+package io.kestra.cli.commands.flows.namespaces;
+
+import io.micronaut.configuration.picocli.PicocliRunner;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.env.Environment;
+import io.micronaut.runtime.server.EmbeddedServer;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.zip.ZipFile;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.StringContains.containsString;
+
+class FlowNamespaceExtractCommandTest {
+    @Test
+    void run() throws IOException {
+        URL directory = FlowNamespaceUpdateCommandTest.class.getClassLoader().getResource("flows");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            // we use the update command to add flows to extract
+            String[] updateArgs = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.cli",
+                directory.getPath(),
+
+            };
+            PicocliRunner.call(FlowNamespaceUpdateCommand.class, ctx, updateArgs);
+            assertThat(out.toString(), containsString("3 flow(s)"));
+
+            // then we extract them
+            String[] extractArgs = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.cli",
+                "/tmp",
+            };
+            PicocliRunner.call(FlowNamespaceExtractCommand.class, ctx, extractArgs);
+            File file = new File("/tmp/flows.zip");
+            assertThat(file.exists(), is(true));
+            ZipFile zipFile = new ZipFile(file);
+            assertThat(zipFile.stream().count(), is(3L));
+
+            file.delete();
+        }
+    }
+}

cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommandTest.java

@@ -0,0 +1,64 @@
+package io.kestra.cli.commands.templates.namespaces;
+
+import io.micronaut.configuration.picocli.PicocliRunner;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.env.Environment;
+import io.micronaut.runtime.server.EmbeddedServer;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.zip.ZipFile;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.hamcrest.core.Is.is;
+
+class TemplateNamespaceExtractCommandTest {
+    @Test
+    void run() throws IOException {
+        URL directory = TemplateNamespaceExtractCommandTest.class.getClassLoader().getResource("templates");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(out));
+
+        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+
+            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
+            embeddedServer.start();
+
+            // we use the update command to add templates to extract
+            String[] args = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.tests",
+                directory.getPath(),
+
+            };
+            PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args);
+            assertThat(out.toString(), containsString("3 template(s)"));
+
+            // then we extract them
+            String[] extractArgs = {
+                "--server",
+                embeddedServer.getURL().toString(),
+                "--user",
+                "myuser:pass:word",
+                "io.kestra.tests",
+                "/tmp",
+            };
+            PicocliRunner.call(TemplateNamespaceExtractCommand.class, ctx, extractArgs);
+            File file = new File("/tmp/templates.zip");
+            assertThat(file.exists(), is(true));
+            ZipFile zipFile = new ZipFile(file);
+            assertThat(zipFile.stream().count(), is(3L));
+
+            file.delete();
+        }
+    }
+
+}

core/src/main/java/io/kestra/core/models/templates/Template.java

@@ -1,9 +1,15 @@
 package io.kestra.core.models.templates;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
+import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
 import io.kestra.core.models.DeletedInterface;
 import io.kestra.core.models.tasks.Task;
 import io.kestra.core.models.validations.ManualConstraintViolation;
+import io.kestra.core.serializers.JacksonMapper;
 import io.micronaut.core.annotation.Introspected;
 import lombok.*;
 import lombok.experimental.SuperBuilder;
@@ -25,6 +31,16 @@
 @ToString
 @EqualsAndHashCode
 public class Template implements DeletedInterface {
+
+    private static final ObjectMapper jsonMapper = JacksonMapper.ofJson().copy()
+        .setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
+            @Override
+            public boolean hasIgnoreMarker(final AnnotatedMember m) {
+                List<String> exclusions = Arrays.asList("revision", "deleted", "source");
+                return exclusions.contains(m.getName()) || super.hasIgnoreMarker(m);
+            }
+        });
+
     @NotNull
     @NotBlank
     @Pattern(regexp = "[a-zA-Z0-9._-]+")
@@ -93,6 +109,23 @@ public Optional<ConstraintViolationException> validateUpdate(Template updated) {
         }
     }
 
+    public String generateSource() {
+        try {
+            return JacksonMapper.ofYaml()
+                .writeValueAsString(
+                    JacksonMapper
+                        .ofJson()
+                        .readTree(
+                            jsonMapper.copy()
+                                .setSerializationInclusion(JsonInclude.Include.NON_DEFAULT)
+                                .writeValueAsString(this)
+                        )
+                );
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public Template toDeleted() {
         return new Template(
             this.id,

core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java

@@ -55,6 +55,12 @@ ArrayListTotal<Flow> find(
         @Nullable Map<String, String> labels
     );
 
+    List<FlowWithSource> findWithSource(
+        @Nullable String query,
+        @Nullable String namespace,
+        @Nullable Map<String, String> labels
+    );
+
     ArrayListTotal<SearchResult<Flow>> findSourceCode(Pageable pageable, @Nullable String query, @Nullable String namespace);
 
     List<String> findDistinctNamespace();

core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java

@@ -18,6 +18,12 @@ ArrayListTotal<Template> find(
         @Nullable String namespace
     );
 
+    // Should normally be TemplateWithSource but it didn't exist yet
+    List<Template> find(
+        @Nullable String query,
+        @Nullable String namespace
+    );
+
     List<Template> findByNamespace(String namespace);
 
     Template create(Template template);

core/src/test/java/io/kestra/core/repositories/AbstractFlowRepositoryTest.java

@@ -19,6 +19,7 @@
 import io.kestra.core.utils.IdUtils;
 import io.kestra.core.utils.TestsUtils;
 import io.micronaut.context.event.ApplicationEventListener;
+import io.micronaut.data.model.Pageable;
 import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
@@ -216,6 +217,24 @@ void findByNamespace() {
         assertThat((long) save.size(), is(1L));
     }
 
+    @Test
+    void find() {
+        List<Flow> save = flowRepository.find(Pageable.from(1, 10),null, "io.kestra.tests", Collections.emptyMap());
+        assertThat((long) save.size(), is(10L));
+
+        save = flowRepository.find(Pageable.from(1),null, "io.kestra.tests.minimal.bis", Collections.emptyMap());
+        assertThat((long) save.size(), is(1L));
+    }
+
+    @Test
+    void findWithSource() {
+        List<FlowWithSource> save = flowRepository.findWithSource(null, "io.kestra.tests", Collections.emptyMap());
+        assertThat((long) save.size(), is(Helpers.FLOWS_COUNT));
+
+        save = flowRepository.findWithSource(null, "io.kestra.tests.minimal.bis", Collections.emptyMap());
+        assertThat((long) save.size(), is(1L));
+    }
+
     @Test
     void delete() {
         Flow flow = builder().build();