From c22fe91499b4e200b4a856856f216e73b2cb9ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Mathieu?= Date: Tue, 14 Feb 2023 10:18:07 +0100 Subject: [PATCH 1/9] feat: extract flows and templates --- .../FlowNamespaceExtractCommand.java | 54 ++++++++++++++++ .../TemplateNamespaceExtractCommand.java | 63 ++++++++++++++++++ .../FlowNamespaceExtractCommandTest.java | 63 ++++++++++++++++++ .../TemplateNamespaceExtractCommandTest.java | 64 +++++++++++++++++++ .../core/models/templates/Template.java | 33 ++++++++++ .../repositories/FlowRepositoryInterface.java | 6 ++ .../TemplateRepositoryInterface.java | 6 ++ .../AbstractFlowRepositoryTest.java | 19 ++++++ .../AbstractTemplateRepositoryTest.java | 26 ++++++++ .../AbstractJdbcFlowRepository.java | 27 ++++++++ .../AbstractJdbcTemplateRepository.java | 27 ++++++++ .../memory/MemoryFlowRepository.java | 26 +++++++- .../memory/MemoryTemplateRepository.java | 12 +++- .../memory/MemoryFlowRepositoryTest.java | 10 +++ .../webserver/controllers/FlowController.java | 60 +++++++++++++++++ .../controllers/TemplateController.java | 59 ++++++++++++++++- .../controllers/FlowControllerTest.java | 31 +++++++++ .../controllers/TemplateControllerTest.java | 47 +++++++++++++- 18 files changed, 627 insertions(+), 6 deletions(-) create mode 100644 cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommand.java create mode 100644 cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommand.java create mode 100644 cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommandTest.java create mode 100644 cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommandTest.java diff --git a/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommand.java b/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommand.java new file mode 100644 index 00000000000..37356ce3c5f --- /dev/null +++ b/cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommand.java @@ -0,0 +1,54 @@ +package io.kestra.cli.commands.flows.namespaces; + +import io.kestra.cli.AbstractApiCommand; +import io.kestra.cli.commands.flows.FlowValidateCommand; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.MediaType; +import io.micronaut.http.MutableHttpRequest; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.client.netty.DefaultHttpClient; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; + +import java.nio.file.Files; +import java.nio.file.Path; + +@CommandLine.Command( + name = "extract", + description = "extract namespace flows", + mixinStandardHelpOptions = true +) +@Slf4j +public class FlowNamespaceExtractCommand extends AbstractApiCommand { + private static final String DEFAULT_FILE_NAME = "flows.zip"; + + @CommandLine.Parameters(index = "0", description = "the namespace of templates to extract") + public String namespace; + + @CommandLine.Parameters(index = "1", description = "the directory to extract the file to") + public Path directory; + + @Override + public Integer call() throws Exception { + super.call(); + + try(DefaultHttpClient client = client()) { + MutableHttpRequest request = HttpRequest + .GET("/api/v1/flows/extract/by_query?namespace=" + namespace).accept(MediaType.APPLICATION_OCTET_STREAM); + + HttpResponse response = client.toBlocking().exchange(this.requestOptions(request), byte[].class); + Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME); + zipFile.toFile().createNewFile(); + Files.write(zipFile, response.body()); + + stdOut("Extracted flow(s) for namespace '" + namespace + "' successfully done !"); + } catch (HttpClientResponseException e) { + FlowValidateCommand.handleHttpException(e, "flow"); + return 1; + } + + return 0; + } + +} diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommand.java new file mode 100644 index 00000000000..088b84b530a --- /dev/null +++ b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommand.java @@ -0,0 +1,63 @@ +package io.kestra.cli.commands.templates.namespaces; + +import io.kestra.cli.AbstractApiCommand; +import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand; +import io.kestra.cli.commands.flows.FlowValidateCommand; +import io.kestra.cli.commands.templates.TemplateValidateCommand; +import io.kestra.core.models.templates.Template; +import io.kestra.core.serializers.YamlFlowParser; +import io.micronaut.core.type.Argument; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.MediaType; +import io.micronaut.http.MutableHttpRequest; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.client.netty.DefaultHttpClient; +import jakarta.inject.Inject; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import javax.validation.ConstraintViolationException; + +@CommandLine.Command( + name = "extract", + description = "extract namespace templates", + mixinStandardHelpOptions = true +) +@Slf4j +public class TemplateNamespaceExtractCommand extends AbstractApiCommand { + private static final String DEFAULT_FILE_NAME = "templates.zip"; + + @CommandLine.Parameters(index = "0", description = "the namespace of templates to extract") + public String namespace; + + @CommandLine.Parameters(index = "1", description = "the directory to extract the file to") + public Path directory; + + @Override + public Integer call() throws Exception { + super.call(); + + try(DefaultHttpClient client = client()) { + MutableHttpRequest request = HttpRequest + .GET("/api/v1/templates/extract/by_query?namespace=" + namespace).accept(MediaType.APPLICATION_OCTET_STREAM); + + HttpResponse response = client.toBlocking().exchange(this.requestOptions(request), byte[].class); + Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME); + zipFile.toFile().createNewFile(); + Files.write(zipFile, response.body()); + + stdOut("Extracted template(s) for namespace '" + namespace + "' successfully done !"); + } catch (HttpClientResponseException e) { + TemplateValidateCommand.handleHttpException(e, "template"); + return 1; + } + + return 0; + } + +} diff --git a/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommandTest.java new file mode 100644 index 00000000000..92a5db02d69 --- /dev/null +++ b/cli/src/test/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceExtractCommandTest.java @@ -0,0 +1,63 @@ +package io.kestra.cli.commands.flows.namespaces; + +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.env.Environment; +import io.micronaut.runtime.server.EmbeddedServer; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URL; +import java.util.zip.ZipFile; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringContains.containsString; + +class FlowNamespaceExtractCommandTest { + @Test + void run() throws IOException { + URL directory = FlowNamespaceUpdateCommandTest.class.getClassLoader().getResource("flows"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + + EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); + embeddedServer.start(); + + // we use the update command to add flows to extract + String[] updateArgs = { + "--server", + embeddedServer.getURL().toString(), + "--user", + "myuser:pass:word", + "io.kestra.cli", + directory.getPath(), + + }; + PicocliRunner.call(FlowNamespaceUpdateCommand.class, ctx, updateArgs); + assertThat(out.toString(), containsString("3 flow(s)")); + + // then we extract them + String[] extractArgs = { + "--server", + embeddedServer.getURL().toString(), + "--user", + "myuser:pass:word", + "io.kestra.cli", + "/tmp", + }; + PicocliRunner.call(FlowNamespaceExtractCommand.class, ctx, extractArgs); + File file = new File("/tmp/flows.zip"); + assertThat(file.exists(), is(true)); + ZipFile zipFile = new ZipFile(file); + assertThat(zipFile.stream().count(), is(3L)); + + file.delete(); + } + } +} \ No newline at end of file diff --git a/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommandTest.java new file mode 100644 index 00000000000..9e1b9f6012a --- /dev/null +++ b/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceExtractCommandTest.java @@ -0,0 +1,64 @@ +package io.kestra.cli.commands.templates.namespaces; + +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.env.Environment; +import io.micronaut.runtime.server.EmbeddedServer; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URL; +import java.util.zip.ZipFile; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.StringContains.containsString; +import static org.hamcrest.core.Is.is; + +class TemplateNamespaceExtractCommandTest { + @Test + void run() throws IOException { + URL directory = TemplateNamespaceExtractCommandTest.class.getClassLoader().getResource("templates"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + + EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); + embeddedServer.start(); + + // we use the update command to add templates to extract + String[] args = { + "--server", + embeddedServer.getURL().toString(), + "--user", + "myuser:pass:word", + "io.kestra.tests", + directory.getPath(), + + }; + PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args); + assertThat(out.toString(), containsString("3 template(s)")); + + // then we extract them + String[] extractArgs = { + "--server", + embeddedServer.getURL().toString(), + "--user", + "myuser:pass:word", + "io.kestra.tests", + "/tmp", + }; + PicocliRunner.call(TemplateNamespaceExtractCommand.class, ctx, extractArgs); + File file = new File("/tmp/templates.zip"); + assertThat(file.exists(), is(true)); + ZipFile zipFile = new ZipFile(file); + assertThat(zipFile.stream().count(), is(3L)); + + file.delete(); + } + } + +} \ No newline at end of file diff --git a/core/src/main/java/io/kestra/core/models/templates/Template.java b/core/src/main/java/io/kestra/core/models/templates/Template.java index 573a49e6c83..85f5e7c4c94 100644 --- a/core/src/main/java/io/kestra/core/models/templates/Template.java +++ b/core/src/main/java/io/kestra/core/models/templates/Template.java @@ -1,9 +1,15 @@ package io.kestra.core.models.templates; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.introspect.AnnotatedMember; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; import io.kestra.core.models.DeletedInterface; import io.kestra.core.models.tasks.Task; import io.kestra.core.models.validations.ManualConstraintViolation; +import io.kestra.core.serializers.JacksonMapper; import io.micronaut.core.annotation.Introspected; import lombok.*; import lombok.experimental.SuperBuilder; @@ -25,6 +31,16 @@ @ToString @EqualsAndHashCode public class Template implements DeletedInterface { + + private static final ObjectMapper jsonMapper = JacksonMapper.ofJson().copy() + .setAnnotationIntrospector(new JacksonAnnotationIntrospector() { + @Override + public boolean hasIgnoreMarker(final AnnotatedMember m) { + List exclusions = Arrays.asList("revision", "deleted", "source"); + return exclusions.contains(m.getName()) || super.hasIgnoreMarker(m); + } + }); + @NotNull @NotBlank @Pattern(regexp = "[a-zA-Z0-9._-]+") @@ -93,6 +109,23 @@ public Optional validateUpdate(Template updated) { } } + public String generateSource() { + try { + return JacksonMapper.ofYaml() + .writeValueAsString( + JacksonMapper + .ofJson() + .readTree( + jsonMapper.copy() + .setSerializationInclusion(JsonInclude.Include.NON_DEFAULT) + .writeValueAsString(this) + ) + ); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public Template toDeleted() { return new Template( this.id, diff --git a/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java index 1dafc129cc5..689be4eb01b 100644 --- a/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java +++ b/core/src/main/java/io/kestra/core/repositories/FlowRepositoryInterface.java @@ -55,6 +55,12 @@ ArrayListTotal find( @Nullable Map labels ); + List findWithSource( + @Nullable String query, + @Nullable String namespace, + @Nullable Map labels + ); + ArrayListTotal> findSourceCode(Pageable pageable, @Nullable String query, @Nullable String namespace); List findDistinctNamespace(); diff --git a/core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java b/core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java index c88baff4b10..5731eb30822 100644 --- a/core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java +++ b/core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java @@ -18,6 +18,12 @@ ArrayListTotal