feat(core): import flows and templates (#1000)

Co-authored-by: Yann C <yann.coornaert62@orange.fr>

ui/src/components/flows/Flows.vue

@@ -113,14 +113,30 @@
                     </li>
                 </ul>
                 <li class="spacer" />
+                <li>
+                    <div class="el-input el-input-file custom-upload">
+                        <div class="el-input__wrapper">
+                            <label for="importFlows">
+                                <Upload />
+                                {{ $t('import') }}
+                            </label>
+                            <input
+                                id="importFlows"
+                                class="el-input__inner"
+                                type="file"
+                                @change="importFlows()"
+                                ref="file"
+                            >
+                        </div>
+                    </div>
+                </li>
                 <li>
                     <router-link :to="{name: 'flows/search'}">
                         <el-button :icon="TextBoxSearch">
                             {{ $t('source search') }}
                         </el-button>
                     </router-link>
                 </li>
-
                 <li v-if="user && user.hasAnyAction(permission.FLOW, action.CREATE)">
                     <router-link :to="{name: 'flows/create'}">
                         <el-button :icon="Plus" type="primary">
@@ -159,7 +175,7 @@
     import Kicon from "../Kicon.vue"
     import Labels from "../layout/Labels.vue"
     import BottomLineCounter from "../layout/BottomLineCounter.vue";
-
+    import Upload from "vue-material-design-icons/Upload.vue";
     export default {
         mixins: [RouteContext, RestoreUrl, DataTableActions],
         components: {
@@ -175,6 +191,7 @@
             Kicon,
             Labels,
             BottomLineCounter,
+            Upload
         },
         data() {
             return {
@@ -184,7 +201,8 @@
                 dailyGroupByFlowReady: false,
                 dailyReady: false,
                 flowsSelection: [],
-                queryBulkAction: false
+                queryBulkAction: false,
+                file: undefined,
             };
         },
         computed: {
@@ -249,6 +267,16 @@
                     () => {}
                 )
             },
+            importFlows() {
+                const formData = new FormData();
+                formData.append("fileUpload", this.$refs.file.files[0]);
+                this.$store
+                    .dispatch("flow/importFlows", formData)
+                    .then(_ => {
+                        this.$toast().success(this.$t("flows imported"));
+                        this.loadData(() => {})
+                    })
+            },
             chartData(row) {
                 if (this.dailyGroupByFlow && this.dailyGroupByFlow[row.namespace] && this.dailyGroupByFlow[row.namespace][row.id]) {
                     return this.dailyGroupByFlow[row.namespace][row.id];

ui/src/components/templates/Templates.vue

@@ -74,6 +74,23 @@
                 </ul>
 
                 <li class="spacer" />
+                <li>
+                    <div class="el-input el-input-file custom-upload">
+                        <div class="el-input__wrapper">
+                            <label for="importFlows">
+                                <Upload />
+                                {{ $t('import') }}
+                            </label>
+                            <input
+                                id="importFlows"
+                                class="el-input__inner"
+                                type="file"
+                                @change="importTemplates()"
+                                ref="file"
+                            >
+                        </div>
+                    </div>
+                </li>
                 <li>
                     <router-link :to="{name: 'templates/create'}">
                         <el-button :icon="Plus" type="primary">
@@ -107,6 +124,7 @@
     import _merge from "lodash/merge";
     import MarkdownTooltip from "../../components/layout/MarkdownTooltip.vue";
     import BottomLineCounter from "../layout/BottomLineCounter.vue";
+    import Upload from "vue-material-design-icons/Upload.vue";
 
     export default {
         mixins: [RouteContext, RestoreUrl, DataTableActions],
@@ -119,6 +137,7 @@
             Kicon,
             MarkdownTooltip,
             BottomLineCounter,
+            Upload
         },
         data() {
             return {
@@ -199,6 +218,17 @@
                     () => {}
                 )
             },
+            importTemplates() {
+                const formData = new FormData();
+                formData.append("fileUpload", this.$refs.file.files[0]);
+                this.$store
+                    .dispatch("template/importTemplates", formData)
+                    .then(_ => {
+                        this.$toast().success(this.$t("templates imported"));
+                        this.loadData(() => {})
+                    })
+            },
         },
     };
 </script>
+

ui/src/stores/flow.js

@@ -150,6 +150,9 @@ export default {
                     Utils.downloadUrl(response.request.responseURL, "flows.zip");
                 });
         },
+        importFlows(_, options) {
+            return this.$http.post("/api/v1/flows/import", options, {headers: {"Content-Type": "multipart/form-data"}})
+        }
     },
     mutations: {
         setFlows(state, flows) {

ui/src/stores/template.js

@@ -73,6 +73,9 @@ export default {
                     Utils.downloadUrl(response.request.responseURL, "templates.zip");
                 });
         },
+        importTemplates(_, options) {
+            return this.$http.post("/api/v1/templates/import", options, {headers: {"Content-Type": "multipart/form-data"}})
+        }
     },
     mutations: {
         setTemplates(state, templates) {

ui/src/styles/layout/element-plus-overload.scss

@@ -52,6 +52,43 @@
         }
     }
 }
+
+
+.el-input-file.custom-upload {
+    border-color: var(--el-button-border-color);
+    font-size: var(--el-font-size-base);
+    border-radius: var(--el-border-radius-base);
+    color: var(--el-button-text-color);
+
+    .el-input__wrapper {
+        background-color: transparent;
+    }
+
+    label {
+        display: flex;
+        cursor: pointer;
+        margin-left: 10px;
+        line-height: 30px;
+
+        .material-design-icon {
+            margin-right: 6px;
+        }
+    }
+
+    input[type="file"] {
+        display: none;
+    }
+
+    ::-webkit-file-upload-button {
+        display: none;
+    }
+
+    ::file-selector-button {
+        display: none;
+    }
+}
+
+
 .el-select {
     .el-tag {
         color: var(--el-select-input-color);

ui/src/translations.json

@@ -275,7 +275,8 @@
     "template export": "Are you sure you want to export <code>{templateCount}</code> template(s)?",
     "templates exported": "Templates exported",
     "flow export": "Are you sure you want to export <code>{flowCount}</code> flow(s)?",
-    "flows exported": "Flows exported"
+    "flows exported": "Flows exported",
+    "import": "Import"
   },
   "fr": {
     "id": "Identifiant",
@@ -554,7 +555,8 @@
     "template export": "Êtes vous sûr de vouloir exporter <code>{templateCount}</code> template(s)?",
     "templates exported": "Templates exportés",
     "flow export": "Êtes vous sûr de vouloir exporter <code>{flowCount}</code> flow(s)?",
-    "flows exported": "Flows exportés"
+    "flows exported": "Flows exportés",
+    "import": "Importer"
   },
   "de": {
     "id": "Id",
@@ -832,6 +834,7 @@
     "template export": "Sind Sie sicher, dass Sie <code>{templateCount}</code> Template(n) exportieren möchten?",
     "templates exported": "Template exportiert",
     "flow export": "Sind Sie sicher, dass Sie <code>{flowCount}</code> Flows exportieren möchten?",
-    "flows exported": "Flows exportiert"
+    "flows exported": "Flows exportiert",
+    "import": "Importieren"
   }
 }

webserver/src/main/java/io/kestra/webserver/controllers/FlowController.java

@@ -27,6 +27,7 @@
 import io.micronaut.http.MediaType;
 import io.micronaut.http.annotation.*;
 import io.micronaut.http.exceptions.HttpStatusException;
+import io.micronaut.http.multipart.CompletedFileUpload;
 import io.micronaut.scheduling.TaskExecutors;
 import io.micronaut.scheduling.annotation.ExecuteOn;
 import io.micronaut.validation.Validated;
@@ -36,6 +37,8 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import jakarta.inject.Inject;
 
+import javax.validation.ConstraintViolationException;
+import javax.validation.Valid;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.*;
@@ -44,9 +47,8 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
-import javax.validation.ConstraintViolationException;
-import javax.validation.Valid;
 
 import static io.kestra.core.utils.Rethrow.throwFunction;
 
@@ -496,4 +498,51 @@ private static byte[] zipFlows(List<FlowWithSource> flows) throws IOException {
             return bos.toByteArray();
         }
     }
+
+    @ExecuteOn(TaskExecutors.IO)
+    @Post(uri = "/import", consumes = MediaType.MULTIPART_FORM_DATA)
+    @Operation(
+        tags = {"Flows"},
+        summary = "Import flows as a ZIP archive of yaml sources or a multi-objects YAML file."
+    )
+    @ApiResponse(responseCode = "204", description = "On success")
+    public HttpResponse<Void> importFlows(
+        @Parameter(description = "The file to import, can be a ZIP archive or a multi-objects YAML file")
+        @Part CompletedFileUpload fileUpload
+    ) throws IOException {
+        String fileName = fileUpload.getFilename().toLowerCase();
+        if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) {
+            List<String> sources = List.of(new String(fileUpload.getBytes()).split("---"));
+            for (String source : sources) {
+                Flow parsed = new YamlFlowParser().parse(source, Flow.class);
+                importFlow(source, parsed);
+            }
+        } else if (fileName.endsWith(".zip")) {
+            try (ZipInputStream archive = new ZipInputStream(fileUpload.getInputStream())) {
+                ZipEntry entry;
+                while ((entry = archive.getNextEntry()) != null) {
+                    if (entry.isDirectory() || !entry.getName().endsWith(".yml") && !entry.getName().endsWith(".yaml")) {
+                        continue;
+                    }
+
+                    String source = new String(archive.readAllBytes());
+                    Flow parsed = new YamlFlowParser().parse(source, Flow.class);
+                    importFlow(source, parsed);
+                }
+            }
+        } else {
+            throw new IllegalArgumentException("Cannot import file of type " + fileName.substring(fileName.lastIndexOf('.')));
+        }
+
+        return HttpResponse.status(HttpStatus.NO_CONTENT);
+    }
+
+    protected void importFlow(String source, Flow parsed) {
+        flowRepository
+            .findById(parsed.getNamespace(), parsed.getId())
+            .ifPresentOrElse(
+                previous -> flowRepository.update(parsed, previous, source, taskDefaultService.injectDefaults(parsed)),
+                () -> flowRepository.create(parsed, source, taskDefaultService.injectDefaults(parsed))
+            );
+    }
 }

webserver/src/main/java/io/kestra/webserver/controllers/TemplateController.java

@@ -15,6 +15,7 @@
 import io.micronaut.http.MediaType;
 import io.micronaut.http.annotation.*;
 import io.micronaut.http.exceptions.HttpStatusException;
+import io.micronaut.http.multipart.CompletedFileUpload;
 import io.micronaut.scheduling.TaskExecutors;
 import io.micronaut.scheduling.annotation.ExecuteOn;
 import io.micronaut.validation.Validated;
@@ -23,16 +24,17 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import jakarta.inject.Inject;
 
+import javax.validation.ConstraintViolationException;
+import javax.validation.Valid;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
-import javax.validation.ConstraintViolationException;
-import javax.validation.Valid;
 
 @Validated
 @Controller("/api/v1/templates")
@@ -200,7 +202,7 @@ private List<Template> updateCompleteNamespace(String namespace, List<Template>
                 .stream()
                 .filter(template -> !ids.contains(template.getId()))
                 .peek(template -> templateRepository.delete(template))
-                .collect(Collectors.toList());;
+                .collect(Collectors.toList());
         }
 
         // update or create templates
@@ -291,4 +293,45 @@ private static byte[] zipTemplates(List<Template> templates) throws IOException
             return bos.toByteArray();
         }
     }
+
+    @ExecuteOn(TaskExecutors.IO)
+    @Post(uri = "/import", consumes = MediaType.MULTIPART_FORM_DATA)
+    @Operation(
+        summary = "Import templates as a ZIP archive of yaml sources or a multi-objects YAML file."
+    )
+    @ApiResponse(responseCode = "204", description = "On success")
+    public HttpResponse<Void> importTemplates(@Parameter(description = "The file to import, can be a ZIP archive or a multi-objects YAML file") @Part CompletedFileUpload fileUpload) throws IOException {
+        String fileName = fileUpload.getFilename().toLowerCase();
+        if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) {
+            List<String> sources = List.of(new String(fileUpload.getBytes()).split("---"));
+            for (String source : sources) {
+                Template parsed = new YamlFlowParser().parse(source, Template.class);
+                importTemplate(parsed);
+            }
+        } else if (fileName.endsWith(".zip")) {
+            try (ZipInputStream archive = new ZipInputStream(fileUpload.getInputStream())) {
+                ZipEntry entry;
+                while ((entry = archive.getNextEntry()) != null) {
+                    if (entry.isDirectory() || !entry.getName().endsWith(".yml") && !entry.getName().endsWith(".yaml")) {
+                        continue;
+                    }
+
+                    String source = new String(archive.readAllBytes());
+                    Template parsed = new YamlFlowParser().parse(source, Template.class);
+                    importTemplate(parsed);
+                }
+            }
+        } else {
+            throw new IllegalArgumentException("Cannot import file of type " + fileName.substring(fileName.lastIndexOf('.')));
+        }
+
+        return HttpResponse.status(HttpStatus.NO_CONTENT);
+    }
+
+    protected void importTemplate(Template parsed) {
+        templateRepository.findById(parsed.getNamespace(), parsed.getId()).ifPresentOrElse(
+            previous -> templateRepository.update(parsed, previous),
+            () -> templateRepository.create(parsed)
+        );
+    }
 }