Add dry-run apps deployment (LangStream#533)

vectorize-io · Oct 6, 2023 · 7979db4 · 7979db4
1 parent f5d5499
commit 7979db4
Show file tree

Hide file tree

Showing 12 changed files with 345 additions and 37 deletions.
diff --git a/langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java b/langstream-admin-client/src/main/java/ai/langstream/admin/client/AdminClient.java
@@ -181,16 +181,22 @@ public Applications applications() {
     }
 
     private class ApplicationsImpl implements Applications {
+        @Override
+        public String deploy(String application, MultiPartBodyPublisher multiPartBodyPublisher) {
+            return deploy(application, multiPartBodyPublisher, false);
+        }
+
         @Override
         @SneakyThrows
-        public void deploy(String application, MultiPartBodyPublisher multiPartBodyPublisher) {
-            final String path = tenantAppPath("/" + application);
+        public String deploy(
+                String application, MultiPartBodyPublisher multiPartBodyPublisher, boolean dryRun) {
+            final String path = tenantAppPath("/" + application) + "?dry-run=" + dryRun;
             final String contentType =
                     String.format(
                             "multipart/form-data; boundary=%s",
                             multiPartBodyPublisher.getBoundary());
             final HttpRequest request = newPost(path, contentType, multiPartBodyPublisher.build());
-            http(request);
+            return http(request).body();
         }
 
         @Override

diff --git a/langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Applications.java b/langstream-admin-client/src/main/java/ai/langstream/admin/client/model/Applications.java
@@ -19,7 +19,10 @@
 import java.net.http.HttpResponse;
 
 public interface Applications {
-    void deploy(String application, MultiPartBodyPublisher multiPartBodyPublisher);
+    String deploy(String application, MultiPartBodyPublisher multiPartBodyPublisher);
+
+    String deploy(
+            String application, MultiPartBodyPublisher multiPartBodyPublisher, boolean dryRun);
 
     void update(String application, MultiPartBodyPublisher multiPartBodyPublisher);
 

diff --git a/...am-api/src/main/java/ai/langstream/api/webservice/application/ApplicationDescription.java b/...am-api/src/main/java/ai/langstream/api/webservice/application/ApplicationDescription.java
@@ -64,7 +64,7 @@ public ApplicationDescription(
     @JsonInclude(JsonInclude.Include.NON_NULL)
     public static class ApplicationDefinition {
 
-        private ApplicationDefinition(Application application) {
+        public ApplicationDefinition(Application application) {
             this.resources = application.getResources();
             this.modules =
                     application.getModules().values().stream().map(ModuleDefinition::new).toList();

diff --git a/...i/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java b/...i/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java
@@ -55,6 +55,17 @@ public static class DeployApplicationCmd extends AbstractDeployApplicationCmd {
                 description = "Secrets file path")
         private String secretFilePath;
 
+        @CommandLine.Option(
+                names = {"--dry-run"},
+                description =
+                        "Dry-run mode. Do not deploy the application but only resolves placeholders and display the result.")
+        private boolean dryRun;
+
+        @CommandLine.Option(
+                names = {"-o"},
+                description = "Output format for dry-run mode.")
+        private Formats format = Formats.raw;
+
         @Override
         String applicationId() {
             return name;
@@ -79,6 +90,16 @@ String secretFilePath() {
         boolean isUpdate() {
             return false;
         }
+
+        @Override
+        boolean isDryRun() {
+            return dryRun;
+        }
+
+        @Override
+        Formats format() {
+            return format;
+        }
     }
 
     @CommandLine.Command(name = "update", header = "Update an existing LangStream application")
@@ -126,6 +147,16 @@ String secretFilePath() {
         boolean isUpdate() {
             return true;
         }
+
+        @Override
+        boolean isDryRun() {
+            return false;
+        }
+
+        @Override
+        Formats format() {
+            return null;
+        }
     }
 
     abstract String applicationId();
@@ -138,6 +169,10 @@ boolean isUpdate() {
 
     abstract boolean isUpdate();
 
+    abstract boolean isDryRun();
+
+    abstract Formats format();
+
     @Override
     @SneakyThrows
     public void run() {
@@ -160,19 +195,15 @@ public void run() {
         final Path tempZip = buildZip(appDirectory, this::log);
 
         long size = Files.size(tempZip);
-        log(String.format("deploying application: %s (%d KB)", applicationId, size / 1024));
-        String secretsContents = null;
-        String instanceContents = null;
 
         final Map<String, Object> contents = new HashMap<>();
         contents.put("app", tempZip);
         if (instanceFile != null) {
             try {
                 contents.put(
                         "instance",
-                        instanceContents =
-                                LocalFileReferenceResolver.resolveFileReferencesInYAMLFile(
-                                        instanceFile.toPath()));
+                        LocalFileReferenceResolver.resolveFileReferencesInYAMLFile(
+                                instanceFile.toPath()));
             } catch (Exception e) {
                 log(
                         "Failed to resolve instance file references. Please double check the file path: "
@@ -185,9 +216,8 @@ public void run() {
             try {
                 contents.put(
                         "secrets",
-                        secretsContents =
-                                LocalFileReferenceResolver.resolveFileReferencesInYAMLFile(
-                                        secretsFile.toPath()));
+                        LocalFileReferenceResolver.resolveFileReferencesInYAMLFile(
+                                secretsFile.toPath()));
             } catch (Exception e) {
                 log(
                         "Failed to resolve secrets file references. Please double check the file path: "
@@ -199,11 +229,27 @@ public void run() {
         final MultiPartBodyPublisher bodyPublisher = buildMultipartContentForAppZip(contents);
 
         if (isUpdate()) {
+            log(String.format("updating application: %s (%d KB)", applicationId, size / 1024));
             getClient().applications().update(applicationId, bodyPublisher);
             log(String.format("application %s updated", applicationId));
         } else {
-            getClient().applications().deploy(applicationId, bodyPublisher);
-            log(String.format("application %s deployed", applicationId));
+            final boolean dryRun = isDryRun();
+            if (dryRun) {
+                log(
+                        String.format(
+                                "resolving application: %s. Dry run mode is enabled, the application will NOT be deployed",
+                                applicationId));
+            } else {
+                log(String.format("deploying application: %s (%d KB)", applicationId, size / 1024));
+            }
+            final String response =
+                    getClient().applications().deploy(applicationId, bodyPublisher, dryRun);
+            if (dryRun) {
+                final Formats format = format();
+                print(format == Formats.raw ? Formats.yaml : format, response, null, null);
+            } else {
+                log(String.format("application %s deployed", applicationId));
+            }
         }
     }
 

diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/docker/LocalRunApplicationCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/docker/LocalRunApplicationCmd.java
@@ -102,6 +102,12 @@ public class LocalRunApplicationCmd extends BaseDockerCmd {
             description = "Docker image of the LangStream runtime to use")
     private String dockerImageName;
 
+    @CommandLine.Option(
+            names = {"--dry-run"},
+            description =
+                    "Dry-run mode. Do not deploy the application but only resolves placeholders and display the result.")
+    private boolean dryRun;
+
     @Override
     @SneakyThrows
     public void run() {
@@ -120,6 +126,10 @@ public void run() {
                 dockerImageName = "ghcr.io/langstream/langstream-runtime-tester";
             }
         }
+        startBroker = !dryRun && startBroker;
+        startDatabase = !dryRun && startDatabase;
+        startS3 = !dryRun && startS3;
+        startWebservices = !dryRun && startWebservices;
 
         final File appDirectory = checkFileExistsOrDownload(appPath);
         final File instanceFile;
@@ -170,7 +180,7 @@ public void run() {
                 throw e;
             }
         } else {
-            if (startBroker) {
+            if (startBroker || dryRun) {
                 instanceContents =
                         "instance:\n"
                                 + "  streamingCluster:\n"
@@ -211,7 +221,8 @@ public void run() {
                 startBroker,
                 startS3,
                 startWebservices,
-                startDatabase);
+                startDatabase,
+                dryRun);
     }
 
     private void executeOnDocker(
@@ -224,7 +235,8 @@ private void executeOnDocker(
             boolean startBroker,
             boolean startS3,
             boolean startWebservices,
-            boolean startDatabase)
+            boolean startDatabase,
+            boolean dryRun)
             throws Exception {
         File tmpInstanceFile = Files.createTempFile("instance", ".yaml").toFile();
         Files.write(tmpInstanceFile.toPath(), instanceContents.getBytes(StandardCharsets.UTF_8));
@@ -251,6 +263,8 @@ private void executeOnDocker(
         commandLine.add("LANSGSTREAM_TESTER_APPLICATIONID=" + applicationId);
         commandLine.add("-e");
         commandLine.add("LANSGSTREAM_TESTER_STARTWEBSERVICES=" + startWebservices);
+        commandLine.add("-e");
+        commandLine.add("LANSGSTREAM_TESTER_DRYRUN=" + dryRun);
 
         if (singleAgentId != null && !singleAgentId.isEmpty()) {
             commandLine.add("-e");

diff --git a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java
@@ -48,7 +48,7 @@ public void testDeploy() throws Exception {
                 AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println);
 
         wireMock.register(
-                WireMock.post(String.format("/api/applications/%s/my-app", TENANT))
+                WireMock.post(String.format("/api/applications/%s/my-app?dry-run=false", TENANT))
                         .withMultipartRequestBody(
                                 aMultipart("app")
                                         .withBody(binaryEqualTo(Files.readAllBytes(zipFile))))
@@ -108,7 +108,7 @@ public void testDeployWithDependencies() throws Exception {
         final Path zipFile =
                 AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println);
         wireMock.register(
-                WireMock.post(String.format("/api/applications/%s/my-app", TENANT))
+                WireMock.post(String.format("/api/applications/%s/my-app?dry-run=false", TENANT))
                         .withMultipartRequestBody(
                                 aMultipart("app")
                                         .withBody(binaryEqualTo(Files.readAllBytes(zipFile))))
@@ -168,6 +168,62 @@ public void testUpdateAll() throws Exception {
         Assertions.assertEquals("", result.err());
     }
 
+    @Test
+    public void testDeployDryRun() throws Exception {
+        Path langstream = Files.createTempDirectory("langstream");
+        final String app = createTempFile("module: module-1", langstream);
+        final String instance = createTempFile("instance: {}");
+        final String secrets = createTempFile("secrets: []");
+
+        final Path zipFile =
+                AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println);
+
+        wireMock.register(
+                WireMock.post(String.format("/api/applications/%s/my-app?dry-run=true", TENANT))
+                        .withMultipartRequestBody(
+                                aMultipart("app")
+                                        .withBody(binaryEqualTo(Files.readAllBytes(zipFile))))
+                        .withMultipartRequestBody(
+                                aMultipart("instance").withBody(equalTo("instance: {}")))
+                        .withMultipartRequestBody(
+                                aMultipart("secrets").withBody(equalTo("secrets: []")))
+                        .willReturn(WireMock.ok("{ \"name\": \"my-app\" }")));
+
+        CommandResult result =
+                executeCommand(
+                        "apps",
+                        "deploy",
+                        "my-app",
+                        "-s",
+                        secrets,
+                        "-app",
+                        langstream.toAbsolutePath().toString(),
+                        "-i",
+                        instance,
+                        "--dry-run");
+        Assertions.assertEquals(0, result.exitCode());
+        Assertions.assertEquals("", result.err());
+        Assertions.assertTrue(result.out().contains("name: \"my-app\""));
+
+        result =
+                executeCommand(
+                        "apps",
+                        "deploy",
+                        "my-app",
+                        "-s",
+                        secrets,
+                        "-app",
+                        langstream.toAbsolutePath().toString(),
+                        "-i",
+                        instance,
+                        "--dry-run",
+                        "-o",
+                        "json");
+        Assertions.assertEquals(0, result.exitCode());
+        Assertions.assertEquals("", result.err());
+        Assertions.assertTrue(result.out().contains("{\n" + "  \"name\" : \"my-app\"\n" + "}"));
+    }
+
     @Test
     public void testUpdateInstance() throws Exception {
         final String instance = createTempFile("instance: {}");
@@ -399,7 +455,7 @@ public void testDeployWithFilePlaceholders() throws Exception {
                 AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println);
 
         wireMock.register(
-                WireMock.post(String.format("/api/applications/%s/my-app", TENANT))
+                WireMock.post(String.format("/api/applications/%s/my-app?dry-run=false", TENANT))
                         .withMultipartRequestBody(
                                 aMultipart("app")
                                         .withBody(binaryEqualTo(Files.readAllBytes(zipFile))))

diff --git a/langstream-runtime/langstream-runtime-tester/src/main/assemble/entrypoint.sh b/langstream-runtime/langstream-runtime-tester/src/main/assemble/entrypoint.sh
@@ -33,4 +33,4 @@ if [ "$START_HERDDB" = "true" ]; then
   /herddb/herddb/bin/service server start
 fi
 
-exec java ${JAVA_OPTS} -Dlogging.config=/app/logback.xml -Djdk.lang.Process.launchMechanism=vfork -cp "/app/lib/*:/app/tester/lib/*" "ai.langstream.runtime.tester.Main"
+exec java ${JAVA_OPTS} -D -Dlogback.configurationFile=/app/logback.xml -Djdk.lang.Process.launchMechanism=vfork -cp "/app/lib/*" "ai.langstream.runtime.tester.Main"
diff --git a/langstream-runtime/langstream-runtime-tester/src/main/docker/Dockerfile b/langstream-runtime/langstream-runtime-tester/src/main/docker/Dockerfile
@@ -41,8 +41,9 @@ RUN chmod -R g+w /kafka \
        && chown 10000:0 -R /herddb
 
 # Add the runtime code at the end. This optimizes docker layers to not depend on artifacts-specific changes.
-ADD maven/lib /app/tester/lib
-RUN rm -f /app/tester/lib/*netty*
+RUN rm -rf /app/lib
+ADD maven/lib /app/lib
+RUN rm -f /app/lib/*netty*
 ADD maven/entrypoint.sh /app/entrypoint.sh
 ADD maven/logback.xml /app/logback.xml