[cli] Allow to refer to local files in secrets.yaml and in instance.y…

…aml using <file:xxx> syntax (LangStream#341)

examples/secrets/secrets.yaml

@@ -32,7 +32,7 @@ secrets:
     data:
       url: https://us-central1-aiplatform.googleapis.com
       token: xxx
-      serviceAccountJson: xxx
+      # serviceAccountJson: "<file:service-account.json>"
       region: us-central1
       project: myproject
   - name: hugging-face
@@ -44,7 +44,7 @@ secrets:
     data:
       username: xxx
       password: xxx
-      secureBundle: xxx
+      # secureBundle: "<file:secure-connect-bundle.zip>"
   - name: s3
     id: s3
     data:

langstream-cli/src/main/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmd.java

@@ -15,6 +15,8 @@
  */
 package ai.langstream.cli.commands.applications;
 
+import static ai.langstream.impl.parser.ModelBuilder.resolveFileReferencesInYAMLFile;
+
 import ai.langstream.admin.client.util.MultiPartBodyPublisher;
 import ai.langstream.api.model.Application;
 import ai.langstream.api.model.Dependency;
@@ -179,10 +181,10 @@ public void run() {
         final Map<String, Object> contents = new HashMap<>();
         contents.put("app", tempZip);
         if (instanceFile != null) {
-            contents.put("instance", Files.readString(instanceFile.toPath()));
+            contents.put("instance", resolveFileReferencesInYAMLFile(instanceFile.toPath()));
         }
         if (secretsFile != null) {
-            contents.put("secrets", Files.readString(secretsFile.toPath()));
+            contents.put("secrets", resolveFileReferencesInYAMLFile(secretsFile.toPath()));
         }
         final MultiPartBodyPublisher bodyPublisher = buildMultipartContentForAppZip(contents);
 

langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AppsCmdTest.java

@@ -19,6 +19,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.binaryEqualTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -29,6 +30,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -1161,4 +1163,70 @@ public void testDownloadToFile() {
         Assertions.assertEquals("", result.err());
         Assertions.assertEquals("Downloaded application code to /tmp/download.zip", result.out());
     }
+
+    @Test
+    public void testDeployWithFilePlaceholders() throws Exception {
+        Path langstream = Files.createTempDirectory("langstream");
+        final String instance = createTempFile("instance: {}");
+        final String jsonFileRelative =
+                Paths.get(createTempFile("{\"client-id\":\"xxx\"}")).getFileName().toString();
+        assertFalse(jsonFileRelative.contains("/"));
+
+        final String secrets =
+                createTempFile(
+                        """
+                                             secrets:
+                                                  - name: vertex-ai
+                                                    id: vertex-ai
+                                                    data:
+                                                      url: https://us-central1-aiplatform.googleapis.com
+                                                      token: xxx
+                                                      serviceAccountJson: "<file:%s>"
+                                                      region: us-central1
+                                                      project: myproject
+                                             """
+                                .formatted(jsonFileRelative));
+
+        final Path zipFile =
+                AbstractDeployApplicationCmd.buildZip(langstream.toFile(), System.out::println);
+
+        wireMock.register(
+                WireMock.post("/api/applications/%s/my-app".formatted(TENANT))
+                        .withMultipartRequestBody(
+                                aMultipart("app")
+                                        .withBody(binaryEqualTo(Files.readAllBytes(zipFile))))
+                        .withMultipartRequestBody(
+                                aMultipart("instance").withBody(equalTo("instance: {}")))
+                        .withMultipartRequestBody(
+                                aMultipart("secrets")
+                                        .withBody(
+                                                equalTo(
+                                                        """
+                                       ---
+                                       secrets:
+                                       - name: "vertex-ai"
+                                         id: "vertex-ai"
+                                         data:
+                                           url: "https://us-central1-aiplatform.googleapis.com"
+                                           token: "xxx"
+                                           serviceAccountJson: "{\\"client-id\\":\\"xxx\\"}"
+                                           region: "us-central1"
+                                           project: "myproject"
+                                        """)))
+                        .willReturn(WireMock.ok("{ \"name\": \"my-app\" }")));
+
+        CommandResult result =
+                executeCommand(
+                        "apps",
+                        "deploy",
+                        "my-app",
+                        "-s",
+                        secrets,
+                        "-app",
+                        langstream.toAbsolutePath().toString(),
+                        "-i",
+                        instance);
+        Assertions.assertEquals(0, result.exitCode());
+        Assertions.assertEquals("", result.err());
+    }
 }

langstream-core/src/main/java/ai/langstream/impl/parser/ModelBuilder.java

@@ -53,6 +53,8 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -109,7 +111,7 @@ public static ApplicationWithPackageInfo buildApplicationInstance(
                                 if (existingSamePipelineName != null) {
                                     throw new IllegalArgumentException(
                                             "Duplicate pipeline file names in the application: "
-                                                    + path.getFileName().toString());
+                                                    + path.getFileName());
                                 }
                             } catch (java.nio.charset.MalformedInputException e) {
                                 log.warn("Skipping file {} due to encoding error", path);
@@ -548,4 +550,112 @@ static ErrorsSpec validateErrorsSpec(ErrorsSpec errorsSpec) {
         }
         return errorsSpec;
     }
+
+    private static final Pattern patternPlaceholder = Pattern.compile("<file:(.*?)>");
+
+    public static String resolveFileReferencesInYAMLFile(Path file) throws Exception {
+        return resolveFileReferencesInYAMLFile(
+                Files.readString(file, StandardCharsets.UTF_8),
+                filename -> {
+                    Path fileToRead = file.getParent().resolve(filename);
+                    try {
+                        String fileNameString = filename.toString();
+                        if (isTextFile(fileNameString)) {
+                            log.info("Reading text file {}", fileToRead);
+                            return Files.readString(fileToRead, StandardCharsets.UTF_8);
+                        } else {
+                            byte[] content = Files.readAllBytes(fileToRead);
+                            log.info(
+                                    "Reading binary file {} and encoding it using base64 encoding",
+                                    fileToRead);
+                            return "base64:"
+                                    + java.util.Base64.getEncoder().encodeToString(content);
+                        }
+                    } catch (IOException error) {
+                        throw new IllegalArgumentException("Cannot read file " + fileToRead, error);
+                    }
+                });
+    }
+
+    private static final Set<String> TEXT_FILES_EXTENSIONS =
+            Set.of("txt", "yaml", "yml", "json", "text");
+
+    private static boolean isTextFile(String fileNameString) {
+        if (fileNameString == null) {
+            return false;
+        }
+        String lowerCase = fileNameString.toLowerCase();
+        for (String ext : TEXT_FILES_EXTENSIONS) {
+            if (lowerCase.endsWith(ext)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static String resolveFileReferencesInYAMLFile(
+            String content, Function<String, String> readFileContents) throws Exception {
+        // first of all, we read te file with a generic YAML parser
+        // in order to fail fast if the file is not valid YAML
+        Map<String, Object> map = mapper.readValue(content, Map.class);
+
+        if (!patternPlaceholder.matcher(content).find()) {
+            return content;
+        }
+
+        resolveFileReferencesInMap(map, readFileContents);
+        return mapper.writeValueAsString(map);
+    }
+
+    private static void resolveFileReferencesInList(
+            List<Object> list, Function<String, String> readFileContents) throws Exception {
+        for (int i = 0; i < list.size(); i++) {
+            Object value = list.get(i);
+            if (value instanceof String string) {
+                String newValue = resolveFileReferences(string, readFileContents);
+                list.set(i, newValue);
+            } else if (value instanceof Map mapChild) {
+                resolveFileReferencesInMap(mapChild, readFileContents);
+            } else if (value instanceof List listChild) {
+                resolveFileReferencesInList(listChild, readFileContents);
+            }
+        }
+    }
+
+    private static void resolveFileReferencesInMap(
+            Map<String, Object> map, Function<String, String> readFileContents) throws Exception {
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            if (entry.getValue() instanceof String string) {
+                String newValue = resolveFileReferences(string, readFileContents);
+                entry.setValue(newValue);
+            } else if (entry.getValue() instanceof Map mapChild) {
+                resolveFileReferencesInMap(mapChild, readFileContents);
+            } else if (entry.getValue() instanceof List listChild) {
+                resolveFileReferencesInList(listChild, readFileContents);
+            }
+        }
+    }
+
+    /**
+     * Resolve references to local files in a text (YAML) file. References are always relative to
+     * the directory that contains the file.
+     *
+     * @param content contents of the YAML file
+     * @return the new content of the file
+     */
+    private static String resolveFileReferences(
+            String content, Function<String, String> readFileContents) throws Exception {
+
+        Matcher matcher = patternPlaceholder.matcher(content);
+        StringBuffer buffer = new StringBuffer();
+
+        while (matcher.find()) {
+            String filename = matcher.group(1);
+            String replacement = readFileContents.apply(filename);
+            matcher.appendReplacement(buffer, replacement);
+        }
+        matcher.appendTail(buffer);
+
+        return buffer.toString();
+    }
 }