CLI: cache github repository for faster deployments (#622)

langstream-cli/src/main/java/ai/langstream/cli/CLILogger.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.langstream.cli;
+
+public interface CLILogger {
+    void log(Object message);
+
+    void error(Object message);
+
+    boolean isDebugEnabled();
+
+    void debug(Object message);
+
+    class SystemCliLogger implements CLILogger {
+        @Override
+        public void log(Object message) {
+            System.out.println(message);
+        }
+
+        @Override
+        public void error(Object message) {
+            System.err.println(message);
+        }
+
+        @Override
+        public boolean isDebugEnabled() {
+            return true;
+        }
+
+        @Override
+        public void debug(Object message) {
+            log(message);
+        }
+    }
+}

langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java

@@ -19,10 +19,24 @@
 import ai.langstream.cli.commands.RootCmd;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import lombok.SneakyThrows;
 import picocli.CommandLine;
 
 public class LangStreamCLI {
 
+    @SneakyThrows
+    public static Path getLangstreamCLIHomeDirectory() {
+        final String userHome = System.getProperty("user.home");
+        if (!userHome.isBlank() && !"?".equals(userHome)) {
+            final Path langstreamDir = Path.of(userHome, ".langstream");
+            Files.createDirectories(langstreamDir);
+            return langstreamDir;
+        }
+        return null;
+    }
+
     public static void main(String... args) {
         int exitCode = execute(args);
         System.exit(exitCode);

langstream-cli/src/main/java/ai/langstream/cli/commands/BaseCmd.java

@@ -20,10 +20,13 @@
 import ai.langstream.admin.client.AdminClientLogger;
 import ai.langstream.admin.client.HttpRequestFailedException;
 import ai.langstream.admin.client.http.HttpClientFacade;
+import ai.langstream.cli.CLILogger;
+import ai.langstream.cli.LangStreamCLI;
 import ai.langstream.cli.LangStreamCLIConfig;
 import ai.langstream.cli.NamedProfile;
 import ai.langstream.cli.commands.applications.GithubRepositoryDownloader;
 import ai.langstream.cli.commands.profiles.BaseProfileCmd;
+import ai.langstream.cli.util.git.JGitClient;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -49,6 +52,7 @@
 import java.util.Map;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -78,7 +82,46 @@ public enum Formats {
 
     protected static final ObjectWriter jsonBodyWriter = new ObjectMapper().writer();
 
+    @AllArgsConstructor
+    static final class CLILoggerImpl implements CLILogger {
+        private final Supplier<RootCmd> rootCmd;
+        protected Supplier<CommandLine.Model.CommandSpec> command;
+
+        @Override
+        public void log(Object message) {
+            command.get().commandLine().getOut().println(message);
+        }
+
+        @Override
+        public void error(Object message) {
+            if (message == null) {
+                return;
+            }
+            final String error = message.toString();
+            if (error.isBlank()) {
+                return;
+            }
+            System.err.println(command.get().commandLine().getColorScheme().errorText(error));
+        }
+
+        @Override
+        public boolean isDebugEnabled() {
+            return rootCmd.get().isVerbose();
+        }
+
+        @Override
+        public void debug(Object message) {
+            if (isDebugEnabled()) {
+                log(message);
+            }
+        }
+    }
+
     @CommandLine.Spec protected CommandLine.Model.CommandSpec command;
+    private final CLILogger logger = new CLILoggerImpl(() -> getRootCmd(), () -> command);
+    private final GithubRepositoryDownloader githubRepositoryDownloader =
+            new GithubRepositoryDownloader(
+                    new JGitClient(), logger, LangStreamCLI.getLangstreamCLIHomeDirectory());
     private AdminClient client;
     private LangStreamCLIConfig config;
     private Map<String, String> applicationDescriptions = new HashMap<>();
@@ -194,11 +237,9 @@ public void updateConfig(Consumer<LangStreamCLIConfig> consumer) {
 
     @SneakyThrows
     private File computeRootConfigFile() {
-        final String userHome = System.getProperty("user.home");
-        if (!userHome.isBlank() && !"?".equals(userHome)) {
-            final Path langstreamDir = Path.of(userHome, ".langstream");
-            Files.createDirectories(langstreamDir);
-            final Path configFile = langstreamDir.resolve("config");
+        final Path langstreamCLIHomeDirectory = LangStreamCLI.getLangstreamCLIHomeDirectory();
+        if (langstreamCLIHomeDirectory != null) {
+            final Path configFile = langstreamCLIHomeDirectory.resolve("config");
             debug(String.format("Using config file %s", configFile));
             if (!Files.exists(configFile)) {
                 debug(String.format("Init config file %s", configFile));
@@ -248,25 +289,19 @@ private void overrideFromEnv(LangStreamCLIConfig config) {
     }
 
     protected void log(Object log) {
-        command.commandLine().getOut().println(log);
+        logger.log(log);
     }
 
     protected void logNoNewline(Object log) {
         command.commandLine().getOut().print(log);
     }
 
     protected void err(Object log) {
-        final String error = log.toString();
-        if (error.isBlank()) {
-            return;
-        }
-        System.err.println(command.commandLine().getColorScheme().errorText(error));
+        logger.error(log);
     }
 
     protected void debug(Object log) {
-        if (getRootCmd().isVerbose()) {
-            log(log);
-        }
+        logger.debug(log);
     }
 
     @SneakyThrows
@@ -400,7 +435,12 @@ protected File checkFileExistsOrDownload(String path) {
             throw new IllegalArgumentException("http is not supported. Please use https instead.");
         }
         if (path.startsWith("https://")) {
-            return downloadHttpsFile(path, getClient().getHttpClientFacade(), this::log);
+            return downloadHttpsFile(
+                    path,
+                    getClient().getHttpClientFacade(),
+                    logger,
+                    githubRepositoryDownloader,
+                    !getRootCmd().isDisableLocalRepositoriesCache());
         }
         if (path.startsWith("file://")) {
             path = path.substring("file://".length());
@@ -413,11 +453,15 @@ protected File checkFileExistsOrDownload(String path) {
     }
 
     public static File downloadHttpsFile(
-            String path, HttpClientFacade client, Consumer<String> logger)
+            String path,
+            HttpClientFacade client,
+            CLILogger logger,
+            GithubRepositoryDownloader githubRepositoryDownloader,
+            boolean useLocalGithubRepos)
             throws IOException, HttpRequestFailedException {
         final URI uri = URI.create(path);
         if ("github.com".equals(uri.getHost())) {
-            return downloadFromGithub(uri, logger);
+            return githubRepositoryDownloader.downloadGithubRepository(uri, useLocalGithubRepos);
         }
 
         final HttpRequest request =
@@ -442,14 +486,10 @@ public static File downloadHttpsFile(
         }
         Files.write(tempFile, response.body());
         final long time = (System.currentTimeMillis() - start) / 1000;
-        logger.accept(String.format("downloaded remote file %s (%d s)", path, time));
+        logger.log(String.format("downloaded remote file %s (%d s)", path, time));
         return tempFile.toFile();
     }
 
-    private static File downloadFromGithub(URI uri, Consumer<String> logger) {
-        return GithubRepositoryDownloader.downloadGithubRepository(uri, logger);
-    }
-
     @AllArgsConstructor
     @Getter
     private static class Dependency {

langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java

@@ -55,4 +55,12 @@ public class RootCmd {
             description = "Verbose mode. Helpful for troubleshooting.")
     @Getter
     private boolean verbose = false;
+
+    @CommandLine.Option(
+            names = {"--disable-local-repositories-cache"},
+            defaultValue = "false",
+            description =
+                    "By default the repositories downloaded are cached. In case of corrupted directories, you might want to set add this parameter to always clone the repositories from scratch.")
+    @Getter
+    private boolean disableLocalRepositoriesCache = false;
 }