diff --git a/langstream-cli/src/main/java/ai/langstream/cli/CLILogger.java b/langstream-cli/src/main/java/ai/langstream/cli/CLILogger.java new file mode 100644 index 000000000..1f6be1ce7 --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/CLILogger.java @@ -0,0 +1,48 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli; + +public interface CLILogger { + void log(Object message); + + void error(Object message); + + boolean isDebugEnabled(); + + void debug(Object message); + + class SystemCliLogger implements CLILogger { + @Override + public void log(Object message) { + System.out.println(message); + } + + @Override + public void error(Object message) { + System.err.println(message); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(Object message) { + log(message); + } + } +} diff --git a/langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java b/langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java index 5a78229d8..61f461196 100644 --- a/langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java +++ b/langstream-cli/src/main/java/ai/langstream/cli/LangStreamCLI.java @@ -19,10 +19,24 @@ import ai.langstream.cli.commands.RootCmd; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import lombok.SneakyThrows; import picocli.CommandLine; public class LangStreamCLI { + @SneakyThrows + public static Path getLangstreamCLIHomeDirectory() { + final String userHome = System.getProperty("user.home"); + if (!userHome.isBlank() && !"?".equals(userHome)) { + final Path langstreamDir = Path.of(userHome, ".langstream"); + Files.createDirectories(langstreamDir); + return langstreamDir; + } + return null; + } + public static void main(String... args) { int exitCode = execute(args); System.exit(exitCode); diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/BaseCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/BaseCmd.java index 8fc92a98e..f83cbe403 100644 --- a/langstream-cli/src/main/java/ai/langstream/cli/commands/BaseCmd.java +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/BaseCmd.java @@ -20,10 +20,13 @@ import ai.langstream.admin.client.AdminClientLogger; import ai.langstream.admin.client.HttpRequestFailedException; import ai.langstream.admin.client.http.HttpClientFacade; +import ai.langstream.cli.CLILogger; +import ai.langstream.cli.LangStreamCLI; import ai.langstream.cli.LangStreamCLIConfig; import ai.langstream.cli.NamedProfile; import ai.langstream.cli.commands.applications.GithubRepositoryDownloader; import ai.langstream.cli.commands.profiles.BaseProfileCmd; +import ai.langstream.cli.util.git.JGitClient; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -49,6 +52,7 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; @@ -78,7 +82,46 @@ public enum Formats { protected static final ObjectWriter jsonBodyWriter = new ObjectMapper().writer(); + @AllArgsConstructor + static final class CLILoggerImpl implements CLILogger { + private final Supplier rootCmd; + protected Supplier command; + + @Override + public void log(Object message) { + command.get().commandLine().getOut().println(message); + } + + @Override + public void error(Object message) { + if (message == null) { + return; + } + final String error = message.toString(); + if (error.isBlank()) { + return; + } + System.err.println(command.get().commandLine().getColorScheme().errorText(error)); + } + + @Override + public boolean isDebugEnabled() { + return rootCmd.get().isVerbose(); + } + + @Override + public void debug(Object message) { + if (isDebugEnabled()) { + log(message); + } + } + } + @CommandLine.Spec protected CommandLine.Model.CommandSpec command; + private final CLILogger logger = new CLILoggerImpl(() -> getRootCmd(), () -> command); + private final GithubRepositoryDownloader githubRepositoryDownloader = + new GithubRepositoryDownloader( + new JGitClient(), logger, LangStreamCLI.getLangstreamCLIHomeDirectory()); private AdminClient client; private LangStreamCLIConfig config; private Map applicationDescriptions = new HashMap<>(); @@ -194,11 +237,9 @@ public void updateConfig(Consumer consumer) { @SneakyThrows private File computeRootConfigFile() { - final String userHome = System.getProperty("user.home"); - if (!userHome.isBlank() && !"?".equals(userHome)) { - final Path langstreamDir = Path.of(userHome, ".langstream"); - Files.createDirectories(langstreamDir); - final Path configFile = langstreamDir.resolve("config"); + final Path langstreamCLIHomeDirectory = LangStreamCLI.getLangstreamCLIHomeDirectory(); + if (langstreamCLIHomeDirectory != null) { + final Path configFile = langstreamCLIHomeDirectory.resolve("config"); debug(String.format("Using config file %s", configFile)); if (!Files.exists(configFile)) { debug(String.format("Init config file %s", configFile)); @@ -248,7 +289,7 @@ private void overrideFromEnv(LangStreamCLIConfig config) { } protected void log(Object log) { - command.commandLine().getOut().println(log); + logger.log(log); } protected void logNoNewline(Object log) { @@ -256,17 +297,11 @@ protected void logNoNewline(Object log) { } protected void err(Object log) { - final String error = log.toString(); - if (error.isBlank()) { - return; - } - System.err.println(command.commandLine().getColorScheme().errorText(error)); + logger.error(log); } protected void debug(Object log) { - if (getRootCmd().isVerbose()) { - log(log); - } + logger.debug(log); } @SneakyThrows @@ -400,7 +435,12 @@ protected File checkFileExistsOrDownload(String path) { throw new IllegalArgumentException("http is not supported. Please use https instead."); } if (path.startsWith("https://")) { - return downloadHttpsFile(path, getClient().getHttpClientFacade(), this::log); + return downloadHttpsFile( + path, + getClient().getHttpClientFacade(), + logger, + githubRepositoryDownloader, + !getRootCmd().isDisableLocalRepositoriesCache()); } if (path.startsWith("file://")) { path = path.substring("file://".length()); @@ -413,11 +453,15 @@ protected File checkFileExistsOrDownload(String path) { } public static File downloadHttpsFile( - String path, HttpClientFacade client, Consumer logger) + String path, + HttpClientFacade client, + CLILogger logger, + GithubRepositoryDownloader githubRepositoryDownloader, + boolean useLocalGithubRepos) throws IOException, HttpRequestFailedException { final URI uri = URI.create(path); if ("github.com".equals(uri.getHost())) { - return downloadFromGithub(uri, logger); + return githubRepositoryDownloader.downloadGithubRepository(uri, useLocalGithubRepos); } final HttpRequest request = @@ -442,14 +486,10 @@ public static File downloadHttpsFile( } Files.write(tempFile, response.body()); final long time = (System.currentTimeMillis() - start) / 1000; - logger.accept(String.format("downloaded remote file %s (%d s)", path, time)); + logger.log(String.format("downloaded remote file %s (%d s)", path, time)); return tempFile.toFile(); } - private static File downloadFromGithub(URI uri, Consumer logger) { - return GithubRepositoryDownloader.downloadGithubRepository(uri, logger); - } - @AllArgsConstructor @Getter private static class Dependency { diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java index 056fb3ff1..e2725afb9 100644 --- a/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/RootCmd.java @@ -55,4 +55,12 @@ public class RootCmd { description = "Verbose mode. Helpful for troubleshooting.") @Getter private boolean verbose = false; + + @CommandLine.Option( + names = {"--disable-local-repositories-cache"}, + defaultValue = "false", + description = + "By default the repositories downloaded are cached. In case of corrupted directories, you might want to set add this parameter to always clone the repositories from scratch.") + @Getter + private boolean disableLocalRepositoriesCache = false; } diff --git a/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloader.java b/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloader.java index abeee12ca..66ac1ed36 100644 --- a/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloader.java +++ b/langstream-cli/src/main/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloader.java @@ -15,15 +15,20 @@ */ package ai.langstream.cli.commands.applications; +import ai.langstream.cli.CLILogger; +import ai.langstream.cli.util.GitClient; import java.io.File; +import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import java.util.function.Consumer; +import java.util.HashMap; +import java.util.Map; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.SneakyThrows; -import org.eclipse.jgit.api.Git; +@AllArgsConstructor public class GithubRepositoryDownloader { @Data @@ -34,27 +39,121 @@ static class RequestedDirectory { private String directory; } + @Data + static class RepoRef { + private String owner; + private String repository; + private String branch; + + public RepoRef(RequestedDirectory requestedDirectory) { + this.owner = requestedDirectory.getOwner(); + this.repository = requestedDirectory.getRepository(); + this.branch = requestedDirectory.getBranch(); + } + } + + private final GitClient client; + private final CLILogger logger; + private final Path cliHomeDirectory; + + /** + * This cache avoids cloning/updating the same repository multiple times in the same process. + * For example passing secrets and applications from the same repository in the same command. + */ + private final Map clonedUpdatedRepos = new HashMap<>(); + @SneakyThrows - public static File downloadGithubRepository(URI uri, Consumer logger) { - RequestedDirectory requestedDirectory = parseRequest(uri); + public File downloadGithubRepository(URI uri, boolean useLocalGithubRepos) { + final RequestedDirectory requestedDirectory = parseRequest(uri); + Path cloneToDirectory; + + final RepoRef repoRef = new RepoRef(requestedDirectory); + final boolean upToDate = clonedUpdatedRepos.containsKey(repoRef); + if (!upToDate || !useLocalGithubRepos) { + final long start = System.currentTimeMillis(); + if (useLocalGithubRepos && cliHomeDirectory != null) { + try { + cloneToDirectory = + cloneOrUpdateFromGithubReposCache(uri, requestedDirectory, start); + } catch (IOException ioException) { + logger.log( + String.format( + "Failed to update local GitHub repository %s, falling back to cloning to a temporary directory", + uri)); + final Path githubReposPath = resolveGithubReposPath(cliHomeDirectory); + try { + deleteDirectory(githubReposPath); + } catch (IOException e) { + logger.log( + String.format( + "Failed to delete local GitHub repository cache, please remove manually at path: %s, error: %s", + githubReposPath, e.getMessage())); + } + cloneToDirectory = Files.createTempDirectory("langstream"); + cloneRepository(uri, requestedDirectory, cloneToDirectory); + } + } else { + cloneToDirectory = Files.createTempDirectory("langstream"); + cloneRepository(uri, requestedDirectory, cloneToDirectory); + } + clonedUpdatedRepos.put(repoRef, cloneToDirectory); + } else { + cloneToDirectory = clonedUpdatedRepos.get(repoRef); + logger.log(String.format("Using cached GitHub repository %s", cloneToDirectory)); + } + final Path result = cloneToDirectory.resolve(requestedDirectory.getDirectory()); + return result.toFile(); + } - final Path directory = Files.createTempDirectory("langstream"); - logger.accept(String.format("Cloning GitHub repository %s locally", uri)); + private void deleteDirectory(Path githubReposPath) throws IOException { + if (githubReposPath.toFile().isDirectory()) { + final File[] listFiles = githubReposPath.toFile().listFiles(); + for (File listFile : listFiles) { + deleteDirectory(listFile.toPath()); + } + } + Files.deleteIfExists(githubReposPath); + } + private static Path resolveGithubReposPath(Path langstreamCLIHomeDirectory) { + return langstreamCLIHomeDirectory.resolve("ghrepos"); + } + + private Path cloneOrUpdateFromGithubReposCache( + URI uri, RequestedDirectory requestedDirectory, long start) throws IOException { + Path cloneToDirectory; + final Path repos = resolveGithubReposPath(cliHomeDirectory); + cloneToDirectory = + repos.resolve( + Path.of( + requestedDirectory.getOwner(), + requestedDirectory.getRepository(), + requestedDirectory.getBranch())); + if (cloneToDirectory.toFile().exists()) { + logger.log(String.format("Updating local GitHub repository %s", cloneToDirectory)); + final String sha = + client.updateRepository(cloneToDirectory, requestedDirectory.getBranch()); + final long time = (System.currentTimeMillis() - start) / 1000; + logger.log(String.format("Updated local GitHub repository to %s (%d s)", sha, time)); + } else { + Files.createDirectories(cloneToDirectory); + cloneRepository(uri, requestedDirectory, cloneToDirectory); + } + return cloneToDirectory; + } + + private void cloneRepository( + URI uri, RequestedDirectory requestedDirectory, Path cloneToDirectory) + throws IOException { final long start = System.currentTimeMillis(); - Git.cloneRepository() - .setURI( - String.format( - "https://github.com/%s/%s.git", - requestedDirectory.getOwner(), requestedDirectory.getRepository())) - .setDirectory(directory.toFile()) - .setBranch(requestedDirectory.getBranch()) - .setDepth(1) - .call(); + logger.log(String.format("Cloning GitHub repository %s locally", uri)); + final String githubUri = + String.format( + "https://github.com/%s/%s.git", + requestedDirectory.getOwner(), requestedDirectory.getRepository()); + client.cloneRepository(cloneToDirectory, githubUri, requestedDirectory.getBranch()); final long time = (System.currentTimeMillis() - start) / 1000; - logger.accept(String.format("downloaded GitHub directory (%d s)", time)); - final Path result = directory.resolve(requestedDirectory.getDirectory()); - return result.toFile(); + logger.log(String.format("Downloaded GitHub repository (%d s)", time)); } static RequestedDirectory parseRequest(URI uri) { diff --git a/langstream-cli/src/main/java/ai/langstream/cli/util/GitClient.java b/langstream-cli/src/main/java/ai/langstream/cli/util/GitClient.java new file mode 100644 index 000000000..822c402ca --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/util/GitClient.java @@ -0,0 +1,26 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli.util; + +import java.io.IOException; +import java.nio.file.Path; + +public interface GitClient { + + void cloneRepository(Path directory, String uri, String branch) throws IOException; + + String updateRepository(Path directory, String branch) throws IOException; +} diff --git a/langstream-cli/src/main/java/ai/langstream/cli/util/git/JGitClient.java b/langstream-cli/src/main/java/ai/langstream/cli/util/git/JGitClient.java new file mode 100644 index 000000000..96e485727 --- /dev/null +++ b/langstream-cli/src/main/java/ai/langstream/cli/util/git/JGitClient.java @@ -0,0 +1,53 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.langstream.cli.util.git; + +import ai.langstream.cli.util.GitClient; +import java.io.IOException; +import java.nio.file.Path; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.transport.RefSpec; + +public class JGitClient implements GitClient { + + @Override + public void cloneRepository(Path directory, String uri, String branch) throws IOException { + try { + Git.cloneRepository() + .setURI(uri) + .setDirectory(directory.toFile()) + .setBranch(branch) + .setDepth(1) + .call(); + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public String updateRepository(Path directory, String branch) throws IOException { + try (final Git open = Git.open(directory.toFile()); ) { + open.fetch().setRefSpecs(new RefSpec("refs/heads/" + branch)).call(); + open.reset().setMode(ResetCommand.ResetType.HARD).setRef("origin/" + branch).call(); + final RevCommit revCommit = open.log().setMaxCount(1).call().iterator().next(); + return revCommit.getId().abbreviate(8).name(); + } catch (Exception e) { + throw new IOException(e); + } + } +} diff --git a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmdTest.java b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmdTest.java index c1ca86528..0f2a98367 100644 --- a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmdTest.java +++ b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/AbstractDeployApplicationCmdTest.java @@ -21,6 +21,7 @@ import ai.langstream.admin.client.AdminClientConfiguration; import ai.langstream.admin.client.HttpRequestFailedException; import ai.langstream.admin.client.http.HttpClientFacade; +import ai.langstream.cli.CLILogger; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.tomakehurst.wiremock.client.WireMock; @@ -58,12 +59,18 @@ void testRemoteFile() throws Exception { AbstractDeployApplicationCmd.downloadHttpsFile( wireMockBaseUrl + "/my-remote-dir/my-remote-file", client, - log -> System.out.println(log)); + new CLILogger.SystemCliLogger(), + null, + false); assertEquals("content!", Files.readString(file.toPath())); try { AbstractDeployApplicationCmd.downloadHttpsFile( - wireMockBaseUrl + "/unknown", client, log -> System.out.println(log)); + wireMockBaseUrl + "/unknown", + client, + new CLILogger.SystemCliLogger(), + null, + false); fail(); } catch (HttpRequestFailedException ex) { assertEquals(404, ex.getResponse().statusCode()); diff --git a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloaderTest.java b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloaderTest.java index 04f9bdfcf..b808821fd 100644 --- a/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloaderTest.java +++ b/langstream-cli/src/test/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloaderTest.java @@ -17,7 +17,14 @@ import static org.junit.jupiter.api.Assertions.*; +import ai.langstream.cli.CLILogger; +import ai.langstream.cli.util.GitClient; +import java.io.File; +import java.io.IOException; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; class GithubRepositoryDownloaderTest { @@ -56,4 +63,107 @@ void testParseUrlRoot() { assertEquals("main", request.getBranch()); assertNull(request.getDirectory()); } + + @Test + void testDownload() throws Exception { + + AtomicBoolean injectUpdateFailure = new AtomicBoolean(false); + + final Path home = Files.createTempDirectory("test"); + + final GitClient client = + new GitClient() { + @Override + public void cloneRepository(Path directory, String uri, String branch) + throws IOException { + Files.createDirectories(directory.resolve("examples")); + Files.writeString( + directory.resolve("examples").resolve("my-file"), + "content! " + branch); + } + + @Override + public String updateRepository(Path directory, String branch) + throws IOException { + if (injectUpdateFailure.get()) { + throw new IOException("inject failure"); + } + Files.writeString( + directory.resolve("examples").resolve("my-file"), + "content updated! " + branch); + return "xxx"; + } + }; + final GithubRepositoryDownloader downloader = + new GithubRepositoryDownloader(client, new CLILogger.SystemCliLogger(), home); + + File file = + downloader.downloadGithubRepository( + URI.create("https://localhost/LangStream/langstream/tree/main/examples"), + false); + + assertEquals("content! main", Files.readString(file.toPath().resolve("my-file"))); + assertFalse(home.resolve("ghrepos").toFile().exists()); + + file = + downloader.downloadGithubRepository( + URI.create("https://localhost/LangStream/langstream2/tree/main/examples"), + true); + assertEquals("content! main", Files.readString(file.toPath().resolve("my-file"))); + assertTrue(home.resolve("ghrepos").toFile().exists()); + assertEquals( + home.resolve("ghrepos") + .resolve("LangStream") + .resolve("langstream2") + .resolve("main") + .resolve("examples") + .toFile() + .getAbsolutePath(), + file.getAbsolutePath()); + + file = + downloader.downloadGithubRepository( + URI.create("https://localhost/LangStream/langstream2/tree/main/examples"), + true); + assertEquals("content! main", Files.readString(file.toPath().resolve("my-file"))); + assertTrue(home.resolve("ghrepos").toFile().exists()); + assertEquals( + home.resolve("ghrepos") + .resolve("LangStream") + .resolve("langstream2") + .resolve("main") + .resolve("examples") + .toFile() + .getAbsolutePath(), + file.getAbsolutePath()); + + file = + new GithubRepositoryDownloader(client, new CLILogger.SystemCliLogger(), home) + .downloadGithubRepository( + URI.create( + "https://localhost/LangStream/langstream2/tree/main/examples"), + true); + assertEquals("content updated! main", Files.readString(file.toPath().resolve("my-file"))); + assertTrue(home.resolve("ghrepos").toFile().exists()); + assertEquals( + home.resolve("ghrepos") + .resolve("LangStream") + .resolve("langstream2") + .resolve("main") + .resolve("examples") + .toFile() + .getAbsolutePath(), + file.getAbsolutePath()); + + injectUpdateFailure.set(true); + + file = + new GithubRepositoryDownloader(client, new CLILogger.SystemCliLogger(), home) + .downloadGithubRepository( + URI.create( + "https://localhost/LangStream/langstream2/tree/main/examples"), + true); + assertEquals("content! main", Files.readString(file.toPath().resolve("my-file"))); + assertFalse(home.resolve("ghrepos").toFile().exists()); + } }