[cli] Print a log message before starting to clone a git repo

.github/workflows/ci.yml

@@ -155,13 +155,13 @@ jobs:
           uname -m
           ./dev/prepare-minikube-for-e2e-tests.sh
           ./mvnw install -pl langstream-e2e-tests -am -DskipTests
-          ./mvnw verify -pl langstream-e2e-tests -De2eTests -DexcludedGroups="needs-credentials"
+          ./mvnw verify -pl langstream-e2e-tests -De2eTests -DexcludedGroups="needs-credentials" -Dlangstream.tests.recycleenv=true
 
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v3
         if: failure()
         continue-on-error: true
         with:
-          name: test-logs-${{ matrix.group }}
+          name: test-logs-e2e
           path: "**/target/e2e-test-logs/*"
           retention-days: 7

examples/applications/webcrawler-source/chatbot.yaml

@@ -67,7 +67,7 @@ pipeline:
       messages:
         - role: system
           content: |
-              An user is going to perform a questions, he documents below may help you in answering to their questions.
+              An user is going to perform a questions, The documents below may help you in answering to their questions.
               Please try to leverage them in your answer as much as possible.
               Take into consideration that the user is always asking questions about the LangStream project.
               If you provide code or YAML snippets, please explicitly state that they are examples.

examples/applications/webcrawler-source/crawler.yaml

@@ -31,7 +31,7 @@ pipeline:
       reindex-interval-seconds: 3600
       max-error-count: 5
       max-urls: 1000
-      max-depth: 10
+      max-depth: 50
       handle-robots-file: true
       user-agent: "" # this is computed automatically, but you can override it
       scan-html-documents: true
@@ -53,7 +53,7 @@ pipeline:
   - name: "Detect language"
     type: "language-detector"
     configuration:
-      allowedLanguages: ["en"]
+      allowedLanguages: ["en", "fr"]
       property: "language"
   - name: "Split into chunks"
     type: "text-splitter"

langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/datasource/CassandraDataSource.java

@@ -74,7 +74,9 @@ protected TypeCodec<?> createCodec(
 
     @Override
     public void initialize(Map<String, Object> dataSourceConfig) {
-        log.info("Initializing AstraDBDataSource with config {}", dataSourceConfig);
+        log.info(
+                "Initializing CassandraDataSource with config {}",
+                ConfigurationUtils.redactSecrets(dataSourceConfig));
         this.astraToken = ConfigurationUtils.getString("token", "", dataSourceConfig);
         this.astraEnvironment =
                 ConfigurationUtils.getString("environment", "PROD", dataSourceConfig);

langstream-api/src/main/java/ai/langstream/api/util/ConfigurationUtils.java

@@ -15,8 +15,10 @@
  */
 package ai.langstream.api.util;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -252,4 +254,49 @@ public static void requiredListField(
                     "Expecting a list in the field '" + name + "' in " + definition.get());
         }
     }
+
+    /**
+     * Remove all the secrets from the configuration. This method is used to avoid logging secrets
+     *
+     * @param object
+     * @return the object without secrets
+     */
+    public static Object redactSecrets(Object object) {
+        if (object == null) {
+            return null;
+        }
+
+        if (object instanceof List list) {
+            List<Object> other = new ArrayList<>(list.size());
+            list.forEach(o -> other.add(redactSecrets(o)));
+            return other;
+        }
+        if (object instanceof Set set) {
+            Set<Object> other = new HashSet<>(set.size());
+            set.forEach(o -> other.add(redactSecrets(o)));
+            return other;
+        }
+
+        if (object instanceof Map map) {
+            Map<Object, Object> other = new HashMap<>();
+            map.forEach(
+                    (k, v) -> {
+                        String keyLowercase = (String.valueOf(k)).toLowerCase();
+                        if (keyLowercase.contains("password")
+                                || keyLowercase.contains("pwd")
+                                || keyLowercase.contains("secure")
+                                || keyLowercase.contains("secret")
+                                || keyLowercase.contains("serviceaccountjson")
+                                || keyLowercase.contains("access-key")
+                                || keyLowercase.contains("token")) {
+                            other.put(k, "<REDACTED>");
+                        } else {
+                            other.put(k, redactSecrets(v));
+                        }
+                    });
+            return other;
+        }
+
+        return object;
+    }
 }

langstream-cli/src/main/java/ai/langstream/cli/commands/applications/GithubRepositoryDownloader.java

@@ -39,6 +39,7 @@ public static File downloadGithubRepository(URI uri, Consumer<String> logger) {
         RequestedDirectory requestedDirectory = parseRequest(uri);
 
         final Path directory = Files.createTempDirectory("langstream");
+        logger.accept(String.format("Cloning GitHub repository %s locally", uri));
 
         final long start = System.currentTimeMillis();
         Git.cloneRepository()

langstream-e2e-tests/pom.xml

@@ -52,6 +52,17 @@
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>langstream-core</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

langstream-e2e-tests/src/test/java/ai/langstream/tests/CassandraSinkIT.java

@@ -18,6 +18,7 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
+import ai.langstream.tests.util.BaseEndToEndTest;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;

langstream-e2e-tests/src/test/java/ai/langstream/tests/ChatCompletionsIT.java

@@ -15,6 +15,7 @@
  */
 package ai.langstream.tests;
 
+import ai.langstream.tests.util.BaseEndToEndTest;
 import ai.langstream.tests.util.ConsumeGatewayMessage;
 import java.util.List;
 import java.util.Map;

langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonFunctionIT.java

@@ -17,6 +17,7 @@
 
 import ai.langstream.deployer.k8s.api.crds.agents.AgentCustomResource;
 import ai.langstream.deployer.k8s.api.crds.apps.ApplicationCustomResource;
+import ai.langstream.tests.util.BaseEndToEndTest;
 import io.fabric8.kubernetes.api.model.Secret;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -96,7 +97,7 @@ public void test(String appDir) {
                                             .size());
                         });
 
-        final List<String> topics = getAllTopicsFromKafka();
-        Assertions.assertEquals(List.of("TEST_TOPIC_0"), topics);
+        final List<String> topics = getAllTopics();
+        Assertions.assertEquals(List.of("ls-test-topic0"), topics);
     }
 }

langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java

@@ -15,6 +15,7 @@
  */
 package ai.langstream.tests;
 
+import ai.langstream.tests.util.BaseEndToEndTest;
 import ai.langstream.tests.util.ConsumeGatewayMessage;
 import java.util.List;
 import java.util.Map;