secrets

LangStream · Sep 28, 2023 · 890bfb2 · 890bfb2
1 parent 4632ebb
commit 890bfb2
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 10 deletions.
diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java
@@ -83,7 +83,9 @@ public void testProcessor() throws Exception {
         deleteAppAndAwaitCleanup(tenant, applicationId);
 
         final List<String> topics = getAllTopics();
-        Assertions.assertEquals(List.of("ls-test-topic0"), topics);
+        log.info("all topics: {}", topics);
+        Assertions.assertTrue(topics.contains("ls-test-topic0"));
+        Assertions.assertFalse(topics.contains("ls-test-topic1"));
     }
 
     @Test

diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java
@@ -56,6 +56,15 @@ public static void checkCredentials() {
 
     @Test
     public void test() throws Exception {
+        if (!codeStorageConfig.type().equals("s3")) {
+            throw new IllegalStateException(
+                    "This test can only run with S3 code storage, but got: "
+                            + codeStorageConfig.type());
+        }
+
+        appEnv.put("S3_ENDPOINT", codeStorageConfig.configuration().get("endpoint"));
+        appEnv.put("S3_ACCESS_KEY", codeStorageConfig.configuration().get("access-key"));
+        appEnv.put("S3_SECRET_KEY", codeStorageConfig.configuration().get("secret-key"));
         installLangStreamCluster(true);
         final String tenant = "ten-" + System.currentTimeMillis();
         setupTenant(tenant);

diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java
@@ -526,8 +526,7 @@ private static KubeCluster getKubeCluster() {
     public void setupSingleTest() {
         // cleanup previous runs
         cleanupAllEndToEndTestsNamespaces();
-        codeStorageProvider.cleanup();
-        streamingClusterProvider.cleanup();
+        cleanupEnv();
 
         namespace = "ls-test-" + UUID.randomUUID().toString().substring(0, 8);
 
@@ -541,18 +540,29 @@ public void setupSingleTest() {
                 .serverSideApply();
     }
 
+    private void cleanupEnv() {
+        if (codeStorageProvider != null) {
+            codeStorageProvider.cleanup();
+        }
+        if (streamingClusterProvider != null) {
+            streamingClusterProvider.cleanup();
+        }
+    }
+
     @AfterEach
     public void cleanupAfterEach() {
         cleanupAllEndToEndTestsNamespaces();
-        streamingClusterProvider.cleanup();
+        cleanupEnv();
     }
 
     private static void cleanupAllEndToEndTestsNamespaces() {
-        client.namespaces().withLabel("app", "ls-test").delete();
-        client.namespaces().list().getItems().stream()
-                .map(ns -> ns.getMetadata().getName())
-                .filter(ns -> ns.startsWith(TENANT_NAMESPACE_PREFIX))
-                .forEach(ns -> deleteTenantNamespace(ns));
+        if (client != null) {
+            client.namespaces().withLabel("app", "ls-test").delete();
+            client.namespaces().list().getItems().stream()
+                    .map(ns -> ns.getMetadata().getName())
+                    .filter(ns -> ns.startsWith(TENANT_NAMESPACE_PREFIX))
+                    .forEach(ns -> deleteTenantNamespace(ns));
+        }
     }
 
     private static void deleteTenantNamespace(String ns) {

diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml
@@ -39,9 +39,15 @@ pipeline:
       handle-cookies: true
       max-unflushed-pages: 100
       bucketName: "langstream-test-crawler-to-vector"
+<<<<<<< Updated upstream
       endpoint: http://minio.ls-test-minio.svc.cluster.local:9000
       access-key: minioadmin
       secret-key: minioadmin
+=======
+      endpoint: "{{{secrets.s3.endpoint}}}"
+      access-key: "{{{secrets.s3.access-key}}}"
+      secret-key: "{{{secrets.s3.secret-key}}}"
+>>>>>>> Stashed changes
   - name: "Extract text"
     type: "text-extractor"
   - name: "Normalise text"

diff --git a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml
@@ -64,4 +64,10 @@ secrets:
       environment: "${ASTRA_ENVIRONMENT}"
   - id: kafka
     data:
-      bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS}"
+      bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS}"
+
+  - id: s3
+    data:
+      endpoint: "${S3_ENDPOINT}"
+      access-key: "${S3_ACCESS_KEY}"
+      secret-key: "${S3_SECRET_KEY}"