From 4632ebb7c5e29b4c8a8e4794a9cf2c539e2e056b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 27 Sep 2023 19:10:23 +0200 Subject: [PATCH 1/3] E2e Tests: allow to pass google json sa --- .../ai/langstream/tests/TextCompletionsIT.java | 6 +++--- .../langstream/tests/util/BaseEndToEndTest.java | 16 ++++++++++------ .../apps/chat-completions/configuration.yaml | 2 +- .../apps/text-completions/configuration.yaml | 2 +- .../apps/webcrawler-to-vector/configuration.yaml | 2 +- .../apps/webcrawler-to-vector/crawler.yaml | 2 +- .../src/test/resources/secrets/secret1.yaml | 2 +- 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/TextCompletionsIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/TextCompletionsIT.java index 6662aafae..e44de603a 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/TextCompletionsIT.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/TextCompletionsIT.java @@ -51,7 +51,7 @@ public static Map getAppEnvForAIServiceProvider() { return getAppEnvMapFromSystem( List.of( "VERTEX_AI_URL", - "VERTEX_AI_TOKEN", + "VERTEX_AI_SERVICE_ACCOUNT_JSON", "VERTEX_AI_REGION", "VERTEX_AI_PROJECT")); } @@ -69,7 +69,7 @@ public void test() throws Exception { final String sessionId = UUID.randomUUID().toString(); executeCommandOnClient( - "bin/langstream gateway produce %s produce-input -v 'Translate \"Good morning\" to French.' -p sessionId=%s" + "bin/langstream gateway produce %s produce-input -v 'Translate \"Apple\" to French, lowercase.' -p sessionId=%s" .formatted(applicationId, sessionId) .split(" ")); @@ -81,6 +81,6 @@ public void test() throws Exception { .formatted(sessionId) .split(" ")); log.info("Output: {}", message); - Assertions.assertTrue(message.getAnswerFromChatCompletionsValue().contains("Bonjour")); + Assertions.assertTrue(message.getAnswerFromChatCompletionsValue().contains("pomme")); } } diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java index 7dec10a46..cff93a74b 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java @@ -1095,7 +1095,12 @@ private static void deployLocalApplicationAndAwaitReady( if (env != null && !env.isEmpty()) { beforeCmd = env.entrySet().stream() - .map(e -> "export \"%s\"=\"%s\"".formatted(e.getKey(), e.getValue())) + .map( + e -> + "export '%s'='%s'" + .formatted( + e.getKey(), + e.getValue().replace("'", "''"))) .collect(Collectors.joining(" && ")); beforeCmd += " && "; } @@ -1122,11 +1127,10 @@ private static void deployLocalApplicationAndAwaitReady( } else { podUids = ""; } - executeCommandOnClient( - (beforeCmd - + "bin/langstream apps %s %s -app /tmp/app -i /tmp/instance.yaml -s /tmp/secrets.yaml") - .formatted(isUpdate ? "update" : "deploy", applicationId) - .split(" ")); + final String command = + "bin/langstream apps %s %s -app /tmp/app -i /tmp/instance.yaml -s /tmp/secrets.yaml" + .formatted(isUpdate ? "update" : "deploy", applicationId); + executeCommandOnClient((beforeCmd + command).split(" ")); awaitApplicationReady(applicationId, expectedNumExecutors); Awaitility.await() diff --git a/langstream-e2e-tests/src/test/resources/apps/chat-completions/configuration.yaml b/langstream-e2e-tests/src/test/resources/apps/chat-completions/configuration.yaml index 79c593070..44cf803d9 100644 --- a/langstream-e2e-tests/src/test/resources/apps/chat-completions/configuration.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/chat-completions/configuration.yaml @@ -29,6 +29,6 @@ configuration: id: "vertex" configuration: url: "{{ secrets.vertex-ai.url }}" - token: "{{ secrets.vertex-ai.token }}" + serviceAccountJson: "{{ secrets.vertex-ai.service-json }}" region: "{{ secrets.vertex-ai.region }}" project: "{{ secrets.vertex-ai.project }}" diff --git a/langstream-e2e-tests/src/test/resources/apps/text-completions/configuration.yaml b/langstream-e2e-tests/src/test/resources/apps/text-completions/configuration.yaml index 79c593070..44cf803d9 100644 --- a/langstream-e2e-tests/src/test/resources/apps/text-completions/configuration.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/text-completions/configuration.yaml @@ -29,6 +29,6 @@ configuration: id: "vertex" configuration: url: "{{ secrets.vertex-ai.url }}" - token: "{{ secrets.vertex-ai.token }}" + serviceAccountJson: "{{ secrets.vertex-ai.service-json }}" region: "{{ secrets.vertex-ai.region }}" project: "{{ secrets.vertex-ai.project }}" diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/configuration.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/configuration.yaml index 429ba9e7d..0a9d8beee 100644 --- a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/configuration.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/configuration.yaml @@ -38,6 +38,6 @@ configuration: id: "vertex" configuration: url: "{{ secrets.vertex-ai.url }}" - token: "{{ secrets.vertex-ai.token }}" + serviceAccountJson: "{{ secrets.vertex-ai.service-json }}" region: "{{ secrets.vertex-ai.region }}" project: "{{ secrets.vertex-ai.project }}" \ No newline at end of file diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml index 1a6de0357..d20b79940 100644 --- a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml @@ -39,7 +39,7 @@ pipeline: handle-cookies: true max-unflushed-pages: 100 bucketName: "langstream-test-crawler-to-vector" - endpoint: http://minio.minio-dev.svc.cluster.local:9000 + endpoint: http://minio.ls-test-minio.svc.cluster.local:9000 access-key: minioadmin secret-key: minioadmin - name: "Extract text" diff --git a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml index f8a31879a..4c652b698 100644 --- a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml +++ b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml @@ -52,7 +52,7 @@ secrets: - id: vertex-ai data: url: "${VERTEX_AI_URL:-}" - token: "${VERTEX_AI_TOKEN:-}" + service-json: "${VERTEX_AI_SERVICE_ACCOUNT_JSON:-}" region: "${VERTEX_AI_REGION:-}" project: "${VERTEX_AI_PROJECT:-}" - id: astra From 890bfb29cd47d51b335fc50bccdb3a1e9c458d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 28 Sep 2023 10:52:15 +0200 Subject: [PATCH 2/3] secrets --- .../ai/langstream/tests/PythonAgentsIT.java | 4 ++- .../tests/WebCrawlerToVectorIT.java | 9 +++++++ .../tests/util/BaseEndToEndTest.java | 26 +++++++++++++------ .../apps/webcrawler-to-vector/crawler.yaml | 6 +++++ .../src/test/resources/secrets/secret1.yaml | 8 +++++- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java index 4ede17d59..a4a333f26 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java @@ -83,7 +83,9 @@ public void testProcessor() throws Exception { deleteAppAndAwaitCleanup(tenant, applicationId); final List topics = getAllTopics(); - Assertions.assertEquals(List.of("ls-test-topic0"), topics); + log.info("all topics: {}", topics); + Assertions.assertTrue(topics.contains("ls-test-topic0")); + Assertions.assertFalse(topics.contains("ls-test-topic1")); } @Test diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java index cea83df7e..a6c29671b 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java @@ -56,6 +56,15 @@ public static void checkCredentials() { @Test public void test() throws Exception { + if (!codeStorageConfig.type().equals("s3")) { + throw new IllegalStateException( + "This test can only run with S3 code storage, but got: " + + codeStorageConfig.type()); + } + + appEnv.put("S3_ENDPOINT", codeStorageConfig.configuration().get("endpoint")); + appEnv.put("S3_ACCESS_KEY", codeStorageConfig.configuration().get("access-key")); + appEnv.put("S3_SECRET_KEY", codeStorageConfig.configuration().get("secret-key")); installLangStreamCluster(true); final String tenant = "ten-" + System.currentTimeMillis(); setupTenant(tenant); diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java index cff93a74b..b96d827d9 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java @@ -526,8 +526,7 @@ private static KubeCluster getKubeCluster() { public void setupSingleTest() { // cleanup previous runs cleanupAllEndToEndTestsNamespaces(); - codeStorageProvider.cleanup(); - streamingClusterProvider.cleanup(); + cleanupEnv(); namespace = "ls-test-" + UUID.randomUUID().toString().substring(0, 8); @@ -541,18 +540,29 @@ public void setupSingleTest() { .serverSideApply(); } + private void cleanupEnv() { + if (codeStorageProvider != null) { + codeStorageProvider.cleanup(); + } + if (streamingClusterProvider != null) { + streamingClusterProvider.cleanup(); + } + } + @AfterEach public void cleanupAfterEach() { cleanupAllEndToEndTestsNamespaces(); - streamingClusterProvider.cleanup(); + cleanupEnv(); } private static void cleanupAllEndToEndTestsNamespaces() { - client.namespaces().withLabel("app", "ls-test").delete(); - client.namespaces().list().getItems().stream() - .map(ns -> ns.getMetadata().getName()) - .filter(ns -> ns.startsWith(TENANT_NAMESPACE_PREFIX)) - .forEach(ns -> deleteTenantNamespace(ns)); + if (client != null) { + client.namespaces().withLabel("app", "ls-test").delete(); + client.namespaces().list().getItems().stream() + .map(ns -> ns.getMetadata().getName()) + .filter(ns -> ns.startsWith(TENANT_NAMESPACE_PREFIX)) + .forEach(ns -> deleteTenantNamespace(ns)); + } } private static void deleteTenantNamespace(String ns) { diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml index d20b79940..a93bdd2a9 100644 --- a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml @@ -39,9 +39,15 @@ pipeline: handle-cookies: true max-unflushed-pages: 100 bucketName: "langstream-test-crawler-to-vector" +<<<<<<< Updated upstream endpoint: http://minio.ls-test-minio.svc.cluster.local:9000 access-key: minioadmin secret-key: minioadmin +======= + endpoint: "{{{secrets.s3.endpoint}}}" + access-key: "{{{secrets.s3.access-key}}}" + secret-key: "{{{secrets.s3.secret-key}}}" +>>>>>>> Stashed changes - name: "Extract text" type: "text-extractor" - name: "Normalise text" diff --git a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml index 4c652b698..898933ffe 100644 --- a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml +++ b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml @@ -64,4 +64,10 @@ secrets: environment: "${ASTRA_ENVIRONMENT}" - id: kafka data: - bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS}" \ No newline at end of file + bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS}" + + - id: s3 + data: + endpoint: "${S3_ENDPOINT}" + access-key: "${S3_ACCESS_KEY}" + secret-key: "${S3_SECRET_KEY}" \ No newline at end of file From 34c70066b9cbd5b92790bc7fd27be313f6b4c3a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 28 Sep 2023 12:28:38 +0200 Subject: [PATCH 3/3] fixe --- .../test/resources/apps/webcrawler-to-vector/chatbot.yaml | 3 ++- .../test/resources/apps/webcrawler-to-vector/crawler.yaml | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/chatbot.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/chatbot.yaml index f6c8aff53..1e71d7dc0 100644 --- a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/chatbot.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/chatbot.yaml @@ -34,7 +34,8 @@ pipeline: - name: "compute-embeddings" type: "compute-ai-embeddings" configuration: - model: "{{{secrets.open-ai.embeddings-model}}}" + ai-service: "{{{secrets.embeddings.service}}}" + model: "{{{secrets.embeddings.model}}}" embeddings-field: "value.question_embeddings" text: "{{% value.question }}" flush-interval: 0 diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml index a93bdd2a9..648bc44b8 100644 --- a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml @@ -39,15 +39,9 @@ pipeline: handle-cookies: true max-unflushed-pages: 100 bucketName: "langstream-test-crawler-to-vector" -<<<<<<< Updated upstream - endpoint: http://minio.ls-test-minio.svc.cluster.local:9000 - access-key: minioadmin - secret-key: minioadmin -======= endpoint: "{{{secrets.s3.endpoint}}}" access-key: "{{{secrets.s3.access-key}}}" secret-key: "{{{secrets.s3.secret-key}}}" ->>>>>>> Stashed changes - name: "Extract text" type: "text-extractor" - name: "Normalise text"