From 890bfb29cd47d51b335fc50bccdb3a1e9c458d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 28 Sep 2023 10:52:15 +0200 Subject: [PATCH] secrets --- .../ai/langstream/tests/PythonAgentsIT.java | 4 ++- .../tests/WebCrawlerToVectorIT.java | 9 +++++++ .../tests/util/BaseEndToEndTest.java | 26 +++++++++++++------ .../apps/webcrawler-to-vector/crawler.yaml | 6 +++++ .../src/test/resources/secrets/secret1.yaml | 8 +++++- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java index 4ede17d59..a4a333f26 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/PythonAgentsIT.java @@ -83,7 +83,9 @@ public void testProcessor() throws Exception { deleteAppAndAwaitCleanup(tenant, applicationId); final List topics = getAllTopics(); - Assertions.assertEquals(List.of("ls-test-topic0"), topics); + log.info("all topics: {}", topics); + Assertions.assertTrue(topics.contains("ls-test-topic0")); + Assertions.assertFalse(topics.contains("ls-test-topic1")); } @Test diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java index cea83df7e..a6c29671b 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/WebCrawlerToVectorIT.java @@ -56,6 +56,15 @@ public static void checkCredentials() { @Test public void test() throws Exception { + if (!codeStorageConfig.type().equals("s3")) { + throw new IllegalStateException( + "This test can only run with S3 code storage, but got: " + + codeStorageConfig.type()); + } + + appEnv.put("S3_ENDPOINT", codeStorageConfig.configuration().get("endpoint")); + appEnv.put("S3_ACCESS_KEY", codeStorageConfig.configuration().get("access-key")); + appEnv.put("S3_SECRET_KEY", codeStorageConfig.configuration().get("secret-key")); installLangStreamCluster(true); final String tenant = "ten-" + System.currentTimeMillis(); setupTenant(tenant); diff --git a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java index cff93a74b..b96d827d9 100644 --- a/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java +++ b/langstream-e2e-tests/src/test/java/ai/langstream/tests/util/BaseEndToEndTest.java @@ -526,8 +526,7 @@ private static KubeCluster getKubeCluster() { public void setupSingleTest() { // cleanup previous runs cleanupAllEndToEndTestsNamespaces(); - codeStorageProvider.cleanup(); - streamingClusterProvider.cleanup(); + cleanupEnv(); namespace = "ls-test-" + UUID.randomUUID().toString().substring(0, 8); @@ -541,18 +540,29 @@ public void setupSingleTest() { .serverSideApply(); } + private void cleanupEnv() { + if (codeStorageProvider != null) { + codeStorageProvider.cleanup(); + } + if (streamingClusterProvider != null) { + streamingClusterProvider.cleanup(); + } + } + @AfterEach public void cleanupAfterEach() { cleanupAllEndToEndTestsNamespaces(); - streamingClusterProvider.cleanup(); + cleanupEnv(); } private static void cleanupAllEndToEndTestsNamespaces() { - client.namespaces().withLabel("app", "ls-test").delete(); - client.namespaces().list().getItems().stream() - .map(ns -> ns.getMetadata().getName()) - .filter(ns -> ns.startsWith(TENANT_NAMESPACE_PREFIX)) - .forEach(ns -> deleteTenantNamespace(ns)); + if (client != null) { + client.namespaces().withLabel("app", "ls-test").delete(); + client.namespaces().list().getItems().stream() + .map(ns -> ns.getMetadata().getName()) + .filter(ns -> ns.startsWith(TENANT_NAMESPACE_PREFIX)) + .forEach(ns -> deleteTenantNamespace(ns)); + } } private static void deleteTenantNamespace(String ns) { diff --git a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml index d20b79940..a93bdd2a9 100644 --- a/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml +++ b/langstream-e2e-tests/src/test/resources/apps/webcrawler-to-vector/crawler.yaml @@ -39,9 +39,15 @@ pipeline: handle-cookies: true max-unflushed-pages: 100 bucketName: "langstream-test-crawler-to-vector" +<<<<<<< Updated upstream endpoint: http://minio.ls-test-minio.svc.cluster.local:9000 access-key: minioadmin secret-key: minioadmin +======= + endpoint: "{{{secrets.s3.endpoint}}}" + access-key: "{{{secrets.s3.access-key}}}" + secret-key: "{{{secrets.s3.secret-key}}}" +>>>>>>> Stashed changes - name: "Extract text" type: "text-extractor" - name: "Normalise text" diff --git a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml index 4c652b698..898933ffe 100644 --- a/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml +++ b/langstream-e2e-tests/src/test/resources/secrets/secret1.yaml @@ -64,4 +64,10 @@ secrets: environment: "${ASTRA_ENVIRONMENT}" - id: kafka data: - bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS}" \ No newline at end of file + bootstrap-servers: "${KAFKA_BOOTSTRAP_SERVERS}" + + - id: s3 + data: + endpoint: "${S3_ENDPOINT}" + access-key: "${S3_ACCESS_KEY}" + secret-key: "${S3_SECRET_KEY}" \ No newline at end of file