Skip to content

Commit

Permalink
Add e2e test for S3 source (#552)
Browse files Browse the repository at this point in the history
  • Loading branch information
nicoloboschi authored Oct 9, 2023
1 parent 1deef92 commit e26329a
Show file tree
Hide file tree
Showing 24 changed files with 32,071 additions and 32 deletions.
17 changes: 15 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,15 @@ jobs:
name: End to End tests
runs-on: LangStream-4-cores
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- name: Python
category: python
- name: Other
category: other

steps:
- name: 'Setup: checkout project'
uses: actions/checkout@v2
Expand Down Expand Up @@ -149,13 +158,17 @@ jobs:

- uses: azure/setup-kubectl@v3

- name: 'Build and test'
- name: Build docker
run: |
chmod +x mvnw
uname -m
./dev/prepare-minikube-for-e2e-tests.sh
- name: Run tests
run: |
chmod +x mvnw
./mvnw -ntp install -pl langstream-e2e-tests -am -DskipTests
./mvnw -ntp verify -pl langstream-e2e-tests -De2eTests -DexcludedGroups="needs-credentials" -Dlangstream.tests.recycleenv=true
./mvnw -ntp verify -pl langstream-e2e-tests -De2eTests -Dgroups="${{ matrix.category }}" -Dlangstream.tests.recycleenv=true
- name: Upload Surefire reports
uses: actions/upload-artifact@v3
Expand Down
10 changes: 5 additions & 5 deletions examples/applications/text-processing/extract-text.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ pipeline:
- name: "Read from S3"
type: "s3-source"
configuration:
bucketName: "${secrets.s3-credentials.bucket-name}"
endpoint: "${secrets.s3-credentials.endpoint}"
access-key: "${secrets.s3-credentials.access-key}"
secret-key: "${secrets.s3-credentials.secret}"
region: "${secrets.s3-credentials.region}"
bucketName: "${secrets.s3.bucket-name}"
endpoint: "${secrets.s3.endpoint}"
access-key: "${secrets.s3.access-key}"
secret-key: "${secrets.s3.secret}"
region: "${secrets.s3.region}"
idle-time: 5
- name: "Extract text"
type: "text-extractor"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static org.junit.jupiter.api.Assertions.fail;

import ai.langstream.tests.util.BaseEndToEndTest;
import ai.langstream.tests.util.TestSuites;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
Expand All @@ -27,11 +28,13 @@
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;

@Slf4j
@ExtendWith(BaseEndToEndTest.class)
@Tag(TestSuites.CATEGORY_OTHER)
public class CassandraSinkIT extends BaseEndToEndTest {

@BeforeEach
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import ai.langstream.tests.util.BaseEndToEndTest;
import ai.langstream.tests.util.ConsumeGatewayMessage;
import ai.langstream.tests.util.TestSuites;
import java.util.List;
import java.util.Map;
import java.util.UUID;
Expand All @@ -31,7 +32,7 @@

@Slf4j
@ExtendWith(BaseEndToEndTest.class)
@Tag(BaseEndToEndTest.CATEGORY_NEEDS_CREDENTIALS)
@Tag(TestSuites.CATEGORY_NEEDS_CREDENTIALS)
public class ChatCompletionsIT extends BaseEndToEndTest {

static Map<String, String> appEnv;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@
package ai.langstream.tests;

import ai.langstream.tests.util.BaseEndToEndTest;
import ai.langstream.tests.util.TestSuites;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;

@Slf4j
@ExtendWith(BaseEndToEndTest.class)
@Tag(TestSuites.CATEGORY_PYTHON)
public class PythonAgentsIT extends BaseEndToEndTest {

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright DataStax, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ai.langstream.tests;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;

import ai.langstream.tests.util.BaseEndToEndTest;
import ai.langstream.tests.util.ConsumeGatewayMessage;
import ai.langstream.tests.util.TestSuites;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;

@Slf4j
@ExtendWith(BaseEndToEndTest.class)
@Tag(TestSuites.CATEGORY_OTHER)
public class S3AzureSourceIT extends BaseEndToEndTest {

@Test
public void test() throws Exception {
final boolean isS3 = codeStorageConfig.type().equals("s3");
final String tenant = "ten-" + System.currentTimeMillis();
final String bucketName = "langstream-qa-source-" + tenant;
final String objectName = "simple-" + tenant + ".pdf";

installLangStreamCluster(true);

codeStorageProvider.createBucket(bucketName);
codeStorageProvider.uploadFromFile(
Paths.get("src/test/resources/files/simple.pdf").toFile().getAbsolutePath(),
bucketName,
objectName);
setupTenant(tenant);
final String applicationId = "my-test-app";

final HashMap<String, String> env = new HashMap<>();
if (isS3) {
env.put("S3_ENDPOINT", codeStorageConfig.configuration().get("endpoint"));
env.put("S3_BUCKET_NAME", bucketName);
env.put("S3_ACCESS_KEY", codeStorageConfig.configuration().get("access-key"));
env.put("S3_SECRET_KEY", codeStorageConfig.configuration().get("secret-key"));
} else {
env.put("AZURE_ENDPOINT", codeStorageConfig.configuration().get("endpoint"));
env.put("AZURE_CONTAINER_NAME", bucketName);
env.put(
"AZURE_ACCOUNT_NAME",
codeStorageConfig.configuration().get("storage-account-name"));
env.put(
"AZURE_ACCOUNT_KEY",
codeStorageConfig.configuration().get("storage-account-key"));
}

deployLocalApplicationAndAwaitReady(
tenant, applicationId, isS3 ? "s3-source" : "azure-blob-source", env, 1);

// in this timeout is also included the runtime pod startup time
Awaitility.await()
.atMost(1, TimeUnit.MINUTES)
.pollInterval(5, TimeUnit.SECONDS)
.untilAsserted(
() -> {
assertFalse(codeStorageProvider.objectExists(bucketName, objectName));
});

final ConsumeGatewayMessage message =
consumeOneMessageFromGateway(
applicationId, "consume-chunks", "--position", "earliest");
log.info("output {}", message);
final Map<String, Object> asMap = message.recordValueAsMap();
assertEquals("en", asMap.get("language"));
assertEquals("6", asMap.get("chunk_num_tokens"));
assertEquals(objectName, asMap.get("name"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import ai.langstream.tests.util.BaseEndToEndTest;
import ai.langstream.tests.util.ConsumeGatewayMessage;
import ai.langstream.tests.util.TestSuites;
import java.util.List;
import java.util.Map;
import java.util.UUID;
Expand All @@ -29,7 +30,7 @@

@Slf4j
@ExtendWith(BaseEndToEndTest.class)
@Tag(BaseEndToEndTest.CATEGORY_NEEDS_CREDENTIALS)
@Tag(TestSuites.CATEGORY_NEEDS_CREDENTIALS)
public class TextCompletionsIT extends BaseEndToEndTest {

static Map<String, String> appEnv;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import ai.langstream.tests.util.BaseEndToEndTest;
import ai.langstream.tests.util.ConsumeGatewayMessage;
import ai.langstream.tests.util.TestSuites;
import java.util.List;
import java.util.Map;
import java.util.UUID;
Expand All @@ -31,7 +32,7 @@

@Slf4j
@ExtendWith(BaseEndToEndTest.class)
@Tag(BaseEndToEndTest.CATEGORY_NEEDS_CREDENTIALS)
@Tag(TestSuites.CATEGORY_NEEDS_CREDENTIALS)
public class WebCrawlerToVectorIT extends BaseEndToEndTest {

static Map<String, String> appEnv;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,6 @@ public class BaseEndToEndTest implements TestWatcher {

public static final String TOPICS_PREFIX = "ls-test-";

public static final String CATEGORY_NEEDS_CREDENTIALS = "needs-credentials";

private static final boolean LANGSTREAM_RECYCLE_ENV =
Boolean.parseBoolean(System.getProperty("langstream.tests.recycleenv", "false"));

Expand Down Expand Up @@ -153,6 +151,10 @@ public void testFailed(ExtensionContext context, Throwable cause) {
dumpTest(prefix);
}

public static KubernetesClient getClient() {
return client;
}

private static void dumpTest(String prefix) {
dumpAllPodsLogs(prefix);
dumpEvents(prefix);
Expand All @@ -178,9 +180,14 @@ public static void applyManifestNoNamespace(String manifest) {
}

@SneakyThrows
protected static void copyFileToClientContainer(File file, String toPath) {
public static void copyFileToClientContainer(File file, String toPath) {
final String podName =
getFirstPodFromDeployment("langstream-client").getMetadata().getName();
copyFileToPod(podName, namespace, file, toPath);
}

@SneakyThrows
public static void copyFileToPod(String podName, String namespace, File file, String toPath) {
if (file.isFile()) {
runProcess(
"kubectl cp %s %s:%s -n %s"
Expand Down Expand Up @@ -208,7 +215,7 @@ private static Pod getFirstPodFromDeployment(String deploymentName) {
}

@SneakyThrows
protected static String executeCommandOnClient(String... args) {
public static String executeCommandOnClient(String... args) {
return executeCommandOnClient(2, TimeUnit.MINUTES, args);
}

Expand Down Expand Up @@ -501,7 +508,7 @@ private static StreamingClusterProvider getStreamingClusterProvider() {
private static CodeStorageProvider getCodeStorageProvider() {
switch (LANGSTREAM_CODESTORAGE) {
case "local-minio":
return new LocalMinioCodeStorageProvider(client);
return new LocalMinioCodeStorageProvider();
case "remote":
return new RemoteCodeStorageProvider();
default:
Expand Down Expand Up @@ -1106,11 +1113,13 @@ private static void deployLocalApplicationAndAwaitReady(
beforeCmd =
env.entrySet().stream()
.map(
e ->
"export '%s'='%s'"
.formatted(
e.getKey(),
e.getValue().replace("'", "''")))
e -> {
final String safeValue =
e.getValue() == null ? "" : e.getValue();
return "export '%s'='%s'"
.formatted(
e.getKey(), safeValue.replace("'", "''"));
})
.collect(Collectors.joining(" && "));
beforeCmd += " && ";
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package ai.langstream.tests.util;

import java.io.IOException;
import java.util.Map;

public interface CodeStorageProvider {
Expand All @@ -26,4 +27,10 @@ record CodeStorageConfig(String type, Map<String, String> configuration) {}
void cleanup();

void stop();

void createBucket(String bucketName) throws IOException;

void uploadFromFile(String path, String bucketName, String objectName) throws IOException;

boolean objectExists(String bucketName, String objectName) throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,9 @@ public String getAnswerFromChatCompletionsValue() {
final String answer = chatHistoryModel.get("answer").toString();
return answer;
}

@SneakyThrows
public Map<String, Object> recordValueAsMap() {
return JSON_MAPPER.readValue((String) record.getValue(), Map.class);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Copyright DataStax, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ai.langstream.tests.util;

public class TestSuites {

public static final String CATEGORY_PYTHON = "python";
public static final String CATEGORY_OTHER = "other";

public static final String CATEGORY_NEEDS_CREDENTIALS = "needs-credentials";
}
Loading

0 comments on commit e26329a

Please sign in to comment.