From 0c9833ba6c8a2eab1344fef9f25318b11f020ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edd=C3=BA=20Mel=C3=A9ndez=20Gonzales?= Date: Wed, 21 Feb 2024 11:39:58 -0500 Subject: [PATCH] Add Milvus module (#8352) --- .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/enhancement.yaml | 1 + .github/ISSUE_TEMPLATE/feature.yaml | 1 + .github/dependabot.yml | 5 ++ .github/labeler.yml | 4 ++ .github/settings.yml | 3 + docs/modules/milvus.md | 36 ++++++++++ mkdocs.yml | 1 + modules/milvus/build.gradle | 8 +++ .../milvus/MilvusContainer.java | 61 +++++++++++++++++ .../resources/testcontainers/embedEtcd.yaml | 2 + .../milvus/MilvusContainerTest.java | 66 +++++++++++++++++++ .../src/test/resources/logback-test.xml | 16 +++++ 13 files changed, 205 insertions(+) create mode 100644 docs/modules/milvus.md create mode 100644 modules/milvus/build.gradle create mode 100644 modules/milvus/src/main/java/org/testcontainers/milvus/MilvusContainer.java create mode 100644 modules/milvus/src/main/resources/testcontainers/embedEtcd.yaml create mode 100644 modules/milvus/src/test/java/org/testcontainers/milvus/MilvusContainerTest.java create mode 100644 modules/milvus/src/test/resources/logback-test.xml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 216076ee084..d7c304a2cd4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -33,6 +33,7 @@ body: - Kafka - LocalStack - MariaDB + - Milvus - MinIO - MockServer - MongoDB diff --git a/.github/ISSUE_TEMPLATE/enhancement.yaml b/.github/ISSUE_TEMPLATE/enhancement.yaml index 80b20178318..2bc0e8a3e20 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.yaml +++ b/.github/ISSUE_TEMPLATE/enhancement.yaml @@ -33,6 +33,7 @@ body: - Kafka - LocalStack - MariaDB + - Milvus - MinIO - MockServer - MongoDB diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml index 96af8c19ea7..02310b638e1 100644 --- a/.github/ISSUE_TEMPLATE/feature.yaml +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -33,6 +33,7 @@ body: - Kafka - LocalStack - MariaDB + - Milvus - MinIO - MockServer - MongoDB diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4f81e37b26b..24ced388caf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -166,6 +166,11 @@ updates: ignore: - dependency-name: "org.mariadb:r2dbc-mariadb" update-types: [ "version-update:semver-minor" ] + - package-ecosystem: "gradle" + directory: "/modules/milvus" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 - package-ecosystem: "gradle" directory: "/modules/minio" schedule: diff --git a/.github/labeler.yml b/.github/labeler.yml index 70e69146d25..b7c1aa2ac01 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -99,6 +99,10 @@ - changed-files: - any-glob-to-any-file: - modules/mariadb/**/* +"modules/milvus": + - changed-files: + - any-glob-to-any-file: + - modules/milvus/**/* "modules/minio": - changed-files: - any-glob-to-any-file: diff --git a/.github/settings.yml b/.github/settings.yml index 12629f2f355..fdf582198d2 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -169,6 +169,9 @@ labels: - name: modules/mariadb color: '#006b75' + - name: modules/milvus + color: '#006b75' + - name: modules/minio color: '#006b75' diff --git a/docs/modules/milvus.md b/docs/modules/milvus.md new file mode 100644 index 00000000000..2808f9d2199 --- /dev/null +++ b/docs/modules/milvus.md @@ -0,0 +1,36 @@ +# Milvus + +Testcontainers module for [Milvus](https://hub.docker.com/r/milvusdb/milvus). + +## Milvus's usage examples + +You can start a Milvus container instance from any Java application by using: + + +[Default config](../../modules/milvus/src/test/java/org/testcontainers/milvus/MilvusContainerTest.java) inside_block:milvusContainer + + +With external Etcd: + + +[External Etcd](../../modules/milvus/src/test/java/org/testcontainers/milvus/MilvusContainerTest.java) inside_block:externalEtcd + + +## Adding this module to your project dependencies + +Add the following dependency to your `pom.xml`/`build.gradle` file: + +=== "Gradle" + ```groovy + testImplementation "org.testcontainers:milvus:{{latest_version}}" + ``` + +=== "Maven" + ```xml + + org.testcontainers + milvus + {{latest_version}} + test + + ``` diff --git a/mkdocs.yml b/mkdocs.yml index cda56106885..bcd53d294d9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,6 +84,7 @@ nav: - modules/k3s.md - modules/kafka.md - modules/localstack.md + - modules/milvus.md - modules/minio.md - modules/mockserver.md - modules/nginx.md diff --git a/modules/milvus/build.gradle b/modules/milvus/build.gradle new file mode 100644 index 00000000000..8826e6764ba --- /dev/null +++ b/modules/milvus/build.gradle @@ -0,0 +1,8 @@ +description = "Testcontainers :: ActiveMQ" + +dependencies { + api project(':testcontainers') + + testImplementation 'org.assertj:assertj-core:3.25.1' + testImplementation 'io.milvus:milvus-sdk-java:2.3.4' +} diff --git a/modules/milvus/src/main/java/org/testcontainers/milvus/MilvusContainer.java b/modules/milvus/src/main/java/org/testcontainers/milvus/MilvusContainer.java new file mode 100644 index 00000000000..57b1b483cf8 --- /dev/null +++ b/modules/milvus/src/main/java/org/testcontainers/milvus/MilvusContainer.java @@ -0,0 +1,61 @@ +package org.testcontainers.milvus; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +/** + * Testcontainers implementation for Milvus. + *

+ * Supported image: {@code milvusdb/milvus} + *

+ * Exposed ports: + *

+ */ +public class MilvusContainer extends GenericContainer { + + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("milvusdb/milvus"); + + private String etcdEndpoint; + + public MilvusContainer(String image) { + this(DockerImageName.parse(image)); + } + + public MilvusContainer(DockerImageName dockerImageName) { + super(dockerImageName); + dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); + withExposedPorts(9091, 19530); + waitingFor(Wait.forHttp("/healthz").forPort(9091)); + withCommand("milvus", "run", "standalone"); + withCopyFileToContainer( + MountableFile.forClasspathResource("testcontainers/embedEtcd.yaml"), + "/milvus/configs/embedEtcd.yaml" + ); + withEnv("COMMON_STORAGETYPE", "local"); + } + + @Override + protected void configure() { + if (this.etcdEndpoint == null) { + withEnv("ETCD_USE_EMBED", "true"); + withEnv("ETCD_DATA_DIR", "/var/lib/milvus/etcd"); + withEnv("ETCD_CONFIG_PATH", "/milvus/configs/embedEtcd.yaml"); + } else { + withEnv("ETCD_ENDPOINTS", this.etcdEndpoint); + } + } + + public MilvusContainer withEtcdEndpoint(String etcdEndpoint) { + this.etcdEndpoint = etcdEndpoint; + return this; + } + + public String getEndpoint() { + return "http://" + getHost() + ":" + getMappedPort(19530); + } +} diff --git a/modules/milvus/src/main/resources/testcontainers/embedEtcd.yaml b/modules/milvus/src/main/resources/testcontainers/embedEtcd.yaml new file mode 100644 index 00000000000..2bd73bbd59f --- /dev/null +++ b/modules/milvus/src/main/resources/testcontainers/embedEtcd.yaml @@ -0,0 +1,2 @@ +listen-client-urls: http://0.0.0.0:2379 +advertise-client-urls: http://0.0.0.0:2379 diff --git a/modules/milvus/src/test/java/org/testcontainers/milvus/MilvusContainerTest.java b/modules/milvus/src/test/java/org/testcontainers/milvus/MilvusContainerTest.java new file mode 100644 index 00000000000..3a6d6275ec2 --- /dev/null +++ b/modules/milvus/src/test/java/org/testcontainers/milvus/MilvusContainerTest.java @@ -0,0 +1,66 @@ +package org.testcontainers.milvus; + +import io.milvus.client.MilvusServiceClient; +import io.milvus.param.ConnectParam; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MilvusContainerTest { + + @Test + public void withDefaultConfig() { + try ( + // milvusContainer { + MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.3.9") + // } + ) { + milvus.start(); + + assertThat(milvus.getEnvMap()).doesNotContainKey("ETCD_ENDPOINTS"); + assertMilvusVersion(milvus); + } + } + + @Test + public void withExternalEtcd() { + try ( + // externalEtcd { + Network network = Network.newNetwork(); + GenericContainer etcd = new GenericContainer<>("quay.io/coreos/etcd:v3.5.5") + .withNetwork(network) + .withNetworkAliases("etcd") + .withCommand( + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls=http://0.0.0.0:2379", + "--data-dir=/etcd" + ) + .withEnv("ETCD_AUTO_COMPACTION_MODE", "revision") + .withEnv("ETCD_AUTO_COMPACTION_RETENTION", "1000") + .withEnv("ETCD_QUOTA_BACKEND_BYTES", "4294967296") + .withEnv("ETCD_SNAPSHOT_COUNT", "50000") + .waitingFor(Wait.forLogMessage(".*ready to serve client requests.*", 1)); + MilvusContainer milvus = new MilvusContainer("milvusdb/milvus:v2.3.9") + .withNetwork(network) + .withEtcdEndpoint("etcd:2379") + .dependsOn(etcd) + // } + ) { + milvus.start(); + + assertThat(milvus.getEnvMap()).doesNotContainKey("ETCD_USE_EMBED"); + assertMilvusVersion(milvus); + } + } + + private static void assertMilvusVersion(MilvusContainer milvus) { + MilvusServiceClient milvusClient = new MilvusServiceClient( + ConnectParam.newBuilder().withUri(milvus.getEndpoint()).build() + ); + assertThat(milvusClient.getVersion().getData().getVersion()).isEqualTo("v2.3.9"); + } +} diff --git a/modules/milvus/src/test/resources/logback-test.xml b/modules/milvus/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..83ef7a1a3ef --- /dev/null +++ b/modules/milvus/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + %d{HH:mm:ss.SSS} %-5level %logger - %msg%n + + + + + + + + +