diff --git a/src/backend/build.gradle.kts b/src/backend/build.gradle.kts index fd65d4e066..f4f74cf69a 100644 --- a/src/backend/build.gradle.kts +++ b/src/backend/build.gradle.kts @@ -64,7 +64,6 @@ allprojects { dependency("commons-io:commons-io:${Versions.CommonsIO}") dependency("com.squareup.okhttp3:okhttp:${Versions.OKhttp}") dependency("com.google.guava:guava:${Versions.Guava}") - dependency("com.google.protobuf:protobuf-java:${Versions.ProtobufJava}") dependency("com.google.protobuf:protobuf-java-util:${Versions.ProtobufJava}") dependency("com.tencent.polaris:polaris-discovery-factory:${Versions.Polaris}") dependency("org.apache.commons:commons-text:${Versions.CommonsText}") diff --git a/src/backend/buildSrc/src/main/kotlin/Versions.kt b/src/backend/buildSrc/src/main/kotlin/Versions.kt index e60a75bdd8..5fbb7de387 100644 --- a/src/backend/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/buildSrc/src/main/kotlin/Versions.kt @@ -37,7 +37,7 @@ object Versions { const val Redline = "1.2.10" const val SkyWalkingApmToolkit = "8.10.0" const val Gson = "2.9.0" - const val ProtobufJava = "3.24.0" + const val ProtobufJava = "3.19.4" const val Guava = "31.1-jre" const val Shedlock = "4.12.0" const val JGit = "5.11.0.202103091610-r" @@ -70,5 +70,4 @@ object Versions { const val JavaCpp = "1.5.9" const val Notice = "1.0.0" const val SpringCloudFunction = "3.2.11" - const val Milvus = "2.4.1" } diff --git a/src/backend/job/biz-job/build.gradle.kts b/src/backend/job/biz-job/build.gradle.kts index e1e41a57b7..dad93d77d1 100644 --- a/src/backend/job/biz-job/build.gradle.kts +++ b/src/backend/job/biz-job/build.gradle.kts @@ -45,17 +45,6 @@ dependencies { implementation(project(":common:common-operate:operate-service")) implementation("org.springframework.boot:spring-boot-starter-data-mongodb") implementation("io.micrometer:micrometer-registry-prometheus") - implementation("io.milvus:milvus-sdk-java:${Versions.Milvus}") { - exclude(group = "org.slf4j") - exclude(group = "org.apache.logging.log4j") - exclude(group = "org.testcontainers") - exclude(group = "com.azure") - exclude(group = "com.amazonaws") - exclude(group = "io.minio") - exclude(group = "org.apache.hadoop") - exclude(group = "org.apache.parquet") - exclude(group = "com.squareup.okhttp3") - } implementation("com.tencent.devops:devops-schedule-common") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt index 6ad3fd7c7b..b0e78195d2 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt @@ -38,11 +38,11 @@ import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.task.cache.preload.ai.AiProperties import com.tencent.bkrepo.job.batch.task.cache.preload.ai.Document import com.tencent.bkrepo.job.batch.task.cache.preload.ai.EmbeddingModel -import com.tencent.bkrepo.job.batch.task.cache.preload.ai.MilvusVectorStore -import com.tencent.bkrepo.job.batch.task.cache.preload.ai.MilvusVectorStoreProperties +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusVectorStore +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusVectorStoreProperties import com.tencent.bkrepo.job.batch.task.cache.preload.ai.VectorStore +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusClient import com.tencent.bkrepo.job.config.properties.ArtifactAccessLogEmbeddingJobProperties -import io.milvus.client.MilvusClient import org.bson.types.ObjectId import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactSimilarityPreloadPlanGenerator.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactSimilarityPreloadPlanGenerator.kt index 2264f21423..ccfc2bfa79 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactSimilarityPreloadPlanGenerator.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactSimilarityPreloadPlanGenerator.kt @@ -34,11 +34,11 @@ import com.tencent.bkrepo.common.artifact.cache.service.ArtifactPreloadPlanGener import com.tencent.bkrepo.common.mongo.dao.util.sharding.MonthRangeShardingUtils import com.tencent.bkrepo.job.batch.task.cache.preload.ai.AiProperties import com.tencent.bkrepo.job.batch.task.cache.preload.ai.EmbeddingModel -import com.tencent.bkrepo.job.batch.task.cache.preload.ai.MilvusVectorStore -import com.tencent.bkrepo.job.batch.task.cache.preload.ai.MilvusVectorStoreProperties +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusVectorStore +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusVectorStoreProperties import com.tencent.bkrepo.job.batch.task.cache.preload.ai.SearchRequest import com.tencent.bkrepo.job.batch.task.cache.preload.ai.VectorStore -import io.milvus.client.MilvusClient +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusClient import org.slf4j.LoggerFactory import java.time.LocalDateTime import java.time.ZoneId diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/PreloadConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/PreloadConfig.kt index 970a519462..12514e689c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/PreloadConfig.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/PreloadConfig.kt @@ -31,7 +31,7 @@ import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties import com.tencent.bkrepo.common.artifact.cache.service.ArtifactPreloadPlanGenerator import com.tencent.bkrepo.job.batch.task.cache.preload.ai.AiProperties import com.tencent.bkrepo.job.batch.task.cache.preload.ai.EmbeddingModel -import io.milvus.client.MilvusClient +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusClient import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusVectorStore.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusVectorStore.kt deleted file mode 100644 index 2fe6da4b49..0000000000 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusVectorStore.kt +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.job.batch.task.cache.preload.ai - -import com.alibaba.fastjson.JSONObject -import io.milvus.client.MilvusClient -import io.milvus.common.clientenum.ConsistencyLevelEnum -import io.milvus.grpc.DataType -import io.milvus.param.MetricType -import io.milvus.param.R -import io.milvus.param.collection.CreateCollectionParam -import io.milvus.param.collection.DropCollectionParam -import io.milvus.param.collection.FieldType -import io.milvus.param.collection.FlushParam -import io.milvus.param.collection.HasCollectionParam -import io.milvus.param.collection.LoadCollectionParam -import io.milvus.param.dml.DeleteParam -import io.milvus.param.dml.InsertParam -import io.milvus.param.dml.SearchParam -import io.milvus.param.index.CreateIndexParam -import io.milvus.param.index.DescribeIndexParam -import io.milvus.response.QueryResultsWrapper -import io.milvus.response.SearchResultsWrapper -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.InitializingBean - -/** - * 升级到jdk17后迁移到spring-ai - */ -class MilvusVectorStore( - private val config: MilvusVectorStoreProperties, - private val milvusClient: MilvusClient, - private val embeddingModel: EmbeddingModel, -) : VectorStore, InitializingBean { - - override fun insert(documents: List) { - val docIdArray = ArrayList() - val contentArray = ArrayList() - val metadataArray = ArrayList() - val embeddingArray = embeddingModel.embed(documents.map { it.content }) - - for (document in documents) { - docIdArray.add(document.id) - // Use a (future) DocumentTextLayoutFormatter instance to extract - // the content used to compute the embeddings - contentArray.add(document.content) - metadataArray.add(JSONObject(document.metadata)) - } - - val fields: MutableList = ArrayList() - fields.add(InsertParam.Field(DOC_ID_FIELD_NAME, docIdArray)) - fields.add(InsertParam.Field(CONTENT_FIELD_NAME, contentArray)) - fields.add(InsertParam.Field(METADATA_FIELD_NAME, metadataArray)) - fields.add(InsertParam.Field(EMBEDDING_FIELD_NAME, embeddingArray)) - - val insertParam = InsertParam.newBuilder() - .withDatabaseName(config.databaseName) - .withCollectionName(config.collectionName) - .withFields(fields) - .build() - - val status = milvusClient.insert(insertParam) - if (status.exception != null) { - throw RuntimeException("Failed to insert:", status.exception) - } - milvusClient.flush( - FlushParam.newBuilder() - .withDatabaseName(config.databaseName) - .addCollectionName(config.collectionName) - .build() - ) - } - - override fun delete(ids: Set): Boolean { - val deleteExpression = "$DOC_ID_FIELD_NAME in [${ids.joinToString(",") { "'$it'" }}]" - val status = milvusClient.delete( - DeleteParam.newBuilder() - .withCollectionName(config.collectionName) - .withExpr(deleteExpression) - .build() - ) - - val deleteCount = status.data.deleteCnt - if (deleteCount != ids.size.toLong()) { - logger.warn(String.format("Deleted only %s entries from requested %s ", deleteCount, ids.size)) - } - - return status.status == R.Status.Success.code - } - - override fun similaritySearch(request: SearchRequest): List { - val nativeFilterExpressions = request.filterExpression ?: "" - val embedding: List = embeddingModel.embed(request.query) - - val searchParamBuilder = SearchParam.newBuilder() - .withDatabaseName(config.databaseName) - .withCollectionName(config.collectionName) - .withConsistencyLevel(ConsistencyLevelEnum.STRONG) - .withMetricType(config.metricType) - .withOutFields(SEARCH_OUTPUT_FIELDS) - .withTopK(request.topK) - .withFloatVectors(listOf(embedding)) - .withVectorFieldName(EMBEDDING_FIELD_NAME) - - if (nativeFilterExpressions.isNotBlank()) { - searchParamBuilder.withExpr(nativeFilterExpressions) - } - - val respSearch = milvusClient.search(searchParamBuilder.build()) - - if (respSearch.exception != null) { - throw RuntimeException("Search failed!", respSearch.exception) - } - - val wrapperSearch = SearchResultsWrapper(respSearch.data.results) - - return wrapperSearch.getRowRecords(0) - .filter { getResultSimilarity(it) >= request.similarityThreshold } - .map { rowRecord -> - val docId = rowRecord[DOC_ID_FIELD_NAME] as String - val content = rowRecord[CONTENT_FIELD_NAME] as String - val metadata = rowRecord[METADATA_FIELD_NAME] as JSONObject - // inject the distance into the metadata. - metadata[DISTANCE_FIELD_NAME] = 1 - getResultSimilarity(rowRecord) - Document(content, metadata.innerMap, docId) - } - .toList() - } - - private fun getResultSimilarity(rowRecord: QueryResultsWrapper.RowRecord): Float { - val distance = rowRecord[DISTANCE_FIELD_NAME] as Float - return if ((config.metricType == MetricType.IP || config.metricType == MetricType.COSINE)) { - distance - } else { - (1 - distance) - } - } - - override fun afterPropertiesSet() { - this.createCollection() - } - - override fun collectionExists(): Boolean { - return milvusClient.hasCollection( - HasCollectionParam.newBuilder() - .withDatabaseName(config.databaseName) - .withCollectionName(config.collectionName) - .build() - ).data - } - - override fun collectionName(): String { - return config.collectionName - } - - // used by the test as well - override fun createCollection(): Boolean { - var created = false - if (!collectionExists()) { - val docIdFieldType = FieldType.newBuilder() - .withName(DOC_ID_FIELD_NAME) - .withDataType(DataType.VarChar) - .withMaxLength(36) - .withPrimaryKey(true) - .withAutoID(false) - .build() - val contentFieldType = FieldType.newBuilder() - .withName(CONTENT_FIELD_NAME) - .withDataType(DataType.VarChar) - .withMaxLength(65535) - .build() - val metadataFieldType = FieldType.newBuilder() - .withName(METADATA_FIELD_NAME) - .withDataType(DataType.JSON) - .build() - val embeddingFieldType = FieldType.newBuilder() - .withName(EMBEDDING_FIELD_NAME) - .withDataType(DataType.FloatVector) - .withDimension(this.embeddingDimensions()) - .build() - - val createCollectionReq = CreateCollectionParam.newBuilder() - .withDatabaseName(this.config.databaseName) - .withCollectionName(this.config.collectionName) - .withDescription("Spring AI Vector Store") - .withConsistencyLevel(ConsistencyLevelEnum.STRONG) - .withShardsNum(2) - .addFieldType(docIdFieldType) - .addFieldType(contentFieldType) - .addFieldType(metadataFieldType) - .addFieldType(embeddingFieldType) - .build() - - val collectionStatus = milvusClient.createCollection(createCollectionReq) - if (collectionStatus.exception != null) { - throw RuntimeException("Failed to create collection", collectionStatus.exception) - } - created = true - } - - val indexDescriptionResponse = milvusClient - .describeIndex( - DescribeIndexParam.newBuilder() - .withDatabaseName(this.config.databaseName) - .withCollectionName(this.config.collectionName) - .build() - ) - - if (indexDescriptionResponse.data == null) { - val indexStatus = milvusClient.createIndex( - CreateIndexParam.newBuilder() - .withDatabaseName(this.config.databaseName) - .withCollectionName(this.config.collectionName) - .withFieldName(EMBEDDING_FIELD_NAME) - .withIndexType(this.config.indexType) - .withMetricType(this.config.metricType) - .withExtraParam(this.config.indexParameters) - .withSyncMode(false) - .build() - ) - - if (indexStatus.exception != null) { - throw RuntimeException("Failed to create Index", indexStatus.exception) - } - } - - val loadCollectionStatus = milvusClient.loadCollection( - LoadCollectionParam.newBuilder() - .withDatabaseName(this.config.databaseName) - .withCollectionName(this.config.collectionName) - .build() - ) - - if (loadCollectionStatus.exception != null) { - throw RuntimeException("Collection loading failed!", loadCollectionStatus.exception) - } - return created - } - - override fun dropCollection() { - val exception = milvusClient.dropCollection( - DropCollectionParam.newBuilder() - .withDatabaseName(config.databaseName) - .withCollectionName(config.collectionName) - .build() - ).exception - if (exception != null) { - throw RuntimeException("Failed to drop collection[${config.collectionName}]", exception) - } - } - - private fun embeddingDimensions(): Int { - if (config.embeddingDimension != INVALID_EMBEDDING_DIMENSION) { - return config.embeddingDimension - } - return embeddingModel.dimensions() - } - - companion object { - private val logger = LoggerFactory.getLogger(MilvusVectorStore::class.java) - - const val INVALID_EMBEDDING_DIMENSION: Int = -1 - const val DOC_ID_FIELD_NAME: String = "doc_id" - const val CONTENT_FIELD_NAME: String = "content" - const val METADATA_FIELD_NAME: String = "metadata" - const val EMBEDDING_FIELD_NAME: String = "embedding" - - // Metadata, automatically assigned by Milvus. - const val DISTANCE_FIELD_NAME: String = "distance" - - val SEARCH_OUTPUT_FIELDS = listOf(DOC_ID_FIELD_NAME, CONTENT_FIELD_NAME, METADATA_FIELD_NAME) - } -} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/SpringAiConfiguration.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/SpringAiConfiguration.kt index 3772bfed2d..932c20d90e 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/SpringAiConfiguration.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/SpringAiConfiguration.kt @@ -27,14 +27,12 @@ package com.tencent.bkrepo.job.batch.task.cache.preload.ai -import io.milvus.client.MilvusClient -import io.milvus.client.MilvusServiceClient -import io.milvus.param.ConnectParam +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusClient +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.MilvusClientProperties import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration -import java.util.concurrent.TimeUnit @Configuration @EnableConfigurationProperties( @@ -51,26 +49,6 @@ class SpringAiConfiguration { @Bean fun milvusClient(properties: MilvusClientProperties): MilvusClient { - with(properties) { - val builder = ConnectParam.newBuilder() - .withHost(host) - .withPort(port) - .withUri(uri) - .withConnectTimeout(connectTimeoutMs, TimeUnit.MILLISECONDS) - .withKeepAliveTime(keepAliveTimeMs, TimeUnit.MILLISECONDS) - .withKeepAliveTimeout(keepAliveTimeoutMs, TimeUnit.MILLISECONDS) - .withRpcDeadline(rpcDeadlineMs, TimeUnit.MILLISECONDS) - .withIdleTimeout(idleTimeoutMs, TimeUnit.MILLISECONDS) - - if (username.isNotEmpty() && password.isNotEmpty()) { - builder.withAuthorization(username, password) - } - - if (!token.isNullOrEmpty()) { - builder.withToken(token) - } - - return MilvusServiceClient(builder.build()) - } + return MilvusClient(properties) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusClient.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusClient.kt new file mode 100644 index 0000000000..a3bf519364 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusClient.kt @@ -0,0 +1,139 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus + +import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.bkrepo.common.api.constant.HttpHeaders +import com.tencent.bkrepo.common.api.constant.MediaTypes +import com.tencent.bkrepo.common.api.util.JsonUtils +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.storage.innercos.http.toRequestBody +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.CreateCollectionReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.DeleteVectorReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.DropCollectionReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.HasCollectionReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.InsertVectorReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.LoadCollectionReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.SearchVectorReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.response.HasCollectionRes +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.response.MilvusResponse +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.internal.headersContentLength +import org.slf4j.LoggerFactory + +class MilvusClient( + private val clientProperties: MilvusClientProperties +) { + val client = OkHttpClient.Builder().addInterceptor { + val newReq = it.request().newBuilder() + + // add token to header + with(clientProperties) { + val token = if (username.isNotEmpty() && password.isNotEmpty()) { + "Bearer ${username}:${password}" + } else if (!clientProperties.token.isNullOrEmpty()) { + "Bearer ${clientProperties.token}" + } else { + null + } + token?.let { t -> newReq.header(HttpHeaders.AUTHORIZATION, t) } + } + + it.proceed(newReq.build()) + }.build() + + fun createCollection(req: CreateCollectionReq) { + val request = Request.Builder() + .post(req.toJsonString().toRequestBody(APPLICATION_JSON)) + .url("${clientProperties.uri}/v2/vectordb/collections/create") + .build() + client.newCall(request).execute().throwIfFailed { + logger.info("create collection[${req.collectionName}] success") + } + } + + fun collectionExists(dbName: String, collectionName: String): Boolean { + val request = Request.Builder() + .url("${clientProperties.uri}/v2/vectordb/collections/has") + .post(HasCollectionReq(dbName, collectionName).toJsonString().toRequestBody(APPLICATION_JSON)) + .build() + return client.newCall(request).execute().throwIfFailed { it!!.has } + } + + fun dropCollection(dbName: String, collectionName: String) { + val request = Request.Builder() + .url("${clientProperties.uri}/v2/vectordb/collections/drop") + .post(DropCollectionReq(dbName, collectionName).toJsonString().toRequestBody(APPLICATION_JSON)) + .build() + client.newCall(request).execute().throwIfFailed { + logger.info("drop collection[${collectionName}] success") + } + } + + fun loadCollection(dbName: String, collectionName: String) { + val request = Request.Builder() + .url("${clientProperties.uri}/v2/vectordb/collections/load") + .post(LoadCollectionReq(dbName, collectionName).toJsonString().toRequestBody(APPLICATION_JSON)) + .build() + client.newCall(request).execute().throwIfFailed { + logger.info("load collection[${collectionName}] success") + } + } + + fun insert(req: InsertVectorReq) { + val request = Request.Builder() + .url("${clientProperties.uri}/v2/vectordb/entities/insert") + .post(req.toJsonString().toRequestBody(APPLICATION_JSON)) + .build() + client.newCall(request).execute().throwIfFailed { + logger.info("insert ${req.data.size} data into [${req.collectionName}] success") + } + } + + fun delete(req: DeleteVectorReq) { + val request = Request.Builder() + .url("${clientProperties.uri}/v2/vectordb/entities/delete") + .post(req.toJsonString().toRequestBody(APPLICATION_JSON)) + .build() + client.newCall(request).execute().throwIfFailed { + logger.info("delete vector of [${req.dbName}/${req.collectionName}] success") + } + } + + fun search(req: SearchVectorReq): List> { + val request = Request.Builder() + .url("${clientProperties.uri}/v2/vectordb/entities/search") + .post(req.toJsonString().toRequestBody(APPLICATION_JSON)) + .build() + return client.newCall(request).execute().throwIfFailed>, List>> { it!! } + } + + private inline fun Response.throwIfFailed(handler: (data: T?) -> R): R { + use { + if (isSuccessful) { + val res = JsonUtils.objectMapper.readValue( + body!!.byteStream(), + object : TypeReference>() {} + ) + if (res.code != 0) { + throw RuntimeException("request milvus failed, code: $code, message: ${res.message}") + } + return handler(res.data) + } else { + val message = if (headersContentLength() < DEFAULT_MESSAGE_LIMIT) { + body?.string() + } else { + "" + } + throw RuntimeException("request milvus failed, code: $code, message: $message") + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(MilvusClient::class.java) + private const val DEFAULT_MESSAGE_LIMIT = 4096 + private val APPLICATION_JSON = MediaTypes.APPLICATION_JSON.toMediaType() + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusClientProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusClientProperties.kt similarity index 84% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusClientProperties.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusClientProperties.kt index 4d078c9c44..0b2349f13f 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusClientProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusClientProperties.kt @@ -25,10 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.job.batch.task.cache.preload.ai +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus import org.springframework.boot.context.properties.ConfigurationProperties -import java.util.concurrent.TimeUnit @ConfigurationProperties("spring.ai.vectorstore.milvus.client") data class MilvusClientProperties( @@ -36,11 +35,6 @@ data class MilvusClientProperties( var port: Int = 19530, var uri: String? = null, var token: String? = null, - var connectTimeoutMs: Long = 10000L, - var keepAliveTimeMs: Long = 55000L, - var keepAliveTimeoutMs: Long = 20000L, - var rpcDeadlineMs: Long = 0L, - var idleTimeoutMs: Long = TimeUnit.MILLISECONDS.convert(24L, TimeUnit.HOURS), var username: String = "root", var password: String = "milvus" ) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusVectorStore.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusVectorStore.kt new file mode 100644 index 0000000000..b5ed0afc52 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusVectorStore.kt @@ -0,0 +1,244 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus + +import cn.hutool.core.codec.Base64Decoder +import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.Document +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.EmbeddingModel +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.SearchRequest +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.VectorStore +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.CollectionSchema +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.ConsistencyLevel +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.CreateCollectionReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.DataType +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.DeleteVectorReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.ElementTypeParams +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.FieldSchema +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.IndexParam +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.InsertVectorReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.MetricType +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.Params +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.SearchParams +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.SearchVectorReq +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.Vector +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.InitializingBean + +/** + * 升级到jdk17后迁移到spring-ai + */ +class MilvusVectorStore( + private val config: MilvusVectorStoreProperties, + private val milvusClient: MilvusClient, + private val embeddingModel: EmbeddingModel, +) : VectorStore, InitializingBean { + + override fun insert(documents: List) { + val embeddingArray = embeddingModel.embed(documents.map { it.content }) + val insertRequest = InsertVectorReq( + dbName = config.databaseName, + collectionName = config.collectionName, + data = documents.mapIndexed { i, d -> + Vector( + d.id, + d.content, + embeddingArray[i], + d.metadata.toJsonString() + ) + } + ) + milvusClient.insert(insertRequest) + } + + override fun delete(ids: Set): Boolean { + val deleteExpression = "$DOC_ID_FIELD_NAME in [${ids.joinToString(",") { "'$it'" }}]" + val req = DeleteVectorReq( + dbName = config.databaseName, + collectionName = config.collectionName, + filter = deleteExpression + ) + milvusClient.delete(req) + return true + } + + override fun similaritySearch(request: SearchRequest): List { + val nativeFilterExpressions = request.filterExpression ?: "" + val embedding: List = embeddingModel.embed(request.query) + + val req = SearchVectorReq( + dbName = config.databaseName, + collectionName = config.collectionName, + data = listOf(embedding), + limit = request.topK, + outputFields = SEARCH_OUTPUT_FIELDS, + annsField = EMBEDDING_FIELD_NAME, + filter = nativeFilterExpressions, + searchParams = SearchParams( + metricType = config.metricType, + ) + ) + + val results = ArrayList() + val respSearch = milvusClient.search(req) + for (vector in respSearch) { + val similarity = getResultSimilarity((vector[DISTANCE_FIELD_NAME] as Double).toFloat()) + if (similarity < request.similarityThreshold) { + continue + } + val docId = vector[DOC_ID_FIELD_NAME] as String + val content = vector[CONTENT_FIELD_NAME] as String + val metadata = Base64Decoder + .decodeStr(vector[METADATA_FIELD_NAME] as String) + .readJsonString>() + // inject the distance into the metadata. + metadata[DISTANCE_FIELD_NAME] = 1 - similarity + results.add(Document(content, metadata, docId)) + } + return results + } + + private fun getResultSimilarity(distance: Float): Float { + return if ((config.metricType == MetricType.IP.name || config.metricType == MetricType.COSINE.name)) { + distance + } else { + (1 - distance) + } + } + + override fun afterPropertiesSet() { + this.createCollection() + } + + override fun collectionExists(): Boolean { + return milvusClient.collectionExists(config.databaseName, config.collectionName) + } + + override fun collectionName(): String { + return config.collectionName + } + + // used by the test as well + override fun createCollection(): Boolean { + var created = false + if (!collectionExists()) { + val collectionSchema = CollectionSchema(enableDynamicField = false) + collectionSchema.addField( + FieldSchema( + fieldName = DOC_ID_FIELD_NAME, + dataType = DataType.VarChar.name, + isPrimary = true, + elementTypeParams = ElementTypeParams(maxLength = 36), + ) + ) + collectionSchema.addField( + FieldSchema( + fieldName = CONTENT_FIELD_NAME, + dataType = DataType.VarChar.name, + elementTypeParams = ElementTypeParams(maxLength = 65535), + ) + ) + collectionSchema.addField( + FieldSchema( + fieldName = METADATA_FIELD_NAME, + dataType = DataType.JSON.name, + ) + ) + collectionSchema.addField( + FieldSchema( + fieldName = EMBEDDING_FIELD_NAME, + dataType = DataType.FloatVector.name, + elementTypeParams = ElementTypeParams(dim = embeddingDimensions()) + ) + ) + + val indexParams = ArrayList() + indexParams.add( + IndexParam( + indexName = null, + metricType = config.metricType, + fieldName = EMBEDDING_FIELD_NAME, + params = Params( + indexType = config.indexType, + nlist = config.nList, + ) + ) + ) + + val createCollectionReq = CreateCollectionReq( + dbName = config.databaseName, + collectionName = config.collectionName, + dimension = embeddingDimensions(), + metricType = config.metricType, + idType = DataType.VarChar, + autoId = false, + primaryFieldName = DOC_ID_FIELD_NAME, + vectorFieldName = EMBEDDING_FIELD_NAME, + schema = collectionSchema, + indexParams = indexParams, + params = CreateCollectionReq.Params( + consistencyLevel = ConsistencyLevel.Strong.name, + shardsNum = 2 + ) + ) + + milvusClient.createCollection(createCollectionReq) + created = true + } + + milvusClient.loadCollection(config.databaseName, config.collectionName) + return created + } + + override fun dropCollection() { + milvusClient.dropCollection(config.databaseName, config.collectionName) + } + + private fun embeddingDimensions(): Int { + if (config.embeddingDimension != INVALID_EMBEDDING_DIMENSION) { + return config.embeddingDimension + } + return embeddingModel.dimensions() + } + + companion object { + private val logger = LoggerFactory.getLogger(MilvusVectorStore::class.java) + + const val INVALID_EMBEDDING_DIMENSION: Int = -1 + const val DOC_ID_FIELD_NAME: String = "doc_id" + const val CONTENT_FIELD_NAME: String = "content" + const val METADATA_FIELD_NAME: String = "metadata" + const val EMBEDDING_FIELD_NAME: String = "embedding" + + // Metadata, automatically assigned by Milvus. + const val DISTANCE_FIELD_NAME: String = "distance" + + val SEARCH_OUTPUT_FIELDS = listOf(DOC_ID_FIELD_NAME, CONTENT_FIELD_NAME, METADATA_FIELD_NAME) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusVectorStoreProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusVectorStoreProperties.kt similarity index 82% rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusVectorStoreProperties.kt rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusVectorStoreProperties.kt index 06cbb1cf70..d6156ce372 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/MilvusVectorStoreProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/MilvusVectorStoreProperties.kt @@ -25,16 +25,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.job.batch.task.cache.preload.ai +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus -import io.milvus.param.IndexType -import io.milvus.param.MetricType +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.IndexType +import com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request.MetricType data class MilvusVectorStoreProperties( var databaseName: String = "default", var collectionName: String = "vector_store", var embeddingDimension: Int = 1536, - var indexType: IndexType = IndexType.IVF_FLAT, - var metricType: MetricType = MetricType.COSINE, + var indexType: String = IndexType.IVF_FLAT.name, + var metricType: String = MetricType.COSINE.name, var indexParameters: String = "{\"nlist\":1024}", + var nList: Int = 1024, ) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/CollectionSchema.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/CollectionSchema.kt new file mode 100644 index 0000000000..48932aa3ca --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/CollectionSchema.kt @@ -0,0 +1,39 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonProperty + +data class CollectionSchema( + val enableDynamicField: Boolean = true, + val fields: MutableList = ArrayList(), + @JsonProperty("autoID") + val autoId: Boolean = false, +) { + fun addField(fieldSchema: FieldSchema): CollectionSchema { + if (fieldSchema.dataType == DataType.Array.name) { + if (fieldSchema.elementDataType.isNullOrEmpty() || fieldSchema.elementTypeParams?.maxCapacity == null) { + throw IllegalArgumentException("Element type, maxCapacity are required for array field") + } + } + + val isVectorType = fieldSchema.dataType == DataType.FloatVector.name || + fieldSchema.dataType == DataType.BinaryVector.name || + fieldSchema.dataType == DataType.Float16Vector.name || + fieldSchema.dataType == DataType.BFloat16Vector.name + + if (isVectorType && fieldSchema.elementTypeParams?.dim == null) { + throw IllegalArgumentException("Dimension is required for vector field") + } + + fields.add(fieldSchema) + return this + } + + fun getField(fieldName: String): FieldSchema? { + for (field in fields) { + if (field.fieldName == fieldName) { + return field + } + } + return null + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/ConsistencyLevel.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/ConsistencyLevel.kt new file mode 100644 index 0000000000..395cff0cbd --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/ConsistencyLevel.kt @@ -0,0 +1,7 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +enum class ConsistencyLevel(private val code: Int) { + Strong(0), + Bounded(2), + Eventually(3), +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/CreateCollectionReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/CreateCollectionReq.kt new file mode 100644 index 0000000000..9315e10279 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/CreateCollectionReq.kt @@ -0,0 +1,32 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonInclude(Include.NON_NULL) +data class CreateCollectionReq( + val dbName: String, + val collectionName: String, + val dimension: Int, + val metricType: String = MetricType.COSINE.name, + val idType: DataType = DataType.Int64, + @JsonProperty("autoID") + val autoId: Boolean = false, + val primaryFieldName: String = "id", + val vectorFieldName: String = "vector", + val schema: CollectionSchema? = null, + val indexParams: List = ArrayList(), + val params: Params? = null, +) { + @JsonInclude(Include.NON_NULL) + data class Params( + @JsonProperty("max_length") + val maxLength: Int? = null, + val enableDynamicField: Boolean? = null, + val shardsNum: Int? = null, + val consistencyLevel: String? = ConsistencyLevel.Bounded.name, + val partitionsNum: Int? = null, + val ttlSeconds: Int? = null, + ) +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DataType.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DataType.kt new file mode 100644 index 0000000000..363dbb9eb1 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DataType.kt @@ -0,0 +1,24 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +enum class DataType(private val code: Int) { + None(0), + Bool(1), + Int8(2), + Int16(3), + Int32(4), + Int64(5), + + Float(10), + Double(11), + + String(20), + VarChar(21), // variable-length strings with a specified maximum length + Array(22), + JSON(23), + + BinaryVector(100), + FloatVector(101), + Float16Vector(102), + BFloat16Vector(103), + SparseFloatVector(104) +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DeleteVectorReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DeleteVectorReq.kt new file mode 100644 index 0000000000..bee5ccc361 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DeleteVectorReq.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include + +@JsonInclude(Include.NON_NULL) +data class DeleteVectorReq( + val dbName: String, + val collectionName: String, + val filter: String, + val partitionName: String? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DropCollectionReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DropCollectionReq.kt new file mode 100644 index 0000000000..ac5cc032ad --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/DropCollectionReq.kt @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +data class DropCollectionReq( + val dbName: String, + val collectionName: String +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/FieldSchema.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/FieldSchema.kt new file mode 100644 index 0000000000..780a3f6f2d --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/FieldSchema.kt @@ -0,0 +1,37 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonInclude(Include.NON_NULL) +data class FieldSchema ( + val fieldName: String, + val dataType: String, + val isPrimary: Boolean = false, + val isPartitionKey: Boolean = false, + /** + * An optional parameter for Array field values + */ + val elementDataType: String? = null, + val elementTypeParams: ElementTypeParams? = null +) + +@JsonInclude(Include.NON_NULL) +data class ElementTypeParams( + /** + * An optional parameter for VarChar values that determines the maximum length of the value in the current field. + */ + @JsonProperty("max_length") + val maxLength: Int? = 65535, + /** + * An optional parameter for FloatVector or BinaryVector fields that determines the vector dimension. + */ + val dim: Int? = null, + /** + * An optional parameter for Array field values that + * determines the maximum number of elements in the current array field. + */ + @JsonProperty("max_capacity") + val maxCapacity: Int? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/HasCollectionReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/HasCollectionReq.kt new file mode 100644 index 0000000000..167b3d1bf1 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/HasCollectionReq.kt @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +data class HasCollectionReq( + val dbName: String, + val collectionName: String +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/IndexParam.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/IndexParam.kt new file mode 100644 index 0000000000..5e2e539837 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/IndexParam.kt @@ -0,0 +1,32 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonInclude(Include.NON_NULL) +data class IndexParam( + var fieldName: String, + val metricType: String = MetricType.COSINE.name, + val indexName: String? = null, + val params: Params? = null, +) + +@JsonInclude(Include.NON_NULL) +data class Params( + @JsonProperty("index_type") + val indexType: String = IndexType.AUTOINDEX.name, + /** + * The maximum degree of the node and applies only when index_type is set to HNSW. + */ + @JsonProperty("M") + val m: Int? = null, + /** + * The search scope. This applies only when index_type is set to HNSW + */ + val efConstruction: Int? = null, + /** + * The number of cluster units. This applies to IVF-related index types. + */ + val nlist: Int? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/IndexType.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/IndexType.kt new file mode 100644 index 0000000000..d62599c5e5 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/IndexType.kt @@ -0,0 +1,39 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +enum class IndexType( + val code: Int, +) { + None(0), + + // Only supported for float vectors + FLAT(1), + IVF_FLAT(2), + IVF_SQ8(3), + IVF_PQ(4), + HNSW(5), + DISKANN(10), + AUTOINDEX(11), + SCANN(12), + + // GPU indexes only for float vectors + GPU_IVF_FLAT(50), + GPU_IVF_PQ(51), + GPU_BRUTE_FORCE(52), + GPU_CAGRA(53), + + // Only supported for binary vectors + BIN_FLAT(80), + BIN_IVF_FLAT(81), + + // Only for varchar type field + Trie(100), + + // Only for scalar type field + STL_SORT(200), // only for numeric type field + INVERTED(201), // works for all scalar fields except JSON type field + BITMAP(202), // works for all scalar fields except JSON, FLOAT and DOUBLE type fields + + // Only for sparse vectors + SPARSE_INVERTED_INDEX(300), + SPARSE_WAND(301); +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/InsertVectorReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/InsertVectorReq.kt new file mode 100644 index 0000000000..7ac1afc618 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/InsertVectorReq.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include + +@JsonInclude(Include.NON_NULL) +data class InsertVectorReq( + val dbName: String, + val collectionName: String, + val data: List, + val partitionName: String? = null +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/LoadCollectionReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/LoadCollectionReq.kt new file mode 100644 index 0000000000..20a4a2acb4 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/LoadCollectionReq.kt @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +data class LoadCollectionReq( + val dbName: String, + val collectionName: String, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/MetricType.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/MetricType.kt new file mode 100644 index 0000000000..27d7d8dfcf --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/MetricType.kt @@ -0,0 +1,14 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +enum class MetricType { + INVALID, + + // Only for float vectors + L2, + IP, + COSINE, + + // Only for binary vectors + HAMMING, + JACCARD, +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/SearchVectorReq.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/SearchVectorReq.kt new file mode 100644 index 0000000000..8c547b853b --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/SearchVectorReq.kt @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.annotation.JsonProperty + +@JsonInclude(Include.NON_NULL) +data class SearchVectorReq( + val dbName: String, + val collectionName: String, + val data: List>, + val annsField: String, + + val limit: Int? = null, + val offset: Int? = null, + val groupingField: String? = null, + val outputFields: List? = null, + + val filter: String? = null, + val searchParams: SearchParams? = null, + val partitionNames: List? = null, +) + +data class SearchParams( + val metricType: String? = null, + val params: VectorSearchParams? = null, +) + +data class VectorSearchParams( + val radius: Int? = null, + @JsonProperty("range_filter") + val rangeFilter: Int? = null, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/Vector.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/Vector.kt new file mode 100644 index 0000000000..9cfb6dc219 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/request/Vector.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.request + +import com.fasterxml.jackson.annotation.JsonProperty + +data class Vector( + @JsonProperty("doc_id") + val docId: String, + /** + * 用于向量化的文档内容 + */ + val content: String, + /** + * content向量化后的内容 + */ + val embedding: List, + /** + * 携带的元数据经过json序列化后的字符串 + */ + val metadata: String, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/response/HasCollectionRes.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/response/HasCollectionRes.kt new file mode 100644 index 0000000000..b8679908af --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/response/HasCollectionRes.kt @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.response + +data class HasCollectionRes( + val has: Boolean +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/response/MilvusResponse.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/response/MilvusResponse.kt new file mode 100644 index 0000000000..24446186f6 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ai/milvus/response/MilvusResponse.kt @@ -0,0 +1,7 @@ +package com.tencent.bkrepo.job.batch.task.cache.preload.ai.milvus.response + +data class MilvusResponse( + val code: Int, + val message: String? = null, + val data: T? = null +)