From 6b82f52b08d0fe71025ea6d3c0e87464cdd91e94 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:17:28 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=20=E5=88=86=E5=8F=91=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=B7=A8=E5=AD=98=E5=82=A8=E6=96=87=E4=BB=B6=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=20#2278?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 过滤集群详情中部分字段 #2278 * feat: 添加跨存储文件读取 #2278 --- .../pojo/cluster/ClusterNodeInfo.kt | 1 + .../replication/manager/LocalDataManager.kt | 37 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/cluster/ClusterNodeInfo.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/cluster/ClusterNodeInfo.kt index a2149d25aa..91b8297f39 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/cluster/ClusterNodeInfo.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/cluster/ClusterNodeInfo.kt @@ -63,6 +63,7 @@ data class ClusterNodeInfo( @ApiModelProperty("集群访问凭证", required = false) var accessKey: String? = null, @ApiModelProperty("集群密钥", required = false) + @JsonIgnore var secretKey: String? = null, @ApiModelProperty("udp端口", required = false) var udpPort: Int? = null, diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt index 6c5fe9215c..78ab787876 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/manager/LocalDataManager.kt @@ -34,7 +34,9 @@ import com.tencent.bkrepo.common.artifact.exception.ProjectNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.replication.constant.MD5 import com.tencent.bkrepo.replication.constant.NODE_FULL_PATH @@ -43,6 +45,7 @@ import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.ProjectClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.packages.PackageListOption @@ -65,14 +68,18 @@ class LocalDataManager( private val repositoryClient: RepositoryClient, private val nodeClient: NodeClient, private val packageClient: PackageClient, - private val storageService: StorageService + private val storageService: StorageService, + private val storageCredentialsClient: StorageCredentialsClient, + private val storageProperties: StorageProperties, ) { /** * 获取blob文件数据 */ fun getBlobData(sha256: String, length: Long, repoInfo: RepositoryDetail): InputStream { - return storageService.load(sha256, Range.full(length), repoInfo.storageCredentials) + val range = Range.full(length) + return storageService.load(sha256, range, repoInfo.storageCredentials) + ?: loadFromOtherStorage(sha256, range, repoInfo.storageCredentials) ?: throw ArtifactNotFoundException(sha256) } @@ -81,9 +88,30 @@ class LocalDataManager( */ fun getBlobDataByRange(sha256: String, range: Range, repoInfo: RepositoryDetail): InputStream { return storageService.load(sha256, range, repoInfo.storageCredentials) + ?: loadFromOtherStorage(sha256, range, repoInfo.storageCredentials) ?: throw ArtifactNotFoundException(sha256) } + /** + * 节点可能是从其他仓库复制过来，仓库存储不一样，对应文件还没有复制 + */ + private fun loadFromOtherStorage( + sha256: String, range: Range, + currentStorageCredentials: StorageCredentials? + ): InputStream? { + val allCredentials = storageCredentialsClient.list().data!! + storageProperties.defaultStorageCredentials() + var result: InputStream? = null + for (credential in allCredentials) { + val key = credential.key + if (key == currentStorageCredentials?.key) continue + result = storageService.load(sha256, range, credential) + if (result != null) { + break + } + } + return result + } + /** * 查找项目 * 项目不存在抛异常 @@ -161,8 +189,9 @@ class LocalDataManager( projectId: String, repoName: String, fullPath: String ): NodeDetail { return findNode(projectId, repoName, fullPath) ?: findDeletedNodeDetail(projectId, repoName, fullPath) - ?: throw NodeNotFoundException(fullPath) + ?: throw NodeNotFoundException(fullPath) } + fun findDeletedNodeDetail( projectId: String, repoName: String, fullPath: String ): NodeDetail? { @@ -202,7 +231,7 @@ class LocalDataManager( ) } -/** + /** * 分页查询包 */ fun listPackagePage(projectId: String, repoName: String, option: PackageListOption): List {