From 87ea527bc3d91094c2a12b2bd8ae179b7f54838e Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Fri, 16 Aug 2024 10:35:56 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20pypi=20=E6=9F=A5=E8=AF=A2=E5=AF=B9?= =?UTF-8?q?=E5=BA=94=E5=8C=85=E7=9A=84=E6=96=87=E4=BB=B6=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E4=BC=9A=E5=AF=BC=E8=87=B4db=E8=B4=9F=E8=BD=BD=E4=B8=8A?= =?UTF-8?q?=E5=8D=87=20#2472?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/pypi/artifact/PypiProperties.kt | 3 +- .../repository/PypiLocalRepository.kt | 107 ++++++++++++++---- 2 files changed, 84 insertions(+), 26 deletions(-) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/PypiProperties.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/PypiProperties.kt index 633d85a464..4d08dfb06f 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/PypiProperties.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/PypiProperties.kt @@ -4,5 +4,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "pypi") data class PypiProperties( - var domain: String = "localhost" + var domain: String = "localhost", + var enableRegexQuery: Boolean = true, ) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt index 65c1401726..2c244cca76 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.query.model.Sort import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.pypi.artifact.PypiProperties import com.tencent.bkrepo.pypi.artifact.PypiSimpleArtifactInfo import com.tencent.bkrepo.pypi.artifact.url.UrlPatternUtil.parameterMaps import com.tencent.bkrepo.pypi.artifact.xml.Value @@ -90,7 +91,8 @@ import org.springframework.stereotype.Component @Component class PypiLocalRepository( - private val stageClient: StageClient + private val stageClient: StageClient, + private val pypiProperties: PypiProperties ) : LocalRepository() { /** @@ -324,30 +326,46 @@ class PypiLocalRepository( return buildPypiPageContent(PACKAGE_INDEX_TITLE, buildPackageListContent(nodeList)) } // 请求中带包名，返回对应包的文件列表。 - else { - // 客户端标准化包名规则：1.连续的[-_.]转换为单个"-"；2.转换为全小写。此处需要以反向规则查询 - var pageNumber = 1 - val nodeList = mutableListOf>() - do { - val queryModel = NodeQueryBuilder() - .select(NAME, FULL_PATH, METADATA, SHA256) - .sortByAsc(NAME) - .page(pageNumber, PAGE_SIZE) - .projectId(projectId) - .repoName(repoName) - .path("^/${packageName!!.replace("-", NON_ALPHANUMERIC_SEQ_REGEX)}/", OperationType.REGEX_I) - .excludeFolder() - .build() - val records = nodeClient.queryWithoutCount(queryModel).data!!.records - nodeList.addAll(records) - pageNumber++ - } while (records.size == PAGE_SIZE) - nodeList.ifEmpty { throw PypiSimpleNotFoundException(packageName!!) } + val nodes = nodeClient.listNode( + projectId = projectId, + repoName = repoName, + path = "/$packageName", + includeFolder = false, + deep = true, + includeMetadata = true + ).data + if (!nodes.isNullOrEmpty()) { return buildPypiPageContent( String.format(VERSION_INDEX_TITLE, packageName), - buildPackageFileNodeListContent(nodeList) + buildPackageFileListContent(nodes) ) } + if (!pypiProperties.enableRegexQuery) { + throw PypiSimpleNotFoundException(packageName!!) + } + logger.info("not found nodeList by packageName[$packageName], use regex query") + // 客户端标准化包名规则：1.连续的[-_.]转换为单个"-"；2.转换为全小写。此处需要以反向规则查询 + var pageNumber = 1 + val nodeList = mutableListOf>() + do { + val queryModel = NodeQueryBuilder() + .select(NAME, FULL_PATH, METADATA, SHA256) + .sortByAsc(NAME) + .page(pageNumber, PAGE_SIZE) + .projectId(projectId) + .repoName(repoName) + .path("^/${packageName!!.replace("-", NON_ALPHANUMERIC_SEQ_REGEX)}/", OperationType.REGEX_I) + .excludeFolder() + .build() + val records = nodeClient.queryWithoutCount(queryModel).data!!.records + nodeList.addAll(records) + pageNumber++ + } while (records.size == PAGE_SIZE) + nodeList.ifEmpty { throw PypiSimpleNotFoundException(packageName!!) } + return buildPypiPageContent( + String.format(VERSION_INDEX_TITLE, packageName), + buildPackageFileNodeListContent(nodeList) + ) } } @@ -372,21 +390,60 @@ class PypiLocalRepository( ?.get("value").toString() ) } catch (ignore: IllegalArgumentException) { - SemVersion(0,0,0) + SemVersion(0, 0, 0) } } // data-requires-python属性值中的"<"和">"需要转换为HTML编码 sortedNodeList.forEachIndexed { i, node -> - val href = "../../packages${node[FULL_PATH]}#sha256=${node[SHA256]}" val requiresPython = (node[NODE_METADATA] as List>) .find { it["key"] == REQUIRES_PYTHON }?.get("value")?.toString()?.ifBlank { null } - ?.let { " $REQUIRES_PYTHON_ATTR=\"${HtmlUtils.partialEncode(it)}\"" } ?: "" - builder.append("$INDENT${node[NAME]}$LINE_BREAK") + builder.append( + buildPackageFileNodeLink( + fullPath = node[FULL_PATH].toString(), + name = node[NAME].toString(), + sha256 = node[SHA256]?.toString(), + requiresPython = requiresPython + ) + ) + if (i != nodeList.size - 1) builder.append("\n") + } + return builder.toString() + } + + /** + * 对应包中的文件列表 + * [nodeList] + */ + private fun buildPackageFileListContent(nodeList: List): String { + val builder = StringBuilder() + val sortedNodeList = nodeList.sortedBy { + try { + SemVersionParser.parse(it.metadata?.get("version").toString()) + } catch (ignore: IllegalArgumentException) { + SemVersion(0, 0, 0) + } + } + sortedNodeList.forEachIndexed { i, node -> + val requiresPython = node.nodeMetadata + ?.find { it.key == REQUIRES_PYTHON }?.value?.toString()?.ifBlank { null } + builder.append(buildPackageFileNodeLink(node.fullPath, node.name, node.sha256, requiresPython)) if (i != nodeList.size - 1) builder.append("\n") } return builder.toString() } + private fun buildPackageFileNodeLink( + fullPath: String, + name: String, + sha256: String?, + requiresPython: String? + ): String { + val href = "../../packages$fullPath#sha256=$sha256" + val requiresPythonAttr = requiresPython + ?.let { " $REQUIRES_PYTHON_ATTR=\"${HtmlUtils.partialEncode(it)}\"" } ?: "" + return "$INDENT$name$LINE_BREAK" + } + /** * 所有包列表 * @param nodeList