diff --git a/client/starwhale/base/client/models/models.py b/client/starwhale/base/client/models/models.py index 77c7563b1f..11a793bea2 100644 --- a/client/starwhale/base/client/models/models.py +++ b/client/starwhale/base/client/models/models.py @@ -1575,6 +1575,7 @@ class ModelVersionViewVo(SwBaseModel): latest: bool tags: Optional[List[str]] = None shared: int + draft: Optional[bool] = None step_specs: List[StepSpec] = Field(..., alias='stepSpecs') built_in_runtime: Optional[str] = Field(None, alias='builtInRuntime') created_time: int = Field(..., alias='createdTime') diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java index 1f72c01988..0fd2d4fe93 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java @@ -20,6 +20,7 @@ import ai.starwhale.mlops.api.protocol.ResponseMessage; import ai.starwhale.mlops.api.protocol.ft.FineTuneSpaceCreateRequest; import ai.starwhale.mlops.api.protocol.ft.FineTuneSpaceVo; +import ai.starwhale.mlops.api.protocol.model.ModelViewVo; import ai.starwhale.mlops.common.IdConverter; import ai.starwhale.mlops.configuration.FeaturesProperties; import ai.starwhale.mlops.domain.ft.FineTuneAppService; @@ -30,8 +31,14 @@ import ai.starwhale.mlops.domain.user.UserService; import com.github.pagehelper.PageInfo; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; import java.util.List; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -195,4 +202,38 @@ public ResponseEntity> exportEval( return ResponseEntity.ok(Code.success.asResponse("")); } + @GetMapping( + value = "/project/{projectId}/ftspace/{spaceId}/model-tree", + produces = MediaType.APPLICATION_JSON_VALUE + ) + @PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER', 'GUEST')") + ResponseEntity>> listModelTree( + @PathVariable("projectId") Long projectId, + @PathVariable("spaceId") Long spaceId + ) { + return ResponseEntity.ok(Code.success.asResponse( + fineTuneAppService.listModelVersionView(projectId, spaceId) + )); + } + + @GetMapping( + value = "/project/{projectId}/ftspace/{spaceId}/recent-model-tree", + produces = MediaType.APPLICATION_JSON_VALUE + ) + @PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER', 'GUEST')") + ResponseEntity>> recentModelTree( + @PathVariable("projectId") Long projectId, + @PathVariable("spaceId") Long spaceId, + @Parameter(in = ParameterIn.QUERY, description = "Data limit", schema = @Schema()) + @RequestParam(required = false, defaultValue = "5") + @Valid + @Min(value = 1, message = "limit must be greater than or equal to 1") + @Max(value = 50, message = "limit must be less than or equal to 50") + Integer limit + ) { + return ResponseEntity.ok(Code.success.asResponse( + fineTuneAppService.listRecentlyModelVersionView(projectId, spaceId, limit) + )); + } + } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/model/ModelVersionViewVo.java b/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/model/ModelVersionViewVo.java index c3edf8d972..0f9b695ffb 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/model/ModelVersionViewVo.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/model/ModelVersionViewVo.java @@ -47,6 +47,8 @@ public class ModelVersionViewVo { @NotNull private Integer shared; + private Boolean draft; + @NotNull private List stepSpecs; diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java index bcb9ccd4d8..4ededfb76f 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java @@ -19,6 +19,7 @@ import static ai.starwhale.mlops.domain.evaluation.EvaluationService.TABLE_NAME_FORMAT; import ai.starwhale.mlops.api.protocol.job.JobRequest; +import ai.starwhale.mlops.api.protocol.model.ModelViewVo; import ai.starwhale.mlops.api.protocol.model.ModelVo; import ai.starwhale.mlops.common.Constants; import ai.starwhale.mlops.common.IdConverter; @@ -372,6 +373,14 @@ public void releaseFt( } } + public List listModelVersionView(Long projectId, Long spaceId) { + return modelService.listFtSpaceModelVersionView(String.valueOf(projectId), spaceId); + } + + public List listRecentlyModelVersionView(Long projectId, Long spaceId, Integer limit) { + return modelService.listRecentlyModelVersionView(String.valueOf(projectId), spaceId, limit); + } + private void checkFeatureEnabled() throws StarwhaleApiException { if (!this.featuresProperties.isFineTuneEnabled()) { throw new StarwhaleApiException( diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java index 30152c77ad..a03d0b3035 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java @@ -381,7 +381,8 @@ public void shareModelVersion(String projectUrl, String modelUrl, String version } public List listModelVersionView( - String projectUrl, boolean includeShared, boolean includeCurrentProject) { + String projectUrl, boolean includeShared, boolean includeCurrentProject + ) { var project = projectService.findProject(projectUrl); var list = new ArrayList(); if (includeCurrentProject) { @@ -402,6 +403,20 @@ public List listRecentlyModelVersionView(String projectUrl, Integer return viewEntityToVo(list, project); } + public List listRecentlyModelVersionView(String projectUrl, Long spaceId, Integer limit) { + var project = projectService.findProject(projectUrl); + var userId = userService.currentUserDetail().getId(); + var list = modelVersionMapper.listModelVersionsByUserRecentlyUsedInFtSpace( + project.getId(), userId, spaceId, limit); + return viewEntityToVo(list, project); + } + + public List listFtSpaceModelVersionView(String projectUrl, Long spaceId) { + var project = projectService.findProject(projectUrl); + var list = modelVersionMapper.listModelVersionViewByFtSpace(project.getId(), spaceId); + return viewEntityToVo(list, project); + } + private List viewEntityToVo(List list, Project currentProject) { Map map = new LinkedHashMap<>(); var tags = new HashMap>>(); @@ -446,6 +461,7 @@ private List viewEntityToVo(List list, Proj .latest(entity.getId() != null && entity.getId().equals(latest)) .createdTime(entity.getCreatedTime().getTime()) .shared(toInt(entity.getShared())) + .draft(entity.getDraft()) .builtInRuntime(entity.getBuiltInRuntime()) .stepSpecs(jobSpecParser.parseAndFlattenStepFromYaml(entity.getJobs())) .build()); diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/mapper/ModelVersionMapper.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/mapper/ModelVersionMapper.java index 8739134012..a760e152bd 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/mapper/ModelVersionMapper.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/mapper/ModelVersionMapper.java @@ -65,37 +65,28 @@ List list( @Update("update model_version set model_id = #{modelId}, draft = 0 where id = #{id}") int updateModelRef(@Param("id") Long id, @Param("modelId") Long modelId); - @Select("select " + VERSION_VIEW_COLUMNS - + " from model_info as m, model_version as v, project_info as p, user_info as u" - + " where v.model_id = m.id" - + " and m.project_id = p.id" - + " and m.owner_id = u.id" - + " and m.deleted_time = 0" - + " and p.is_deleted = 0" - + " and p.id = #{projectId}" - + " order by m.id desc, v.version_order desc") + @SelectProvider(type = ModelVersionProvider.class, method = "listInProject") List listModelVersionViewByProject(@Param("projectId") Long projectId); - @Select({"" - }) + @SelectProvider(type = ModelVersionProvider.class, method = "listUserRecentlyUsed") List listModelVersionsByUserRecentlyUsed( - @Param("projectId") Long projectId, @Param("userId") Long userId, @Param("limit") Integer limit + @Param("projectId") Long projectId, + @Param("userId") Long userId, + @Param("limit") Integer limit + ); + + @SelectProvider(type = ModelVersionProvider.class, method = "listInFtSpace") + List listModelVersionViewByFtSpace( + @Param("projectId") Long projectId, + @Param("spaceId") Long spaceId + ); + + @SelectProvider(type = ModelVersionProvider.class, method = "listUserRecentlyUsedWithFtSpace") + List listModelVersionsByUserRecentlyUsedInFtSpace( + @Param("projectId") Long projectId, + @Param("userId") Long userId, + @Param("spaceId") Long spaceId, + @Param("limit") Integer limit ); @Select("select " + VERSION_VIEW_COLUMNS @@ -155,6 +146,107 @@ ModelVersionEntity findByVersionOrder( class ModelVersionProvider { + public String listInProject(@Param("projectId") Long projectId) { + return new SQL() { + { + SELECT(VERSION_VIEW_COLUMNS); + FROM("model_version as v"); + INNER_JOIN("model_info as m on m.id = v.model_id"); + INNER_JOIN("project_info as p on p.id = m.project_id"); + INNER_JOIN("user_info as u on u.id = m.owner_id"); + // (non-draft) models in current project + WHERE("v.draft=0"); + WHERE("m.deleted_time = 0"); + WHERE("p.id = #{projectId}"); + ORDER_BY("m.id desc"); + ORDER_BY("v.version_order desc"); + } + + }.toString(); + } + + public String listInFtSpace( + @Param("projectId") Long projectId, + @Param("spaceId") Long spaceId) { + return new SQL() { + { + SELECT(VERSION_VIEW_COLUMNS); + FROM("model_version as v"); + INNER_JOIN("model_info as m on m.id = v.model_id"); + INNER_JOIN("project_info as p on p.id = m.project_id"); + INNER_JOIN("user_info as u on u.id = m.owner_id"); + INNER_JOIN("fine_tune as ft on ft.target_model_version_id = v.id"); + // all models in current space + WHERE("ft.space_id = #{spaceId}"); + WHERE("m.deleted_time = 0"); + WHERE("p.id = #{projectId}"); + ORDER_BY("m.id desc"); + ORDER_BY("v.version_order desc"); + } + + }.toString(); + } + + public String listUserRecentlyUsed( + @Param("projectId") Long projectId, + @Param("userId") Long userId, + @Param("limit") Integer limit) { + return new SQL() { + { + SELECT(VERSION_VIEW_COLUMNS + ", MAX(j.id) as job_id"); + FROM("model_version as v"); + INNER_JOIN("model_info as m on m.id = v.model_id"); + INNER_JOIN("job_info as j on j.model_version_id = v.id"); + INNER_JOIN("project_info as p on p.id = m.project_id"); + INNER_JOIN("user_info as u on u.id = m.owner_id"); + // (non-draft) models in current project or other project but is shared + WHERE("(" + + "(m.project_id = #{projectId} and v.draft=0) " + + "or (m.project_id != #{projectId} and v.shared = 1 and v.draft = 0)" + + ")"); + + WHERE("m.deleted_time = 0"); + WHERE("j.owner_id = #{userId}"); // jobs in current user + WHERE("j.project_id = #{projectId}"); // jobs in current project + GROUP_BY("v.id"); + ORDER_BY("job_id desc"); // recently + LIMIT("#{limit}"); + } + }.toString(); + } + + public String listUserRecentlyUsedWithFtSpace( + @Param("projectId") Long projectId, + @Param("userId") Long userId, + @Param("spaceId") Long spaceId, + @Param("limit") Integer limit) { + return new SQL() { + { + SELECT(VERSION_VIEW_COLUMNS + ", MAX(j.id) as job_id"); + FROM("model_version as v"); + INNER_JOIN("model_info as m on m.id = v.model_id"); + INNER_JOIN("job_info as j on j.model_version_id = v.id"); + INNER_JOIN("project_info as p on p.id = m.project_id"); + INNER_JOIN("user_info as u on u.id = m.owner_id"); + // non-draft models in current project + // or all in current space + // or (non-draft) other project but is shared + WHERE("(" + + "(m.project_id = #{projectId} and v.draft=0)" + + "or v.id in " + + " (select target_model_version_id from fine_tune where space_id = #{spaceId})" + + "or (m.project_id != #{projectId} and v.shared = 1 and v.draft = 0)" + + ")"); + WHERE("m.deleted_time = 0"); + WHERE("j.owner_id = #{userId}"); // jobs in current user + WHERE("j.project_id = #{projectId}"); // jobs in current project + GROUP_BY("v.id"); + ORDER_BY("job_id desc"); // recently + LIMIT("#{limit}"); + } + }.toString(); + } + public String listSql( @Param("modelId") Long modelId, @Param("namePrefix") String namePrefix, @Param("draft") Boolean draft diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/po/ModelVersionViewEntity.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/po/ModelVersionViewEntity.java index 43150c4b12..900e9359c3 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/po/ModelVersionViewEntity.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/po/ModelVersionViewEntity.java @@ -49,6 +49,8 @@ public class ModelVersionViewEntity extends BaseEntity implements HasId { private Boolean shared; + private Boolean draft; + private String storagePath; private String builtInRuntime; diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/model/ModelServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/model/ModelServiceTest.java index 040fa975d2..d174744888 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/model/ModelServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/model/ModelServiceTest.java @@ -829,6 +829,24 @@ public void testListModelVersionView() { allOf(hasProperty("versionName", is("v4")), hasProperty("alias", is("v4")), hasProperty("latest", is(true))))); + + res = modelService.listModelVersionView("1", false, true); + assertEquals(2, res.size()); + + res = modelService.listModelVersionView("1", false, true); + assertEquals(2, res.size()); + + res = modelService.listFtSpaceModelVersionView("1", 1L); + assertEquals(0, res.size()); + + res = modelService.listFtSpaceModelVersionView("1", 1L); + assertEquals(0, res.size()); + + res = modelService.listRecentlyModelVersionView("1", 5); + assertEquals(0, res.size()); + + res = modelService.listRecentlyModelVersionView("1", 1L, 5); + assertEquals(0, res.size()); } @Test