Skip to content

Commit

Permalink
split api
Browse files Browse the repository at this point in the history
  • Loading branch information
@goldenxinxing
goldenxinxing committed Nov 22, 2023
1 parent 046d6a0 commit da7adb7
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 79 deletions.
41 changes: 41 additions & 0 deletions server/controller/src/main/java/ai/starwhale/mlops/api/FineTuneController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import ai.starwhale.mlops.api.protocol.ResponseMessage;
import ai.starwhale.mlops.api.protocol.ft.FineTuneSpaceCreateRequest;
import ai.starwhale.mlops.api.protocol.ft.FineTuneSpaceVo;
import ai.starwhale.mlops.api.protocol.model.ModelViewVo;
import ai.starwhale.mlops.common.IdConverter;
import ai.starwhale.mlops.configuration.FeaturesProperties;
import ai.starwhale.mlops.domain.ft.FineTuneAppService;
Expand All @@ -30,8 +31,14 @@
import ai.starwhale.mlops.domain.user.UserService;
import com.github.pagehelper.PageInfo;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.util.List;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -195,4 +202,38 @@ public ResponseEntity<ResponseMessage<String>> exportEval(
return ResponseEntity.ok(Code.success.asResponse(""));
}

@GetMapping(
value = "/project/{projectId}/ftspace/{spaceId}/model-tree",
produces = MediaType.APPLICATION_JSON_VALUE
)
@PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER', 'GUEST')")
ResponseEntity<ResponseMessage<List<ModelViewVo>>> listModelTree(
@PathVariable("projectId") Long projectId,
@PathVariable("spaceId") Long spaceId
) {
return ResponseEntity.ok(Code.success.asResponse(
fineTuneAppService.listModelVersionView(projectId, spaceId)
));
}

@GetMapping(
value = "/project/{projectId}/ftspace/{spaceId}/recent-model-tree",
produces = MediaType.APPLICATION_JSON_VALUE
)
@PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER', 'GUEST')")
ResponseEntity<ResponseMessage<List<ModelViewVo>>> recentModelTree(
@PathVariable("projectId") Long projectId,
@PathVariable("spaceId") Long spaceId,
@Parameter(in = ParameterIn.QUERY, description = "Data limit", schema = @Schema())
@RequestParam(required = false, defaultValue = "5")
@Valid
@Min(value = 1, message = "limit must be greater than or equal to 1")
@Max(value = 50, message = "limit must be less than or equal to 50")
Integer limit
) {
return ResponseEntity.ok(Code.success.asResponse(
fineTuneAppService.listRecentlyModelVersionView(projectId, spaceId, limit)
));
}

}
17 changes: 6 additions & 11 deletions server/controller/src/main/java/ai/starwhale/mlops/api/ModelController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import ai.starwhale.mlops.api.protocol.storage.FileNode;
import ai.starwhale.mlops.common.IdConverter;
import ai.starwhale.mlops.common.PageParams;
import ai.starwhale.mlops.domain.job.BizType;
import ai.starwhale.mlops.domain.model.ModelService;
import ai.starwhale.mlops.domain.model.bo.ModelQuery;
import ai.starwhale.mlops.domain.model.bo.ModelVersion;
Expand Down Expand Up @@ -247,20 +246,18 @@ ResponseEntity<ResponseMessage<List<ModelViewVo>>> listModelTree(
@Parameter(in = ParameterIn.PATH, required = true, description = "Project url", schema = @Schema())
@PathVariable String projectUrl,
@Parameter(in = ParameterIn.QUERY, description = "Data range", schema = @Schema())
@RequestParam(required = false, defaultValue = "all") DataScope scope,
@RequestParam(required = false) BizType bizType,
@RequestParam(required = false) Long bizId
@RequestParam(required = false, defaultValue = "all") DataScope scope
) {
List<ModelViewVo> list;
switch (scope) {
case all:
list = modelService.listModelVersionView(projectUrl, true, true, bizType, bizId);
list = modelService.listModelVersionView(projectUrl, true, true);
break;
case shared:
list = modelService.listModelVersionView(projectUrl, true, false, bizType, bizId);
list = modelService.listModelVersionView(projectUrl, true, false);
break;
case project:
list = modelService.listModelVersionView(projectUrl, false, true, bizType, bizId);
list = modelService.listModelVersionView(projectUrl, false, true);
break;
default:
list = List.of();
Expand All @@ -277,12 +274,10 @@ ResponseEntity<ResponseMessage<List<ModelViewVo>>> recentModelTree(
@Valid
@Min(value = 1, message = "limit must be greater than or equal to 1")
@Max(value = 50, message = "limit must be less than or equal to 50")
Integer limit,
@RequestParam(required = false) BizType bizType,
@RequestParam(required = false) Long bizId
Integer limit
) {
return ResponseEntity.ok(Code.success.asResponse(
modelService.listRecentlyModelVersionView(projectUrl, limit, bizType, bizId)
modelService.listRecentlyModelVersionView(projectUrl, limit)
));
}

Expand Down
9 changes: 9 additions & 0 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static ai.starwhale.mlops.domain.evaluation.EvaluationService.TABLE_NAME_FORMAT;

import ai.starwhale.mlops.api.protocol.job.JobRequest;
import ai.starwhale.mlops.api.protocol.model.ModelViewVo;
import ai.starwhale.mlops.api.protocol.model.ModelVo;
import ai.starwhale.mlops.common.Constants;
import ai.starwhale.mlops.common.IdConverter;
Expand Down Expand Up @@ -372,6 +373,14 @@ public void releaseFt(
}
}

public List<ModelViewVo> listModelVersionView(Long projectId, Long spaceId) {
return modelService.listFtSpaceModelVersionView(String.valueOf(projectId), spaceId);
}

public List<ModelViewVo> listRecentlyModelVersionView(Long projectId, Long spaceId, Integer limit) {
return modelService.listRecentlyModelVersionView(String.valueOf(projectId), spaceId, limit);
}

private void checkFeatureEnabled() throws StarwhaleApiException {
if (!this.featuresProperties.isFineTuneEnabled()) {
throw new StarwhaleApiException(
Expand Down
25 changes: 18 additions & 7 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import ai.starwhale.mlops.domain.bundle.tag.BundleVersionTagDao;
import ai.starwhale.mlops.domain.bundle.tag.po.BundleVersionTagEntity;
import ai.starwhale.mlops.domain.ft.FineTuneDomainService;
import ai.starwhale.mlops.domain.job.BizType;
import ai.starwhale.mlops.domain.job.cache.HotJobHolder;
import ai.starwhale.mlops.domain.job.spec.JobSpecParser;
import ai.starwhale.mlops.domain.job.status.JobStatus;
Expand Down Expand Up @@ -382,12 +381,12 @@ public void shareModelVersion(String projectUrl, String modelUrl, String version
}

public List<ModelViewVo> listModelVersionView(
String projectUrl, boolean includeShared, boolean includeCurrentProject, BizType bizType, Long bizId
String projectUrl, boolean includeShared, boolean includeCurrentProject
) {
var project = projectService.findProject(projectUrl);
var list = new ArrayList<ModelViewVo>();
if (includeCurrentProject) {
var versions = modelVersionMapper.listModelVersionViewByProject(project.getId(), bizType, bizId);
var versions = modelVersionMapper.listModelVersionViewByProject(project.getId());
list.addAll(viewEntityToVo(versions, project));
}
if (includeShared) {
Expand All @@ -397,12 +396,24 @@ public List<ModelViewVo> listModelVersionView(
return list;
}

public List<ModelViewVo> listRecentlyModelVersionView(
String projectUrl, Integer limit, BizType bizType, Long bizId) {
public List<ModelViewVo> listRecentlyModelVersionView(String projectUrl, Integer limit) {
var project = projectService.findProject(projectUrl);
var userId = userService.currentUserDetail().getId();
var list = modelVersionMapper.listModelVersionsByUserRecentlyUsed(
project.getId(), userId, limit, bizType, bizId);
var list = modelVersionMapper.listModelVersionsByUserRecentlyUsed(project.getId(), userId, limit);
return viewEntityToVo(list, project);
}

public List<ModelViewVo> listRecentlyModelVersionView(String projectUrl, Long spaceId, Integer limit) {
var project = projectService.findProject(projectUrl);
var userId = userService.currentUserDetail().getId();
var list = modelVersionMapper.listModelVersionsByUserRecentlyUsedInFtSpace(
project.getId(), userId, spaceId, limit);
return viewEntityToVo(list, project);
}

public List<ModelViewVo> listFtSpaceModelVersionView(String projectUrl, Long spaceId) {
var project = projectService.findProject(projectUrl);
var list = modelVersionMapper.listModelVersionViewByFtSpace(project.getId(), spaceId);
return viewEntityToVo(list, project);
}

Expand Down
130 changes: 81 additions & 49 deletions ...r/controller/src/main/java/ai/starwhale/mlops/domain/model/mapper/ModelVersionMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,11 @@

package ai.starwhale.mlops.domain.model.mapper;

import ai.starwhale.mlops.domain.job.BizType;
import ai.starwhale.mlops.domain.model.po.ModelVersionEntity;
import ai.starwhale.mlops.domain.model.po.ModelVersionViewEntity;
import cn.hutool.core.util.StrUtil;
import java.util.List;
import java.util.Objects;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.SELECT;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Options;
Expand Down Expand Up @@ -68,19 +66,27 @@ List<ModelVersionEntity> list(
int updateModelRef(@Param("id") Long id, @Param("modelId") Long modelId);

@SelectProvider(type = ModelVersionProvider.class, method = "listInProject")
List<ModelVersionViewEntity> listModelVersionViewByProject(
@Param("projectId") Long projectId,
@Param("bizType") BizType bizType,
@Param("bizId") Long bizId
);
List<ModelVersionViewEntity> listModelVersionViewByProject(@Param("projectId") Long projectId);

@SelectProvider(type = ModelVersionProvider.class, method = "listUserRecentlyUsed")
List<ModelVersionViewEntity> listModelVersionsByUserRecentlyUsed(
@Param("projectId") Long projectId,
@Param("userId") Long userId,
@Param("limit") Integer limit,
@Param("bizType") BizType bizType,
@Param("bizId") Long bizId
@Param("limit") Integer limit
);

@SelectProvider(type = ModelVersionProvider.class, method = "listInFtSpace")
List<ModelVersionViewEntity> listModelVersionViewByFtSpace(
@Param("projectId") Long projectId,
@Param("spaceId") Long spaceId
);

@SelectProvider(type = ModelVersionProvider.class, method = "listUserRecentlyUsedWithFtSpace")
List<ModelVersionViewEntity> listModelVersionsByUserRecentlyUsedInFtSpace(
@Param("projectId") Long projectId,
@Param("userId") Long userId,
@Param("spaceId") Long spaceId,
@Param("limit") Integer limit
);

@Select("select " + VERSION_VIEW_COLUMNS
Expand Down Expand Up @@ -140,28 +146,38 @@ ModelVersionEntity findByVersionOrder(

class ModelVersionProvider {

public String listInProject(
public String listInProject(@Param("projectId") Long projectId) {
return new SQL() {
{
SELECT(VERSION_VIEW_COLUMNS);
FROM("model_version as v");
INNER_JOIN("model_info as m on m.id = v.model_id");
INNER_JOIN("project_info as p on p.id = m.project_id");
INNER_JOIN("user_info as u on u.id = m.owner_id");
// (non-draft) models in current project
WHERE("v.draft=0");
WHERE("m.deleted_time = 0");
WHERE("p.id = #{projectId}");
ORDER_BY("m.id desc");
ORDER_BY("v.version_order desc");
}

}.toString();
}

public String listInFtSpace(
@Param("projectId") Long projectId,
@Param("bizType") BizType bizType,
@Param("bizId") Long bizId
) {
@Param("spaceId") Long spaceId) {
return new SQL() {
{
SELECT(VERSION_VIEW_COLUMNS);
FROM("model_version as v");
INNER_JOIN("model_info as m on m.id = v.model_id");
INNER_JOIN("project_info as p on p.id = m.project_id");
INNER_JOIN("user_info as u on u.id = m.owner_id");
if (bizType == BizType.FINE_TUNE && bizId != null) {
// non-draft models in current project or draft models in current space
WHERE("(v.draft=0 or v.id in "
+ "(select target_model_version_id from fine_tune where space_id = #{bizId})"
+ ")"
);
} else {
// (non-draft) models in current project
WHERE("v.draft=0");
}
INNER_JOIN("fine_tune as ft on ft.target_model_version_id = v.id");
// all models in current space
WHERE("ft.space_id = #{spaceId}");
WHERE("m.deleted_time = 0");
WHERE("p.id = #{projectId}");
ORDER_BY("m.id desc");
Expand All @@ -174,10 +190,7 @@ public String listInProject(
public String listUserRecentlyUsed(
@Param("projectId") Long projectId,
@Param("userId") Long userId,
@Param("limit") Integer limit,
@Param("bizType") BizType bizType,
@Param("bizId") Long bizId
) {
@Param("limit") Integer limit) {
return new SQL() {
{
SELECT(VERSION_VIEW_COLUMNS + ", MAX(j.id) as job_id");
Expand All @@ -186,31 +199,50 @@ public String listUserRecentlyUsed(
INNER_JOIN("job_info as j on j.model_version_id = v.id");
INNER_JOIN("project_info as p on p.id = m.project_id");
INNER_JOIN("user_info as u on u.id = m.owner_id");
if (bizType == BizType.FINE_TUNE && bizId != null) {
// models in current project(non-draft or draft in current space)
// or (non-draft) other project but is shared
WHERE("("
+ "(m.project_id = #{projectId} and "
+ " (v.draft=0 or v.id in "
+ " (select target_model_version_id from fine_tune where space_id = #{bizId})"
+ " )"
+ ") "
+ "or "
+ "(m.project_id != #{projectId} and v.shared = 1 and v.draft = 0)"
+ ")");
} else {
// (non-draft) models in current project or other project but is shared
WHERE("("
+ "(m.project_id = #{projectId} and v.draft=0) or "
+ "(m.project_id != #{projectId} and v.shared = 1 and v.draft = 0)"
+ ")");
}
// (non-draft) models in current project or other project but is shared
WHERE("("
+ "(m.project_id = #{projectId} and v.draft=0) "
+ "or (m.project_id != #{projectId} and v.shared = 1 and v.draft = 0)"
+ ")");

WHERE("m.deleted_time = 0");
WHERE("j.owner_id = #{userId}"); // jobs in current user
WHERE("j.project_id = #{projectId}"); // jobs in current project
GROUP_BY("v.id");
ORDER_BY("job_id desc"); // recently
LIMIT("#{limit}");
}
}.toString();
}

public String listUserRecentlyUsedWithFtSpace(
@Param("projectId") Long projectId,
@Param("userId") Long userId,
@Param("spaceId") Long spaceId,
@Param("limit") Integer limit) {
return new SQL() {
{
SELECT(VERSION_VIEW_COLUMNS + ", MAX(j.id) as job_id");
FROM("model_version as v");
INNER_JOIN("model_info as m on m.id = v.model_id");
INNER_JOIN("job_info as j on j.model_version_id = v.id");
INNER_JOIN("project_info as p on p.id = m.project_id");
INNER_JOIN("user_info as u on u.id = m.owner_id");
// non-draft models in current project
// or all in current space
// or (non-draft) other project but is shared
WHERE("("
+ "(m.project_id = #{projectId} and v.draft=0)"
+ "or v.id in "
+ " (select target_model_version_id from fine_tune where space_id = #{spaceId})"
+ "or (m.project_id != #{projectId} and v.shared = 1 and v.draft = 0)"
+ ")");
WHERE("m.deleted_time = 0");
WHERE("j.owner_id = #{userId}"); // jobs in current user
WHERE("j.project_id = #{projectId}"); // jobs in current project
GROUP_BY("v.id");
ORDER_BY("job_id desc");
LIMIT("#{limit}"); // recently
ORDER_BY("job_id desc"); // recently
LIMIT("#{limit}");
}
}.toString();
}
Expand Down
Loading

0 comments on commit da7adb7

Please sign in to comment.