From 288a0a13cb5ad4cad500df6eebef632e82def8fe Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Thu, 20 Jul 2023 19:28:43 +0800 Subject: [PATCH 1/3] fix --- .../java/ai/starwhale/mlops/domain/dataset/DatasetService.java | 2 +- .../ai/starwhale/mlops/domain/evaluation/EvaluationService.java | 2 +- .../src/main/java/ai/starwhale/mlops/domain/job/JobService.java | 2 +- .../main/java/ai/starwhale/mlops/domain/model/ModelService.java | 2 +- .../java/ai/starwhale/mlops/domain/runtime/RuntimeService.java | 2 +- .../main/java/ai/starwhale/mlops/domain/task/TaskService.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java index 265ad7e483..9c86f98fda 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java @@ -178,9 +178,9 @@ public DatasetService(ProjectService projectService, DatasetMapper datasetMapper public PageInfo listDataset(DatasetQuery query, PageParams pageParams) { - PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); Long projectId = projectService.getProjectId(query.getProjectUrl()); Long userId = userService.getUserId(query.getOwner()); + PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); List entities = datasetMapper.list(projectId, query.getNamePrefix(), userId, null); diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java index 98f9d9b5d9..f6e8740d4d 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java @@ -112,8 +112,8 @@ public Boolean createViewConfig(String projectUrl, ConfigRequest configRequest) public PageInfo listEvaluationSummary(String projectUrl, SummaryFilter summaryFilter, PageParams pageParams) { - PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); Long projectId = projectService.getProjectId(projectUrl); + PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); List jobs = jobDao.listJobs(projectId, null); return PageUtil.toPageInfo(jobs, this::toSummary); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java index 746e4f3b9a..e1e716eb6c 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java @@ -140,8 +140,8 @@ public JobService( } public PageInfo listJobs(String projectUrl, Long modelId, PageParams pageParams) { - PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); Long projectId = projectService.getProjectId(projectUrl); + PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); List jobEntities = jobDao.listJobs(projectId, modelId); return PageUtil.toPageInfo(jobEntities, jobConvertor::convert); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java index 6f18a02676..69f18da5fe 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/model/ModelService.java @@ -174,9 +174,9 @@ public ModelService( } public PageInfo listModel(ModelQuery query, PageParams pageParams) { - PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); Long projectId = projectService.getProjectId(query.getProjectUrl()); Long userId = userService.getUserId(query.getOwner()); + PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); List entities = modelMapper.list(projectId, query.getNamePrefix(), userId, null); return PageUtil.toPageInfo(entities, entity -> { ModelVo vo = modelVoConverter.convert(entity); diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/runtime/RuntimeService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/runtime/RuntimeService.java index c50e1af26b..4cde3d9354 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/runtime/RuntimeService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/runtime/RuntimeService.java @@ -197,9 +197,9 @@ public RuntimeService(RuntimeMapper runtimeMapper, } public PageInfo listRuntime(RuntimeQuery runtimeQuery, PageParams pageParams) { - PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); Long projectId = projectService.getProjectId(runtimeQuery.getProjectUrl()); Long userId = userService.getUserId(runtimeQuery.getOwner()); + PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); List entities = runtimeMapper.list(projectId, runtimeQuery.getNamePrefix(), userId, null); return PageUtil.toPageInfo(entities, rt -> { diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java index c4eb720e5c..4e56414bc5 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java @@ -63,8 +63,8 @@ public TaskService(TaskConverter taskConvertor, TaskMapper taskMapper, } public PageInfo listTasks(String jobUrl, PageParams pageParams) { - PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); Job job = jobDao.findJob(jobUrl); + PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); List tasks = taskMapper.listTasks(job.getId()).stream().map(taskConvertor::convert) .peek(taskVo -> { if (!StringUtils.hasText(taskVo.getResourcePool())) { From d3e54b164c6de6066e2d17763d98edc09d6d1d3c Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Fri, 21 Jul 2023 17:06:12 +0800 Subject: [PATCH 2/3] add ut --- .../domain/evaluation/EvaluationService.java | 10 +- .../ai/starwhale/mlops/domain/job/JobDao.java | 15 +- .../mlops/domain/job/JobService.java | 2 +- .../domain/job/step/mapper/StepMapper.java | 2 +- .../mlops/domain/task/TaskService.java | 20 +- .../ai/starwhale/mlops/domain/PageTest.java | 428 ++++++++++++++++++ .../evaluation/EvaluationServiceTest.java | 15 +- .../mlops/domain/job/JobServiceTest.java | 3 +- 8 files changed, 465 insertions(+), 30 deletions(-) create mode 100644 server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java index f6e8740d4d..04886a69ea 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java @@ -29,8 +29,8 @@ import ai.starwhale.mlops.domain.evaluation.mapper.ViewConfigMapper; import ai.starwhale.mlops.domain.evaluation.po.ViewConfigEntity; import ai.starwhale.mlops.domain.job.JobDao; -import ai.starwhale.mlops.domain.job.bo.Job; import ai.starwhale.mlops.domain.job.converter.JobConverter; +import ai.starwhale.mlops.domain.job.po.JobEntity; import ai.starwhale.mlops.domain.job.status.JobStatusMachine; import ai.starwhale.mlops.domain.project.ProjectService; import ai.starwhale.mlops.domain.user.UserService; @@ -114,12 +114,12 @@ public PageInfo listEvaluationSummary(String projectUrl, SummaryFilter summaryFilter, PageParams pageParams) { Long projectId = projectService.getProjectId(projectUrl); PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); - List jobs = jobDao.listJobs(projectId, null); + var jobs = jobDao.listJobs(projectId, null); return PageUtil.toPageInfo(jobs, this::toSummary); } - private SummaryVo toSummary(Job job) { + private SummaryVo toSummary(JobEntity job) { if (summaryCache.containsKey(job.getId())) { return summaryCache.get(job.getId()); } @@ -129,7 +129,7 @@ private SummaryVo toSummary(Job job) { .id(jobVo.getId()) .uuid(jobVo.getUuid()) .projectId(idConvertor.convert(job.getProject().getId())) - .projectName(job.getProject().getName()) + .projectName(job.getProject().getProjectName()) .modelName(jobVo.getModelName()) .modelVersion(jobVo.getModelVersion()) .datasets(StrUtil.join(",", jobVo.getDatasets())) @@ -145,7 +145,7 @@ private SummaryVo toSummary(Job job) { .build(); // only cache the jobs which have the final status - if (jobStatusMachine.isFinal(job.getStatus())) { + if (jobStatusMachine.isFinal(job.getJobStatus())) { summaryCache.put(job.getId(), summaryVo); } return summaryVo; diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobDao.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobDao.java index 3db81a6443..fb40bb202a 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobDao.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobDao.java @@ -116,11 +116,8 @@ private JobEntity convertFromFlatten(JobFlattenEntity flattenEntity) { } - public List listJobs(Long projectId, Long modelId) { - return jobMapper.listJobs(projectId, modelId) - .stream() - .map(jobBoConverter::fromEntity) - .collect(Collectors.toList()); + public List listJobs(Long projectId, Long modelId) { + return jobMapper.listJobs(projectId, modelId); } public List findJobByStatusIn(List jobStatuses) { @@ -177,6 +174,14 @@ public Job findJob(String jobUrl) { } } + public JobEntity findJobEntity(String jobUrl) { + if (idConvertor.isId(jobUrl)) { + return jobMapper.findJobById(idConvertor.revert(jobUrl)); + } else { + return jobMapper.findJobByUuid(jobUrl); + } + } + public boolean updateJobPinStatus(String jobUrl, boolean pinned) { Date pinnedTime = pinned ? Date.from(Instant.now()) : null; diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java index e1e716eb6c..bbdb6a208c 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobService.java @@ -142,7 +142,7 @@ public JobService( public PageInfo listJobs(String projectUrl, Long modelId, PageParams pageParams) { Long projectId = projectService.getProjectId(projectUrl); PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); - List jobEntities = jobDao.listJobs(projectId, modelId); + var jobEntities = jobDao.listJobs(projectId, modelId); return PageUtil.toPageInfo(jobEntities, jobConvertor::convert); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/step/mapper/StepMapper.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/step/mapper/StepMapper.java index dcb8ab68b5..2fad85c91a 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/step/mapper/StepMapper.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/step/mapper/StepMapper.java @@ -35,7 +35,7 @@ public interface StepMapper { + " values (#{step.uuid}, #{step.name}, #{step.jobId}, #{step.lastStepId}, #{step.status}, " + " #{step.concurrency}, #{step.taskNum}, #{step.poolInfo})") @Options(useGeneratedKeys = true, keyColumn = "id", keyProperty = "id") - void save(@Param("step") StepEntity stepEntity); + int save(@Param("step") StepEntity stepEntity); @Select("select id, step_uuid as uuid, step_name as name, job_id, last_step_id, step_status as status," + " finished_time, started_time, created_time, modified_time, concurrency, task_num, pool_info from step" diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java index 4e56414bc5..534ee89d4d 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/task/TaskService.java @@ -19,8 +19,8 @@ import ai.starwhale.mlops.api.protocol.task.TaskVo; import ai.starwhale.mlops.common.IdConverter; import ai.starwhale.mlops.common.PageParams; +import ai.starwhale.mlops.common.util.PageUtil; import ai.starwhale.mlops.domain.job.JobDao; -import ai.starwhale.mlops.domain.job.bo.Job; import ai.starwhale.mlops.domain.task.bo.ResultPath; import ai.starwhale.mlops.domain.task.converter.TaskConverter; import ai.starwhale.mlops.domain.task.mapper.TaskMapper; @@ -63,16 +63,16 @@ public TaskService(TaskConverter taskConvertor, TaskMapper taskMapper, } public PageInfo listTasks(String jobUrl, PageParams pageParams) { - Job job = jobDao.findJob(jobUrl); + var job = jobDao.findJobEntity(jobUrl); PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize()); - List tasks = taskMapper.listTasks(job.getId()).stream().map(taskConvertor::convert) - .peek(taskVo -> { - if (!StringUtils.hasText(taskVo.getResourcePool())) { - taskVo.setResourcePool(job.getResourcePool().getName()); - } - }) - .collect(Collectors.toList()); - return PageInfo.of(tasks); + var entities = taskMapper.listTasks(job.getId()); + return PageUtil.toPageInfo(entities, entity -> { + var vo = taskConvertor.convert(entity); + if (!StringUtils.hasText(vo.getResourcePool())) { + vo.setResourcePool(job.getResourcePool()); + } + return vo; + }); } diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java new file mode 100644 index 0000000000..cae00a8dad --- /dev/null +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java @@ -0,0 +1,428 @@ +/* + * Copyright 2022 Starwhale, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.starwhale.mlops.domain; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import ai.starwhale.mlops.common.PageParams; +import ai.starwhale.mlops.datastore.DataStore; +import ai.starwhale.mlops.domain.dataset.DatasetService; +import ai.starwhale.mlops.domain.dataset.bo.DatasetQuery; +import ai.starwhale.mlops.domain.dataset.mapper.DatasetMapper; +import ai.starwhale.mlops.domain.dataset.po.DatasetEntity; +import ai.starwhale.mlops.domain.job.JobService; +import ai.starwhale.mlops.domain.job.JobType; +import ai.starwhale.mlops.domain.job.ModelServingService; +import ai.starwhale.mlops.domain.job.mapper.JobMapper; +import ai.starwhale.mlops.domain.job.po.JobEntity; +import ai.starwhale.mlops.domain.job.status.JobStatus; +import ai.starwhale.mlops.domain.job.step.mapper.StepMapper; +import ai.starwhale.mlops.domain.job.step.po.StepEntity; +import ai.starwhale.mlops.domain.job.step.status.StepStatus; +import ai.starwhale.mlops.domain.model.ModelService; +import ai.starwhale.mlops.domain.model.bo.ModelQuery; +import ai.starwhale.mlops.domain.model.mapper.ModelMapper; +import ai.starwhale.mlops.domain.model.mapper.ModelVersionMapper; +import ai.starwhale.mlops.domain.model.po.ModelEntity; +import ai.starwhale.mlops.domain.model.po.ModelVersionEntity; +import ai.starwhale.mlops.domain.project.bo.Project; +import ai.starwhale.mlops.domain.project.mapper.ProjectMapper; +import ai.starwhale.mlops.domain.project.po.ProjectEntity; +import ai.starwhale.mlops.domain.runtime.RuntimeService; +import ai.starwhale.mlops.domain.runtime.RuntimeTestConstants; +import ai.starwhale.mlops.domain.runtime.bo.RuntimeQuery; +import ai.starwhale.mlops.domain.runtime.mapper.RuntimeMapper; +import ai.starwhale.mlops.domain.runtime.mapper.RuntimeVersionMapper; +import ai.starwhale.mlops.domain.runtime.po.RuntimeEntity; +import ai.starwhale.mlops.domain.runtime.po.RuntimeVersionEntity; +import ai.starwhale.mlops.domain.task.TaskService; +import ai.starwhale.mlops.domain.task.mapper.TaskMapper; +import ai.starwhale.mlops.domain.task.po.TaskEntity; +import ai.starwhale.mlops.domain.task.status.TaskStatus; +import ai.starwhale.mlops.domain.user.mapper.UserMapper; +import ai.starwhale.mlops.domain.user.po.UserEntity; +import ai.starwhale.mlops.schedule.k8s.K8sJobTemplate; +import ai.starwhale.mlops.schedule.k8s.ResourceEventHolder; +import com.github.pagehelper.autoconfigure.PageHelperAutoConfiguration; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mybatis.spring.boot.test.autoconfigure.MybatisTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.ImportAutoConfiguration; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; + + +@MybatisTest +@ComponentScan( + basePackages = { + "ai.starwhale.mlops.common", + "ai.starwhale.mlops.domain", + "ai.starwhale.mlops.reporting", + "ai.starwhale.mlops.resulting", + "ai.starwhale.mlops.configuration.security"}, + excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = ModelServingService.class) + } +) +@ImportAutoConfiguration(PageHelperAutoConfiguration.class) +@Import({K8sJobTemplate.class, ResourceEventHolder.class, SimpleMeterRegistry.class}) +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +public class PageTest extends MySqlContainerHolder { + @Autowired + private JobService jobService; + @Autowired + private JobMapper jobMapper; + @Autowired + private StepMapper stepMapper; + @Autowired + private TaskService taskService; + @Autowired + private TaskMapper taskMapper; + + @Autowired + private ModelService modelService; + @Autowired + private ModelMapper modelMapper; + @Autowired + private ModelVersionMapper modelVersionMapper; + + @Autowired + private RuntimeService runtimeService; + @Autowired + private RuntimeMapper runtimeMapper; + @Autowired + private RuntimeVersionMapper runtimeVersionMapper; + + @Autowired + private DatasetService datasetService; + @Autowired + private DatasetMapper datasetMapper; + + @Autowired + private ProjectMapper projectMapper; + @Autowired + private UserMapper userMapper; + + @TestConfiguration + static class TestConfig { + @Bean + DataStore dataStore() { + return mock(DataStore.class); + } + } + + String userName = "user-test"; + String projectName = "project-test"; + + @BeforeEach + public void before() { + UserEntity userEntity = UserEntity.builder() + .userName(userName) + .userPwd("123456") + .userPwdSalt("123456") + .userEnabled(1) + .build(); + userMapper.insert(userEntity); + + ProjectEntity entity = ProjectEntity.builder() + .projectName(projectName) + .ownerId(userEntity.getId()) + .privacy(Project.Privacy.PUBLIC.getValue()) + .isDefault(1) + .build(); + projectMapper.insert(entity); + } + + @Test + public void testModelList() { + var userId = userMapper.findByName(userName).getId(); + var projectId = projectMapper.findByNameForUpdateAndOwner(projectName, userId).getId(); + for (int i = 0; i < 19; i++) { + var res = modelMapper.insert( + ModelEntity.builder() + .modelName(String.format("model-%d", i)) + .projectId(projectId) + .ownerId(userId) + .build()); + assertTrue(res > 0); + } + + // no.1 + var page = modelService.listModel( + ModelQuery.builder().projectUrl(projectName).owner(userName).build(), + PageParams.builder().pageNum(1).pageSize(10).build() + ); + assertEquals(10, page.getSize()); + assertEquals(19, page.getTotal()); + + // no.2 + page = modelService.listModel( + ModelQuery.builder().projectUrl(projectName).owner(userName).build(), + PageParams.builder().pageNum(2).pageSize(10).build() + ); + assertEquals(9, page.getSize()); + assertEquals(19, page.getTotal()); + } + + @Test + public void testRuntimeList() { + var userId = userMapper.findByName(userName).getId(); + var projectId = projectMapper.findByNameForUpdateAndOwner(projectName, userId).getId(); + for (int i = 0; i < 19; i++) { + var res = runtimeMapper.insert( + RuntimeEntity.builder() + .runtimeName(String.format("rt-%d", i)) + .projectId(projectId) + .ownerId(userId) + .build()); + assertTrue(res > 0); + } + + // no.1 + var page = runtimeService.listRuntime( + RuntimeQuery.builder().projectUrl(projectName).owner(userName).build(), + PageParams.builder().pageNum(1).pageSize(10).build() + ); + assertEquals(10, page.getSize()); + assertEquals(19, page.getTotal()); + + // no.2 + page = runtimeService.listRuntime( + RuntimeQuery.builder().projectUrl(projectName).owner(userName).build(), + PageParams.builder().pageNum(2).pageSize(10).build() + ); + assertEquals(9, page.getSize()); + assertEquals(19, page.getTotal()); + } + + @Test + public void testDatasetList() { + var userId = userMapper.findByName(userName).getId(); + var projectId = projectMapper.findByNameForUpdateAndOwner(projectName, userId).getId(); + for (int i = 0; i < 19; i++) { + var res = datasetMapper.insert( + DatasetEntity.builder() + .datasetName(String.format("ds-%d", i)) + .projectId(projectId) + .ownerId(userId) + .build()); + assertTrue(res > 0); + } + + // no.1 + var page = datasetService.listDataset( + DatasetQuery.builder().projectUrl(projectName).owner(userName).build(), + PageParams.builder().pageNum(1).pageSize(10).build() + ); + assertEquals(10, page.getSize()); + assertEquals(19, page.getTotal()); + + // no.2 + page = datasetService.listDataset( + DatasetQuery.builder().projectUrl(projectName).owner(userName).build(), + PageParams.builder().pageNum(2).pageSize(10).build() + ); + assertEquals(9, page.getSize()); + assertEquals(19, page.getTotal()); + } + + @Test + public void testJobList() { + var userId = userMapper.findByName(userName).getId(); + var projectId = projectMapper.findByNameForUpdateAndOwner(projectName, userId).getId(); + + // prepare data + var model = ModelEntity.builder() + .modelName(String.format("model-%d", 0)) + .projectId(projectId) + .ownerId(userId) + .build(); + assertTrue(modelMapper.insert(model) > 0); + var modelVersion = ModelVersionEntity.builder() + .modelId(model.getId()) + .versionName("model-version-1") + .ownerId(userId) + .jobs("mnist.evaluator:MNISTInference.cmp:\n" + + "- cls_name: ''\n" + + " concurrency: 1\n" + + " needs: []\n" + + " resources: []\n" + + " name: mnist.evaluator:MNISTInference.ppl\n" + + " replicas: 1\n" + + "- cls_name: ''\n" + + " concurrency: 1\n" + + " needs:\n" + + " - mnist.evaluator:MNISTInference.ppl\n" + + " resources:\n" + + " - type: cpu \n" + + " request: 0.1\n" + + " limit: 0.1\n" + + " - type: nvidia.com/gpu \n" + + " request: 1\n" + + " limit: 1\n" + + " - type: memory \n" + + " request: 1\n" + + " limit: 1\n" + + " name: mnist.evaluator:MNISTInference.cmp\n" + + " replicas: 1\n") + .build(); + assertTrue(modelVersionMapper.insert(modelVersion) > 0); + + var runtime = RuntimeEntity.builder() + .runtimeName(String.format("rt-%d", 0)) + .projectId(projectId) + .ownerId(userId) + .build(); + assertTrue(runtimeMapper.insert(runtime) > 0); + var runtimeVersion = RuntimeVersionEntity.builder() + .runtimeId(runtime.getId()) + .versionName("rt-version-1") + .ownerId(userId) + .versionTag("v1") + .versionMeta(RuntimeTestConstants.MANIFEST_WITH_BUILTIN_IMAGE) + .storagePath("path") + .build(); + assertTrue(runtimeVersionMapper.insert(runtimeVersion) > 0); + + for (int i = 0; i < 19; i++) { + var res = jobMapper.addJob( + JobEntity.builder() + .name(String.format("job-%d", i)) + .jobUuid(String.format("uuid-%d", i)) + .modelVersionId(modelVersion.getId()) + .runtimeVersionId(runtimeVersion.getId()) + .type(JobType.EVALUATION) + .jobStatus(JobStatus.CREATED) + .resultOutputPath("path") + .projectId(projectId) + .ownerId(userId) + .isDeleted(0) + .build()); + assertTrue(res > 0); + } + + // no.1 + var page = jobService.listJobs(projectName, null, + PageParams.builder().pageNum(1).pageSize(10).build()); + assertEquals(10, page.getSize()); + assertEquals(19, page.getTotal()); + + // no.2 + page = jobService.listJobs(projectName, null, + PageParams.builder().pageNum(2).pageSize(10).build()); + assertEquals(9, page.getSize()); + assertEquals(19, page.getTotal()); + } + + @Test + public void testTaskList() { + var userId = userMapper.findByName(userName).getId(); + var projectId = projectMapper.findByNameForUpdateAndOwner(projectName, userId).getId(); + + // prepare data + var model = ModelEntity.builder() + .modelName(String.format("model-%d", 0)) + .projectId(projectId) + .ownerId(userId) + .build(); + assertTrue(modelMapper.insert(model) > 0); + var modelVersion = ModelVersionEntity.builder() + .modelId(model.getId()) + .versionName("model-version-1") + .ownerId(userId) + .jobs("mnist.evaluator:MNISTInference.cmp:\n" + + "- cls_name: ''\n" + + " concurrency: 1\n" + + " needs: []\n" + + " resources: []\n" + + " name: mnist.evaluator:MNISTInference.ppl\n" + + " replicas: 1\n" + + "- cls_name: ''\n" + + " concurrency: 1\n" + + " needs:\n" + + " - mnist.evaluator:MNISTInference.ppl\n" + + " resources:\n" + + " - type: cpu \n" + + " request: 0.1\n" + + " limit: 0.1\n" + + " - type: nvidia.com/gpu \n" + + " request: 1\n" + + " limit: 1\n" + + " - type: memory \n" + + " request: 1\n" + + " limit: 1\n" + + " name: mnist.evaluator:MNISTInference.cmp\n" + + " replicas: 1\n") + .build(); + assertTrue(modelVersionMapper.insert(modelVersion) > 0); + var job = JobEntity.builder() + .name("job-1") + .jobUuid("uuid-1") + .modelVersionId(modelVersion.getId()) + .runtimeVersionId(1L) + .type(JobType.EVALUATION) + .jobStatus(JobStatus.CREATED) + .resultOutputPath("path") + .projectId(projectId) + .ownerId(userId) + .isDeleted(0) + .build(); + assertTrue(jobMapper.addJob(job) > 0); + + var step = StepEntity.builder() + .jobId(job.getId()) + .name("step-1") + .status(StepStatus.CREATED) + .uuid("uuid-step-1") + .concurrency(1) + .taskNum(1) + .poolInfo("") + .build(); + assertTrue(stepMapper.save(step) > 0); + + for (int i = 0; i < 19; i++) { + var res = taskMapper.addTask( + TaskEntity.builder() + .stepId(step.getId()) + .taskUuid(String.format("uuid-%d", i)) + .taskStatus(TaskStatus.CREATED) + .taskRequest("request") + .build()); + assertTrue(res > 0); + } + + // no.1 + var page = taskService.listTasks(job.getJobUuid(), + PageParams.builder().pageNum(1).pageSize(10).build()); + assertEquals(10, page.getSize()); + assertEquals(19, page.getTotal()); + + // no.2 + page = taskService.listTasks(job.getJobUuid(), + PageParams.builder().pageNum(2).pageSize(10).build()); + assertEquals(9, page.getSize()); + assertEquals(19, page.getTotal()); + } +} diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java index 00a4703e69..ed729faebc 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java @@ -41,10 +41,11 @@ import ai.starwhale.mlops.domain.job.JobDao; import ai.starwhale.mlops.domain.job.bo.Job; import ai.starwhale.mlops.domain.job.converter.JobConverter; +import ai.starwhale.mlops.domain.job.po.JobEntity; import ai.starwhale.mlops.domain.job.status.JobStatus; import ai.starwhale.mlops.domain.job.status.JobStatusMachine; import ai.starwhale.mlops.domain.project.ProjectService; -import ai.starwhale.mlops.domain.project.bo.Project; +import ai.starwhale.mlops.domain.project.po.ProjectEntity; import ai.starwhale.mlops.domain.user.UserService; import ai.starwhale.mlops.domain.user.bo.User; import java.util.Date; @@ -125,15 +126,15 @@ public void testCreateViewConfig() { public void testListEvaluationSummary() { given(jobDao.listJobs(same(1L), any())) .willReturn(List.of( - Job.builder() + JobEntity.builder() .id(1L) - .project(Project.builder().id(1L).name("p1").build()) - .status(JobStatus.PAUSED) + .project(ProjectEntity.builder().id(1L).projectName("p1").build()) + .jobStatus(JobStatus.PAUSED) .build(), - Job.builder() + JobEntity.builder() .id(2L) - .project(Project.builder().id(1L).name("p1").build()) - .status(JobStatus.SUCCESS) + .project(ProjectEntity.builder().id(1L).projectName("p1").build()) + .jobStatus(JobStatus.SUCCESS) .build() )); given(jobConvertor.convert(any(Job.class))) diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/job/JobServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/job/JobServiceTest.java index d3d085411a..e684726452 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/job/JobServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/job/JobServiceTest.java @@ -52,6 +52,7 @@ import ai.starwhale.mlops.domain.job.cache.JobLoader; import ai.starwhale.mlops.domain.job.converter.JobBoConverter; import ai.starwhale.mlops.domain.job.converter.JobConverter; +import ai.starwhale.mlops.domain.job.po.JobEntity; import ai.starwhale.mlops.domain.job.po.JobFlattenEntity; import ai.starwhale.mlops.domain.job.spec.JobSpecParser; import ai.starwhale.mlops.domain.job.split.JobSpliterator; @@ -149,7 +150,7 @@ resultQuerier, datasetService, storagePathCoordinator, userService, mock(JobUpda @Test public void testListJobs() { given(jobDao.listJobs(same(1L), same(1L))) - .willReturn(List.of(Job.builder().build(), Job.builder().build())); + .willReturn(List.of(JobEntity.builder().build(), JobEntity.builder().build())); var res = service.listJobs("1", 1L, new PageParams(1, 10)); assertThat(res, allOf( notNullValue(), From 08fdb2c1fbaaf9952b16141f5460ff354e17023f Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Fri, 21 Jul 2023 17:55:21 +0800 Subject: [PATCH 3/3] fix ut --- .../java/ai/starwhale/mlops/domain/PageTest.java | 13 +------------ .../domain/evaluation/EvaluationServiceTest.java | 5 ++--- .../mlops/domain/task/TaskServiceTest.java | 10 +++++----- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java index cae00a8dad..500a1b428f 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/PageTest.java @@ -18,10 +18,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; import ai.starwhale.mlops.common.PageParams; -import ai.starwhale.mlops.datastore.DataStore; import ai.starwhale.mlops.domain.dataset.DatasetService; import ai.starwhale.mlops.domain.dataset.bo.DatasetQuery; import ai.starwhale.mlops.domain.dataset.mapper.DatasetMapper; @@ -67,8 +65,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.ImportAutoConfiguration; import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; @@ -79,6 +75,7 @@ basePackages = { "ai.starwhale.mlops.common", "ai.starwhale.mlops.domain", + "ai.starwhale.mlops.datastore", "ai.starwhale.mlops.reporting", "ai.starwhale.mlops.resulting", "ai.starwhale.mlops.configuration.security"}, @@ -125,14 +122,6 @@ public class PageTest extends MySqlContainerHolder { @Autowired private UserMapper userMapper; - @TestConfiguration - static class TestConfig { - @Bean - DataStore dataStore() { - return mock(DataStore.class); - } - } - String userName = "user-test"; String projectName = "project-test"; diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java index ed729faebc..9e2328f142 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java @@ -39,7 +39,6 @@ import ai.starwhale.mlops.domain.evaluation.mapper.ViewConfigMapper; import ai.starwhale.mlops.domain.evaluation.po.ViewConfigEntity; import ai.starwhale.mlops.domain.job.JobDao; -import ai.starwhale.mlops.domain.job.bo.Job; import ai.starwhale.mlops.domain.job.converter.JobConverter; import ai.starwhale.mlops.domain.job.po.JobEntity; import ai.starwhale.mlops.domain.job.status.JobStatus; @@ -137,9 +136,9 @@ public void testListEvaluationSummary() { .jobStatus(JobStatus.SUCCESS) .build() )); - given(jobConvertor.convert(any(Job.class))) + given(jobConvertor.convert(any(JobEntity.class))) .willAnswer(invocation -> { - Job entity = invocation.getArgument(0); + JobEntity entity = invocation.getArgument(0); return JobVo.builder() .id(String.valueOf(entity.getId())) .uuid("uuid" + entity.getId()) diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/task/TaskServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/task/TaskServiceTest.java index 150247327e..993e976a9e 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/task/TaskServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/task/TaskServiceTest.java @@ -28,7 +28,7 @@ import ai.starwhale.mlops.common.PageParams; import ai.starwhale.mlops.common.proxy.WebServerInTask; import ai.starwhale.mlops.domain.job.JobDao; -import ai.starwhale.mlops.domain.job.bo.Job; +import ai.starwhale.mlops.domain.job.po.JobEntity; import ai.starwhale.mlops.domain.job.step.mapper.StepMapper; import ai.starwhale.mlops.domain.job.step.po.StepEntity; import ai.starwhale.mlops.domain.system.resourcepool.bo.ResourcePool; @@ -71,8 +71,8 @@ public void setup() { @Test public void testListTaskWithJobResourcePool() { - when(jobDao.findJob(any())).thenReturn( - Job.builder().id(1L).resourcePool(ResourcePool.builder().name("a").build()).build()); + when(jobDao.findJobEntity(any())).thenReturn( + JobEntity.builder().id(1L).resourcePool("a").build()); var startedTime = new Date(); var finishedTime = new Date(); when(taskMapper.listTasks(1L)).thenReturn( @@ -100,8 +100,8 @@ public void testListTaskWithJobResourcePool() { @Test public void testListTaskWithStepResourcePool() throws IOException { - when(jobDao.findJob(any())).thenReturn( - Job.builder().id(1L).resourcePool(ResourcePool.builder().name("pool from job").build()).build()); + when(jobDao.findJobEntity(any())).thenReturn( + JobEntity.builder().id(1L).resourcePool("pool from job").build()); var startedTime = new Date(); var finishedTime = new Date(); when(taskMapper.listTasks(1L)).thenReturn(