Skip to content

Commit

Permalink
fix(controller): refactor and fix bugs for job-dataset version (#1563)
Browse files Browse the repository at this point in the history 
* fix bugs with dataset version

* fix bug: can not find dataset version
  • Loading branch information
@dreamlandliu
dreamlandliu authored Nov 29, 2022
1 parent c9b8325 commit 2119afd
Show file tree
Hide file tree
Showing 27 changed files with 253 additions and 315 deletions.
19 changes: 0 additions & 19 deletions server/controller/src/main/java/ai/starwhale/mlops/api/DatasetApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,25 +280,6 @@ ResponseEntity<ResponseMessage<Map>> signLinks(@PathVariable(name = "projectName
@Parameter(name = "expTimeMillis", description = "the link will be expired after expTimeMillis")
@RequestParam(name = "expTimeMillis", required = false) Long expTimeMillis);

@Operation(summary = "Set the tag of the dataset version")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "ok")})
@PutMapping(value = "/project/{projectUrl}/dataset/{datasetUrl}/version/{versionUrl}")
@PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER')")
ResponseEntity<ResponseMessage<String>> modifyDatasetVersionInfo(
@Parameter(
in = ParameterIn.PATH,
description = "Project Url",
schema = @Schema())
@PathVariable("projectUrl")
String projectUrl,
@Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
@PathVariable("datasetUrl")
String datasetUrl,
@Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
@PathVariable("versionUrl")
String versionUrl,
@Valid @RequestBody DatasetTagRequest datasetTagRequest);

@Operation(
summary = "Manage tag of the dataset version",
description = "add|remove|set tags"
Expand Down
22 changes: 4 additions & 18 deletions server/controller/src/main/java/ai/starwhale/mlops/api/DatasetController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion;
import ai.starwhale.mlops.domain.dataset.bo.DatasetVersionQuery;
import ai.starwhale.mlops.domain.dataset.dataloader.DataReadRequest;
import ai.starwhale.mlops.domain.dataset.po.DatasetVersionEntity;
import ai.starwhale.mlops.domain.dataset.upload.DatasetUploader;
import ai.starwhale.mlops.exception.SwProcessException;
import ai.starwhale.mlops.exception.SwProcessException.ErrorType;
Expand Down Expand Up @@ -227,10 +226,10 @@ public void pullLinkContent(String projectUrl, String datasetUrl, String version
new SwValidationException(ValidSubject.DATASET, "please provide name and version for the DS "),
HttpStatus.BAD_REQUEST);
}
DatasetVersionEntity datasetVersionEntity = datasetService.query(projectUrl, datasetUrl, versionUrl);
DatasetVersion datasetVersion = datasetService.query(projectUrl, datasetUrl, versionUrl);
try {
ServletOutputStream outputStream = httpResponse.getOutputStream();
outputStream.write(datasetService.dataOf(datasetVersionEntity.getId(), uri, offset, size));
outputStream.write(datasetService.dataOf(datasetVersion.getId(), uri, offset, size));
outputStream.flush();
} catch (IOException e) {
throw new SwProcessException(ErrorType.NETWORK, "error write data to response", e);
Expand All @@ -241,22 +240,9 @@ public void pullLinkContent(String projectUrl, String datasetUrl, String version
@Override
public ResponseEntity<ResponseMessage<Map>> signLinks(String projectUrl, String datasetUrl, String versionUrl,
Set<String> uris, Long expTimeMillis) {
DatasetVersionEntity datasetVersionEntity = datasetService.query(projectUrl, datasetUrl, versionUrl);
DatasetVersion datasetVersion = datasetService.query(projectUrl, datasetUrl, versionUrl);
return ResponseEntity.ok(Code.success.asResponse(
datasetService.signLinks(datasetVersionEntity.getId(), uris, expTimeMillis)));
}

@Override
public ResponseEntity<ResponseMessage<String>> modifyDatasetVersionInfo(
String projectUrl, String datasetUrl, String versionUrl, DatasetTagRequest datasetTagRequest) {
Boolean res = datasetService.modifyDatasetVersion(projectUrl, datasetUrl, versionUrl,
DatasetVersion.builder().tag(datasetTagRequest.getTag()).build());
if (!res) {
throw new StarwhaleApiException(
new SwValidationException(ValidSubject.DATASET, "Modify dataset failed."),
HttpStatus.INTERNAL_SERVER_ERROR);
}
return ResponseEntity.ok(Code.success.asResponse("success"));
datasetService.signLinks(datasetVersion.getId(), uris, expTimeMillis)));
}

@Override
Expand Down
29 changes: 28 additions & 1 deletion server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,17 @@
import ai.starwhale.mlops.domain.bundle.tag.HasTag;
import ai.starwhale.mlops.domain.bundle.tag.HasTagWrapper;
import ai.starwhale.mlops.domain.bundle.tag.TagAccessor;
import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion;
import ai.starwhale.mlops.domain.dataset.mapper.DatasetMapper;
import ai.starwhale.mlops.domain.dataset.mapper.DatasetVersionMapper;
import ai.starwhale.mlops.domain.dataset.po.DatasetEntity;
import ai.starwhale.mlops.domain.dataset.po.DatasetVersionEntity;
import ai.starwhale.mlops.domain.job.mapper.JobDatasetVersionMapper;
import ai.starwhale.mlops.exception.SwValidationException;
import ai.starwhale.mlops.exception.SwValidationException.ValidSubject;
import ai.starwhale.mlops.exception.api.StarwhaleApiException;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
Expand All @@ -45,13 +50,17 @@ public class DatasetDao implements BundleAccessor, BundleVersionAccessor, TagAcc

private final DatasetMapper datasetMapper;
private final DatasetVersionMapper datasetVersionMapper;

private final JobDatasetVersionMapper jobDatasetVersionMapper;
private final IdConverter idConvertor;
private final VersionAliasConverter versionAliasConvertor;

public DatasetDao(DatasetMapper datasetMapper, DatasetVersionMapper datasetVersionMapper,
JobDatasetVersionMapper jobDatasetVersionMapper,
IdConverter idConvertor, VersionAliasConverter versionAliasConvertor) {
this.datasetMapper = datasetMapper;
this.datasetVersionMapper = datasetVersionMapper;
this.jobDatasetVersionMapper = jobDatasetVersionMapper;
this.idConvertor = idConvertor;
this.versionAliasConvertor = versionAliasConvertor;
}
Expand All @@ -64,12 +73,30 @@ public Long getDatasetVersionId(String versionUrl, Long datasetId) {
if (entity == null) {
throw new StarwhaleApiException(
new SwValidationException(ValidSubject.MODEL,
String.format("Unable to find Runtime %s", versionUrl)),
String.format("Unable to find dataset version %s, %d", versionUrl, datasetId)),
HttpStatus.BAD_REQUEST);
}
return entity.getId();
}

public DatasetVersion getDatasetVersion(Long versionId) {
DatasetVersionEntity versionEntity = datasetVersionMapper.find(versionId);
if (null == versionEntity) {
throw new SwValidationException(ValidSubject.DATASET, "Can not find dataset version" + versionId);
}
DatasetEntity datasetEntity = datasetMapper.find(versionEntity.getDatasetId());
if (null == datasetEntity) {
throw new SwValidationException(ValidSubject.DATASET,
"Can not find dataset" + versionEntity.getDatasetId());
}
return DatasetVersion.fromEntity(datasetEntity, versionEntity);
}

public List<DatasetVersion> listDatasetVersionsOfJob(Long jobId) {
List<Long> versionIds = jobDatasetVersionMapper.listDatasetVersionIdsByJobId(jobId);
return versionIds.stream().map(this::getDatasetVersion).collect(Collectors.toList());
}

@Override
public BundleEntity findById(Long id) {
return datasetMapper.find(id);
Expand Down
21 changes: 2 additions & 19 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,19 +206,6 @@ private DatasetInfoVo toDatasetInfoVo(DatasetEntity ds, DatasetVersionEntity ver
}
}

public Boolean modifyDatasetVersion(String projectUrl, String datasetUrl, String versionUrl,
DatasetVersion version) {
Long versionId = bundleManager.getBundleVersionId(BundleVersionUrl
.create(projectUrl, datasetUrl, versionUrl));
DatasetVersionEntity entity = DatasetVersionEntity.builder()
.id(versionId)
.versionTag(version.getTag())
.build();
int update = datasetVersionMapper.update(entity);
log.info("Dataset Version has been modified. ID={}", entity.getId());
return update > 0;
}

public Boolean manageVersionTag(String projectUrl, String datasetUrl, String versionUrl,
TagAction tagAction) {

Expand Down Expand Up @@ -296,14 +283,10 @@ private List<DatasetInfoVo> swDatasetInfoOfDs(DatasetEntity ds) {
.map(entity -> toDatasetInfoVo(ds, entity)).collect(Collectors.toList());
}

public DatasetVersionEntity query(String projectUrl, String datasetUrl, String versionUrl) {
public DatasetVersion query(String projectUrl, String datasetUrl, String versionUrl) {
Long versionId = bundleManager.getBundleVersionId(BundleVersionUrl
.create(projectUrl, datasetUrl, versionUrl));
DatasetVersionEntity versionEntity = datasetVersionMapper.find(versionId);
if (null == versionEntity) {
throw new StarwhaleApiException(new SwValidationException(ValidSubject.DATASET), HttpStatus.NOT_FOUND);
}
return versionEntity;
return datasetDao.getDatasetVersion(versionId);
}


Expand Down
40 changes: 0 additions & 40 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/bo/DatasetObject.java
View file

This file was deleted.

42 changes: 34 additions & 8 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/bo/DatasetVersion.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

package ai.starwhale.mlops.domain.dataset.bo;

import ai.starwhale.mlops.domain.dataset.po.DatasetEntity;
import ai.starwhale.mlops.domain.dataset.po.DatasetVersionEntity;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
Expand All @@ -27,15 +29,39 @@
@NoArgsConstructor
public class DatasetVersion {

public static final Integer STATUS_AVAILABLE = 1;
public static final Integer STATUS_UN_AVAILABLE = 0;
private Long id;

private String name;

private Long datasetId;
private Long versionOrder;
private String datasetName;
private Long ownerId;

private String tag;

private String meta;

private String versionName;
private String versionTag;
private String versionMeta;
private String filesUploaded;
private String storagePath;
private Long size;
private String indexTable;
/**
* 0 - unavailable 1 - available
*/
private Integer status = STATUS_UN_AVAILABLE;

public static DatasetVersion fromEntity(DatasetEntity datasetEntity, DatasetVersionEntity versionEntity) {
return DatasetVersion.builder()
.id(versionEntity.getId())
.datasetId(versionEntity.getDatasetId())
.versionOrder(versionEntity.getVersionOrder())
.datasetName(datasetEntity.getDatasetName())
.ownerId(versionEntity.getOwnerId())
.versionName(versionEntity.getVersionName())
.versionTag(versionEntity.getVersionTag())
.versionMeta(versionEntity.getVersionMeta())
.filesUploaded(versionEntity.getFilesUploaded())
.storagePath(versionEntity.getStoragePath())
.size(versionEntity.getSize())
.indexTable(versionEntity.getIndexTable())
.build();
}
}
16 changes: 8 additions & 8 deletions ...troller/src/main/java/ai/starwhale/mlops/domain/dataset/converter/DatasetBoConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,20 @@
package ai.starwhale.mlops.domain.dataset.converter;

import ai.starwhale.mlops.domain.dataset.bo.DataSet;
import ai.starwhale.mlops.domain.dataset.po.DatasetVersionEntity;
import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion;
import org.springframework.stereotype.Component;

@Component
public class DatasetBoConverter {

public DataSet fromEntity(DatasetVersionEntity datasetVersionEntity) {
public DataSet fromVersion(DatasetVersion datasetVersion) {
return DataSet.builder()
.id(datasetVersionEntity.getId())
.name(datasetVersionEntity.getDatasetName())
.version(datasetVersionEntity.getVersionName())
.size(datasetVersionEntity.getSize())
.path(datasetVersionEntity.getStoragePath())
.indexTable(datasetVersionEntity.getIndexTable())
.id(datasetVersion.getId())
.name(datasetVersion.getDatasetName())
.version(datasetVersion.getVersionName())
.size(datasetVersion.getSize())
.path(datasetVersion.getStoragePath())
.indexTable(datasetVersion.getIndexTable())
.build();
}
}
4 changes: 3 additions & 1 deletion ...ntroller/src/main/java/ai/starwhale/mlops/domain/dataset/mapper/DatasetVersionMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ public String findByNameAndDatasetIdSql(@Param("datasetId") Long datasetId,
SELECT(COLUMNS);
FROM("dataset_version");
WHERE("version_name = #{versionName}");
WHERE("dataset_id = #{datasetId}");
if (Objects.nonNull(datasetId)) {
WHERE("dataset_id = #{datasetId}");
}
}
}.toString();

Expand Down
11 changes: 4 additions & 7 deletions ...r/controller/src/main/java/ai/starwhale/mlops/domain/dataset/po/DatasetVersionEntity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import ai.starwhale.mlops.common.BaseEntity;
import ai.starwhale.mlops.domain.bundle.base.BundleVersionEntity;
import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
Expand All @@ -31,12 +32,10 @@
@AllArgsConstructor
public class DatasetVersionEntity extends BaseEntity implements BundleVersionEntity {

public static final Integer STATUS_AVAILABLE = 1;
public static final Integer STATUS_UN_AVAILABLE = 0;
private Long id;
private Long datasetId;
private Long versionOrder;
private String datasetName;

private Long ownerId;
private String versionName;
private String versionTag;
Expand All @@ -45,10 +44,8 @@ public class DatasetVersionEntity extends BaseEntity implements BundleVersionEnt
private String storagePath;
private Long size;
private String indexTable;
/**
* 0 - unavailable 1 - available
*/
private Integer status = STATUS_UN_AVAILABLE;

private Integer status = DatasetVersion.STATUS_UN_AVAILABLE;

@Override
public String getName() {
Expand Down
Loading

0 comments on commit 2119afd

Please sign in to comment.