fix(controller): fix dataset uri download error (#1028)

* offset/size parameter to datastore protocol * adapt to swds bin format
star-whale · Aug 28, 2022 · c88a05b · c88a05b
1 parent 0ec775c
commit c88a05b
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 10 deletions.
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/DatasetApi.java b/server/controller/src/main/java/ai/starwhale/mlops/api/DatasetApi.java
@@ -239,9 +239,9 @@ void pullLinkContent(
         @Parameter(name = "authName", description = "auth name the link used")
         @RequestParam(name = "authName",required = false) String authName,
         @Parameter(name = "offset", description = "offset in the content")
-        @RequestParam(name = "offset",required = false) Long offset,
+        @RequestParam(name = "offset",required = false) String offset,
         @Parameter(name = "size", description = "data size")
-        @RequestParam(name = "size",required = false) Long size,
+        @RequestParam(name = "size",required = false) String size,
         HttpServletResponse httpResponse);
 
 

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/DatasetController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/DatasetController.java
@@ -189,7 +189,7 @@ public void pullDS(String projectUrl, String datasetUrl, String versionUrl,
 
     @Override
     public void pullLinkContent(String projectUrl, String datasetUrl, String versionUrl,
-        String uri,String authName,Long offset,Long size, HttpServletResponse httpResponse) {
+        String uri,String authName,String offset,String size, HttpServletResponse httpResponse) {
         if(!StringUtils.hasText(datasetUrl) || !StringUtils.hasText(versionUrl) ){
             throw new StarWhaleApiException(new SWValidationException(ValidSubject.SWDS)
                 .tip("please provide name and version for the DS "), HttpStatus.BAD_REQUEST);

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/swds/SWDatasetService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/swds/SWDatasetService.java
@@ -291,8 +291,8 @@ public SWDatasetVersionEntity query(String projectUrl, String datasetUrl, String
         return versionEntity;
     }
 
-    public byte[] dataOf(Long datasetId, String uri, String authName, Long offset,
-        Long size){
+    public byte[] dataOf(Long datasetId, String uri, String authName, String offset,
+        String size){
         return dsFileGetter.dataOf(datasetId,uri,authName,offset,size);
     }
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/swds/objectstore/DSFileGetter.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/swds/objectstore/DSFileGetter.java
@@ -16,9 +16,13 @@
 
 package ai.starwhale.mlops.domain.swds.objectstore;
 
+import ai.starwhale.mlops.datastore.ColumnType;
+import ai.starwhale.mlops.domain.swds.mapper.SWDatasetVersionMapper;
+import ai.starwhale.mlops.domain.swds.po.SWDatasetVersionEntity;
 import ai.starwhale.mlops.exception.SWProcessException;
 import ai.starwhale.mlops.exception.SWProcessException.ErrorType;
 import ai.starwhale.mlops.storage.StorageAccessService;
+import ai.starwhale.mlops.storage.StorageObjectInfo;
 import java.io.IOException;
 import java.io.InputStream;
 import lombok.extern.slf4j.Slf4j;
@@ -30,15 +34,33 @@ public class DSFileGetter {
 
     final StorageAccessParser storageAccessParser;
 
-    public DSFileGetter(StorageAccessParser storageAccessParser) {
+    final SWDatasetVersionMapper swDatasetVersionMapper;
+
+    public DSFileGetter(StorageAccessParser storageAccessParser,
+        SWDatasetVersionMapper swDatasetVersionMapper) {
         this.storageAccessParser = storageAccessParser;
+        this.swDatasetVersionMapper = swDatasetVersionMapper;
     }
 
-    public byte[] dataOf(Long datasetId, String uri, String authName, Long offset,
-        Long size) {
+    public byte[] dataOf(Long datasetId, String uri, String authName, String offset,
+        String size) {
         StorageAccessService storageAccessService = storageAccessParser.getStorageAccessServiceFromAuth(
             datasetId, uri, authName);
-        try (InputStream inputStream = storageAccessService.get(new StorageUri(uri).getPath(),offset,size)) {
+        String path = new StorageUri(uri).getPath();
+        StorageObjectInfo objectInfo;
+        try {
+            objectInfo = storageAccessService.head(path);
+        } catch (IOException e) {
+            log.error("error while accessing storage ", e);
+            throw new SWProcessException(ErrorType.STORAGE).tip(
+                String.format("error while accessing storage : %s", e.getMessage()));
+        }
+        if(!objectInfo.isExists()){
+            SWDatasetVersionEntity versionById = swDatasetVersionMapper.getVersionById(datasetId);
+            path =  versionById.getStoragePath()+"/"+ path;
+        }
+        try (InputStream inputStream = storageAccessService.get(path,
+            (long)ColumnType.INT64.decode(offset),(long)ColumnType.INT64.decode(size))) {
             return inputStream.readAllBytes();
         } catch (IOException ioException) {
             log.error("error while accessing storage ", ioException);

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/swds/objectstore/StorageUri.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/swds/objectstore/StorageUri.java
@@ -49,7 +49,8 @@ public class StorageUri {
     public StorageUri(String uri){
         Matcher matcher = URI_PATTERN.matcher(uri);
         if(!matcher.matches()){
-            throw new SWValidationException(ValidSubject.SWDS).tip("unsupported data uri: "+uri);
+            this.path=uri;
+            return;
         }
         schema = matcher.group(2);
         String up = matcher.group(4);