handle zip case file with s3

Signed-off-by: Rehili Ghazwa <ghazwarhili@gmail.com>
powsybl · Sep 5, 2024 · bed9105 · bed9105
1 parent 7bd2878
commit bed9105
Show file tree

Hide file tree

Showing 11 changed files with 887 additions and 872 deletions.
diff --git a/src/main/java/com/powsybl/caseserver/datasource/util/FsCaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/util/FsCaseDataSourceService.java
@@ -31,7 +31,7 @@
 public class FsCaseDataSourceService implements CaseDataSourceService {
 
     @Autowired
-    private FsCaseService caseService;
+    private FsCaseService fsCaseService;
 
     @Override
     public String getBaseName(UUID caseUuid) {
@@ -91,12 +91,12 @@ public Set<String> listName(UUID caseUuid, String regex) {
     }
 
     private DataSource initDatasource(UUID caseUuid) {
-        Path file = caseService.getCaseFile(caseUuid);
+        Path file = fsCaseService.getCaseFile(caseUuid);
         return DataSource.fromPath(file);
     }
 
     private DataSource getDatasource(UUID caseUuid) {
-        caseService.checkStorageInitialization();
+        fsCaseService.checkStorageInitialization();
         return initDatasource(caseUuid);
     }
 

diff --git a/src/main/java/com/powsybl/caseserver/datasource/util/S3CaseDataSourceService.java b/src/main/java/com/powsybl/caseserver/datasource/util/S3CaseDataSourceService.java
@@ -9,9 +9,8 @@
 import com.powsybl.caseserver.server.CaseService;
 import com.powsybl.caseserver.server.S3CaseService;
 import com.powsybl.commons.datasource.DataSource;
-
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.Functions.FailableFunction;
+import org.apache.commons.lang3.function.FailableFunction;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.stereotype.Service;
@@ -20,6 +19,7 @@
 import java.io.UncheckedIOException;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.Set;
 import java.util.UUID;
 
@@ -30,6 +30,8 @@
 @ComponentScan(basePackageClasses = CaseService.class)
 public class S3CaseDataSourceService implements CaseDataSourceService {
 
+    private static final String CASES_PREFIX = "gsi-cases/";
+
     @Autowired
     private S3CaseService caseService;
 
@@ -54,8 +56,14 @@ public Boolean datasourceExists(UUID caseUuid, String fileName) {
 
     @Override
     public byte[] getInputStream(UUID caseUuid, String fileName) {
-        return withS3DownloadedDataSource(caseUuid,
-            datasource -> IOUtils.toByteArray(datasource.newInputStream(fileName)));
+        final String parsedFileName = fileName.substring(fileName.indexOf('/') + 1);
+        final var caseFileKey = uuidToPrefixKey(caseUuid) + parsedFileName;
+        return withS3DownloadedDataSource(caseUuid, caseFileKey,
+            datasource -> IOUtils.toByteArray(datasource.newInputStream(parsedFileName)));
+    }
+
+    private String uuidToPrefixKey(UUID uuid) {
+        return CASES_PREFIX + uuid.toString() + "/";
     }
 
     @Override
@@ -77,8 +85,15 @@ public Set<String> listName(UUID caseUuid, String regex) {
     }
 
     public <R, T extends Throwable> R withS3DownloadedDataSource(UUID caseUuid, FailableFunction<DataSource, R, T> f) {
-        // TODO replace with lang3 3.11 FailableFunction::compose
-        return caseService.withS3DownloadedTempPath(caseUuid, path -> f.apply(DataSource.fromPath(path)));
+        FailableFunction<Path, DataSource, T> pathToDataSource = DataSource::fromPath;
+        FailableFunction<Path, R, T> composedFunction = pathToDataSource.andThen(f);
+        return caseService.withS3DownloadedTempPath(caseUuid, composedFunction);
+    }
+
+    public <R, T extends Throwable> R withS3DownloadedDataSource(UUID caseUuid, String caseFileKey, FailableFunction<DataSource, R, T> f) {
+        FailableFunction<Path, DataSource, T> pathToDataSource = DataSource::fromPath;
+        FailableFunction<Path, R, T> composedFunction = pathToDataSource.andThen(f);
+        return caseService.withS3DownloadedTempPath(caseUuid, caseFileKey, composedFunction);
     }
 
 }

diff --git a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java
@@ -56,6 +56,11 @@ public List<CaseInfos> getAllCaseInfos() {
         return Lists.newArrayList(caseInfosRepository.findAll());
     }
 
+    /*
+ The query is an elasticsearch (Lucene) form query, so here it will be :
+ date:XXX AND geographicalCode:(X)
+ date:XXX AND geographicalCode:(X OR Y OR Z)
+*/
     public List<CaseInfos> searchCaseInfos(@NonNull final String query) {
         NativeQuery searchQuery = new NativeQueryBuilder().withQuery(QueryStringQuery.of(qs -> qs.query(query))._toQuery()).build();
         return Lists.newArrayList(operations.search(searchQuery, CaseInfos.class)

diff --git a/src/main/java/com/powsybl/caseserver/repository/StorageConfig.java b/src/main/java/com/powsybl/caseserver/repository/StorageConfig.java
@@ -7,8 +7,8 @@
 package com.powsybl.caseserver.repository;
 
 import com.powsybl.caseserver.server.CaseService;
-import com.powsybl.caseserver.server.FileSystemStorageService;
-import com.powsybl.caseserver.server.ObjectStorageService;
+import com.powsybl.caseserver.server.FsCaseService;
+import com.powsybl.caseserver.server.S3CaseService;
 import com.powsybl.caseserver.datasource.util.CaseDataSourceService;
 import com.powsybl.caseserver.datasource.util.FsCaseDataSourceService;
 import com.powsybl.caseserver.datasource.util.S3CaseDataSourceService;
@@ -35,9 +35,9 @@ public StorageConfig(@Value("${storage.type}")String storageType, CaseMetadataRe
     @Bean
     public CaseService storageService() {
         if ("file".equals(storageType)) {
-            return new FileSystemStorageService(caseMetadataRepository);
+            return new FsCaseService(caseMetadataRepository);
         } else if ("S3".equals(storageType)) {
-            return new ObjectStorageService(caseMetadataRepository);
+            return new S3CaseService(caseMetadataRepository);
         }
         return null;
     }