Use DNS for S3 storage + set storage root directory in application.yml (

#51)

Signed-off-by: Etienne Homer <etiennehomer@gmail.com>

src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java

@@ -54,11 +54,11 @@ public byte[] getInputStream(UUID caseUuid, String fileName) {
         // For archived cases (.zip, .tar, ...), individual files are gzipped in S3 server.
         // Here the requested file is decompressed and simply returned.
         if (S3CaseService.isArchivedCaseFile(caseName)) {
-            caseFileKey = uuidToKeyWithFileName(caseUuid, fileName + GZIP_EXTENSION);
+            caseFileKey = s3CaseService.uuidToKeyWithFileName(caseUuid, fileName + GZIP_EXTENSION);
             return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey,
                     file -> S3CaseService.decompress(Files.readAllBytes(file)));
         } else {
-            caseFileKey = uuidToKeyWithFileName(caseUuid, caseName);
+            caseFileKey = s3CaseService.uuidToKeyWithFileName(caseUuid, caseName);
             return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey,
                     casePath -> IOUtils.toByteArray(DataSource.fromPath(casePath).newInputStream(fileName)));
         }

src/main/java/com/powsybl/caseserver/service/CaseService.java

@@ -168,4 +168,6 @@ default List<CaseInfos> getCasesToReindex() {
     CaseMetadataRepository getCaseMetadataRepository();
 
     ComputationManager getComputationManager();
+
+    String getRootDirectory();
 }

src/main/java/com/powsybl/caseserver/service/FsCaseService.java

@@ -15,6 +15,7 @@
 import com.powsybl.computation.local.LocalComputationManager;
 import com.powsybl.iidm.network.Importer;
 import com.powsybl.iidm.network.Network;
+import jakarta.annotation.PostConstruct;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -35,6 +36,7 @@
 import java.util.stream.Stream;
 
 import static com.powsybl.caseserver.CaseException.createDirectoryNotFound;
+import static com.powsybl.caseserver.service.S3CaseService.DELIMITER;
 
 /**
  * @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
@@ -59,13 +61,28 @@ public class FsCaseService implements CaseService {
     @Autowired
     private CaseInfosService caseInfosService;
 
-    @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}")
+    @Value("${case-home:#{systemProperties['user.home']}}")
+    private String caseHome;
+
+    @Value("${case-subpath}")
+    private String caseSubpath;
+
     private String rootDirectory;
 
     public FsCaseService(CaseMetadataRepository caseMetadataRepository) {
         this.caseMetadataRepository = caseMetadataRepository;
     }
 
+    @PostConstruct
+    private void postConstruct() {
+        rootDirectory = caseHome + DELIMITER + caseSubpath;
+    }
+
+    @Override
+    public String getRootDirectory() {
+        return rootDirectory;
+    }
+
     @Override
     public String getFormat(UUID caseUuid) {
         Path file = getCaseFile(caseUuid);
@@ -300,7 +317,7 @@ public void deleteAllCases() {
     }
 
     public Path getStorageRootDir() {
-        return fileSystem.getPath(rootDirectory);
+        return fileSystem.getPath(getRootDirectory());
     }
 
     private boolean isStorageCreated() {

src/main/java/com/powsybl/caseserver/service/S3CaseService.java

@@ -80,7 +80,8 @@ public class S3CaseService implements CaseService {
     @Value("${spring.cloud.aws.bucket}")
     private String bucketName;
 
-    private static final String CASES_PREFIX = "gsi-cases/";
+    @Value("${case-subpath}")
+    private String rootDirectory;
 
     public static final String NOT_FOUND = " not found";
 
@@ -91,6 +92,11 @@ public S3CaseService(CaseMetadataRepository caseMetadataRepository) {
         this.caseMetadataRepository = caseMetadataRepository;
     }
 
+    @Override
+    public String getRootDirectory() {
+        return rootDirectory;
+    }
+
     String getFormat(Path caseFile) {
         Importer importer = getImporterOrThrowsException(caseFile);
         return importer.getFormat();
@@ -172,7 +178,7 @@ public String getOriginalFilename(UUID caseUuid) {
         return getCaseMetaDataEntity(caseUuid).getOriginalFilename();
     }
 
-    // key format is "gsi-cases/UUID/path/to/file"
+    // key format is "<rootDirectory>/UUID/path/to/file"
     private UUID parseUuidFromKey(String key) {
         int firstSlash = key.indexOf(DELIMITER);
         int secondSlash = key.indexOf(DELIMITER, firstSlash + 1);
@@ -185,11 +191,11 @@ private String parseFilenameFromKey(String key) {
         return key.substring(secondSlash + 1);
     }
 
-    public static String uuidToKeyPrefix(UUID uuid) {
-        return CASES_PREFIX + uuid.toString() + DELIMITER;
+    public String uuidToKeyPrefix(UUID uuid) {
+        return rootDirectory + DELIMITER + uuid.toString() + DELIMITER;
     }
 
-    public static String uuidToKeyWithFileName(UUID uuid, String filename) {
+    public String uuidToKeyWithFileName(UUID uuid, String filename) {
         return uuidToKeyPrefix(uuid) + filename;
     }
 
@@ -257,7 +263,7 @@ public Optional<byte[]> getCaseBytes(UUID caseUuid) {
     public List<CaseInfos> getCases() {
         List<CaseInfos> caseInfosList = new ArrayList<>();
         CaseInfos caseInfos;
-        for (S3Object o : getCaseS3Objects(CASES_PREFIX)) {
+        for (S3Object o : getCaseS3Objects(rootDirectory + DELIMITER)) {
             caseInfos = getCaseInfos(parseUuidFromKey(o.key()));
             if (Objects.nonNull(caseInfos)) {
                 caseInfosList.add(caseInfos);
@@ -322,7 +328,7 @@ public Set<String> listName(UUID caseUuid, String regex) {
             filenames = List.of(removeExtension(originalFilename, "." + getCompressionFormat(caseUuid)));
         } else {
             List<S3Object> s3Objects = getCaseS3Objects(caseUuid);
-            filenames = s3Objects.stream().map(obj -> Paths.get(obj.key()).toString().replace(CASES_PREFIX + caseUuid.toString() + DELIMITER, "")).toList();
+            filenames = s3Objects.stream().map(obj -> Paths.get(obj.key()).toString().replace(rootDirectory + DELIMITER + caseUuid.toString() + DELIMITER, "")).toList();
             // For archived cases :
             if (isArchivedCaseFile(originalFilename)) {
                 filenames = filenames.stream()
@@ -414,9 +420,9 @@ private void copyEntry(UUID sourcecaseUuid, UUID caseUuid, String fileName) {
         // To optimize copy, files to copy are not downloaded on the case-server. They are directly copied on the S3 server.
         CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder()
                 .sourceBucket(bucketName)
-                .sourceKey(CASES_PREFIX + sourcecaseUuid + DELIMITER + fileName)
+                .sourceKey(rootDirectory + DELIMITER + sourcecaseUuid + DELIMITER + fileName)
                 .destinationBucket(bucketName)
-                .destinationKey(CASES_PREFIX + caseUuid + DELIMITER + fileName)
+                .destinationKey(rootDirectory + DELIMITER + caseUuid + DELIMITER + fileName)
                 .build();
         try {
             s3Client.copyObject(copyObjectRequest);
@@ -529,7 +535,7 @@ public void deleteCase(UUID caseUuid) {
     public void deleteAllCases() {
         ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder()
                 .bucket(bucketName)
-                .prefix(CASES_PREFIX)
+                .prefix(rootDirectory + DELIMITER)
                 .build();
 
         ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest);

src/main/resources/application-local.yml

@@ -11,14 +11,12 @@ spring:
       ## to fill if authentication is needed
       # user:
       # password:
+  cloud:
+    aws:
+      endpoint: http://localhost:19000
 
 powsybl-ws:
   database:
     host: localhost
 
-cleaning-cases-cron: 0 * 2 * * ?
-
-cloud:
-  aws:
-    s3:
-      endpoint: 172.17.0.1
+cleaning-cases-cron: 0 * 2 * * ?

src/main/resources/config/application.yaml

@@ -11,12 +11,13 @@ spring:
     aws:
       s3:
         path-style-access-enabled: true
-      endpoint: http://172.17.0.1:19000
+      # classic minio port, useful default for exploring
+      endpoint: http://s3-storage:9000
       region:
         profile:
           name: default
         static: test
-      bucket: bucket-gridsuite
+      bucket: my-bucket
       credentials:
         access-key: minioadmin
         secret-key: minioadmin
@@ -28,4 +29,6 @@ powsybl-ws:
 cleaning-cases-cron: 0 * 2 * * ?
 
 storage:
-  type: FS  # FS or S3
+  type: FS  # FS or S3
+
+case-subpath: cases

src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java

@@ -12,7 +12,6 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.http.MediaType;
@@ -33,7 +32,7 @@
  * @author Jamal KHEYYAD <jamal.kheyyad at rte-international.com>
  */
 @AutoConfigureMockMvc
-@SpringBootTest(classes = {CaseApplication.class}, properties = {"case-store-directory=/cases"})
+@SpringBootTest(classes = {CaseApplication.class})
 abstract class AbstractSupervisionControllerTest {
     @Autowired
     private SupervisionService supervisionService;
@@ -43,9 +42,6 @@ abstract class AbstractSupervisionControllerTest {
     @Autowired
     protected MockMvc mockMvc;
 
-    @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}")
-    String rootDirectory;
-
     private static final String TEST_CASE = "testCase.xiidm";
     FileSystem fileSystem;
 
@@ -112,7 +108,7 @@ void tearDown() throws Exception {
     }
 
     private void createStorageDir() throws IOException {
-        Path path = fileSystem.getPath(rootDirectory);
+        Path path = fileSystem.getPath(caseService.getRootDirectory());
         if (!Files.exists(path)) {
             Files.createDirectories(path);
         }

src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java

@@ -21,7 +21,7 @@
  * @author Jamal KHEYYAD <jamal.kheyyad at rte-international.com>
  */
 @AutoConfigureMockMvc
-@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"})
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK)
 @TestPropertySource(properties = {"storage.type=FS"})
 class FsSupervisionControllerTest extends AbstractSupervisionControllerTest {
 

src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java

@@ -12,7 +12,6 @@
 import com.powsybl.commons.datasource.DataSource;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.cloud.stream.function.StreamBridge;
 import org.springframework.mock.web.MockMultipartFile;
@@ -48,9 +47,6 @@ public abstract class AbstractCaseDataSourceControllerTest {
 
     protected static CaseService caseService;
 
-    @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}")
-    protected String rootDirectory;
-
     static final String CGMES_ZIP_NAME = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2.zip";
 
     static final String CGMES_FILE_NAME = "CGMES_v2415_MicroGridTestConfiguration_BC_BE_v2/MicroGridTestConfiguration_BC_BE_DL_V2.xml";

src/test/java/com/powsybl/caseserver/datasource/FsCaseDataSourceControllerTest.java

@@ -40,11 +40,11 @@ class FsCaseDataSourceControllerTest extends AbstractCaseDataSourceControllerTes
     void setUp() throws URISyntaxException, IOException {
         caseService = fsCaseService;
         cgmesCaseUuid = UUID.randomUUID();
-        Path path = fileSystem.getPath(rootDirectory);
+        Path path = fileSystem.getPath(caseService.getRootDirectory());
         if (!Files.exists(path)) {
             Files.createDirectories(path);
         }
-        Path cgmesCaseDirectory = fileSystem.getPath(rootDirectory).resolve(cgmesCaseUuid.toString());
+        Path cgmesCaseDirectory = fileSystem.getPath(caseService.getRootDirectory()).resolve(cgmesCaseUuid.toString());
         if (!Files.exists(cgmesCaseDirectory)) {
             Files.createDirectories(cgmesCaseDirectory);
         }

src/test/java/com/powsybl/caseserver/service/AbstractCaseControllerTest.java

@@ -18,7 +18,6 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.cloud.stream.binder.test.OutputDestination;
@@ -55,7 +54,7 @@
  * @author Franck Lecuyer <franck.lecuyer at rte-france.com>
  */
 @AutoConfigureMockMvc
-@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"})
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK)
 @ContextConfigurationWithTestChannel
 abstract class AbstractCaseControllerTest {
     private static final String TEST_CASE = "testCase.xiidm";
@@ -81,9 +80,6 @@ abstract class AbstractCaseControllerTest {
     @Autowired
     private ObjectMapper mapper;
 
-    @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}")
-    String rootDirectory;
-
     FileSystem fileSystem;
 
     private final String caseImportDestination = "case.import.destination";
@@ -96,7 +92,7 @@ public void tearDown() throws Exception {
     }
 
     private void createStorageDir() throws IOException {
-        Path path = fileSystem.getPath(rootDirectory);
+        Path path = fileSystem.getPath(caseService.getRootDirectory());
         if (!Files.exists(path)) {
             Files.createDirectories(path);
         }
@@ -726,7 +722,7 @@ private static String getDateSearchTerm(String entsoeFormatDate) {
     @Test
     void invalidFileInCaseDirectoryShouldBeIgnored() throws Exception {
         createStorageDir();
-        Path filePath = fileSystem.getPath(rootDirectory).resolve("randomFile.txt");
+        Path filePath = fileSystem.getPath(caseService.getRootDirectory()).resolve("randomFile.txt");
         Files.createFile(filePath);
         importCase(TEST_CASE, false);
 

src/test/java/com/powsybl/caseserver/service/MinioContainerConfig.java

@@ -19,7 +19,7 @@
  */
 public interface MinioContainerConfig {
     String MINIO_DOCKER_IMAGE_NAME = "minio/minio";
-    String BUCKET_NAME = "bucket-gridsuite";
+    String BUCKET_NAME = "my-bucket";
     // Just a fixed version, latest at the time of writing this
     String MINIO_DOCKER_IMAGE_VERSION = "RELEASE.2023-09-27T15-22-50Z";
     int MINIO_PORT = 9000;