diff --git a/src/main/java/com/powsybl/caseserver/CaseException.java b/src/main/java/com/powsybl/caseserver/CaseException.java index 58fd973..e98fda3 100644 --- a/src/main/java/com/powsybl/caseserver/CaseException.java +++ b/src/main/java/com/powsybl/caseserver/CaseException.java @@ -20,6 +20,7 @@ public final class CaseException extends RuntimeException { public enum Type { FILE_NOT_IMPORTABLE, + FILE_NOT_FOUND, STORAGE_DIR_NOT_CREATED, ILLEGAL_FILE_NAME, DIRECTORY_ALREADY_EXISTS, @@ -27,7 +28,9 @@ public enum Type { DIRECTORY_NOT_FOUND, ORIGINAL_FILE_NOT_FOUND, TEMP_FILE_INIT, - TEMP_FILE_PROCESS, TEMP_DIRECTORY_CREATION, + TEMP_FILE_PROCESS, + TEMP_DIRECTORY_CREATION, + ZIP_FILE_PROCESS, UNSUPPORTED_FORMAT } @@ -43,6 +46,11 @@ public CaseException(Type type, String message, Exception e) { this.type = type; } + public CaseException(Type type, String message, Throwable e) { + super(message, e); + this.type = type; + } + public Type getType() { return type; } @@ -72,9 +80,14 @@ public static CaseException createFileNotImportable(Path file) { return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file); } - public static CaseException createFileNotImportable(String file) { + public static CaseException createFileNotImportable(String file, Exception e) { Objects.requireNonNull(file); - return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file); + return new CaseException(Type.FILE_NOT_IMPORTABLE, "This file cannot be imported: " + file, e); + } + + public static CaseException getFileNameNotFound(UUID uuid) { + Objects.requireNonNull(uuid); + return new CaseException(Type.FILE_NOT_FOUND, "The file name with the following uuid doesn't exist: " + uuid); } public static CaseException createStorageNotInitialized(Path storageRootDir) { @@ -97,6 +110,11 @@ public static CaseException initTempFile(UUID uuid, Exception e) { return new CaseException(Type.TEMP_FILE_INIT, "Error initializing temporary case file: " + uuid, e); } + public static CaseException initTempFile(UUID uuid, Throwable e) { + Objects.requireNonNull(uuid); + return new CaseException(Type.TEMP_FILE_INIT, "Error initializing temporary case file: " + uuid, e); + } + public static CaseException initTempFile(UUID uuid) { return CaseException.initTempFile(uuid, null); } @@ -106,10 +124,19 @@ public static CaseException processTempFile(UUID uuid, Exception e) { return new CaseException(Type.TEMP_FILE_PROCESS, "Error processing temporary case file: " + uuid, e); } + public static CaseException processTempFile(UUID uuid, Throwable e) { + Objects.requireNonNull(uuid); + return new CaseException(Type.TEMP_FILE_PROCESS, "Error processing temporary case file: " + uuid, e); + } + public static CaseException processTempFile(UUID uuid) { return CaseException.processTempFile(uuid, null); } + public static CaseException importZipContent(UUID uuid, Exception e) { + return new CaseException(Type.ZIP_FILE_PROCESS, "Error processing zip content file: " + uuid, e); + } + public static CaseException createUnsupportedFormat(String format) { return new CaseException(Type.UNSUPPORTED_FORMAT, "The format: " + format + " is unsupported"); } diff --git a/src/main/java/com/powsybl/caseserver/service/CaseService.java b/src/main/java/com/powsybl/caseserver/service/CaseService.java index e2b2904..1601bb2 100644 --- a/src/main/java/com/powsybl/caseserver/service/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/CaseService.java @@ -29,6 +29,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -59,6 +60,17 @@ default void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime, withIndexation)); } + default List getMetadata(List ids) { + List cases = new ArrayList<>(); + ids.forEach(caseUuid -> { + CaseInfos caseInfos = getCaseInfos(caseUuid); + if (Objects.nonNull(caseInfos)) { + cases.add(caseInfos); + } + }); + return cases; + } + default Importer getImporterOrThrowsException(Path caseFile, ComputationManager computationManager) { DataSource dataSource = DataSource.fromPath(caseFile); Importer importer = Importer.find(dataSource, computationManager); @@ -114,6 +126,14 @@ default Optional exportCase(UUID caseUuid, String format, Strin } } + default List getCasesToReindex(CaseMetadataRepository caseMetadataRepository) { + Set casesToReindex = caseMetadataRepository.findAllByIndexedTrue() + .stream() + .map(CaseMetadataEntity::getId) + .collect(Collectors.toSet()); + return getCases().stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); + } + List getCasesToReindex(); List getCases(); @@ -142,7 +162,5 @@ default Optional exportCase(UUID caseUuid, String format, Strin List searchCases(String query); - List getMetadata(List ids); - void setComputationManager(ComputationManager mock); } diff --git a/src/main/java/com/powsybl/caseserver/service/FsCaseService.java b/src/main/java/com/powsybl/caseserver/service/FsCaseService.java index 9c1b215..444eba2 100644 --- a/src/main/java/com/powsybl/caseserver/service/FsCaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/FsCaseService.java @@ -9,9 +9,6 @@ import com.powsybl.caseserver.CaseException; import com.powsybl.caseserver.dto.CaseInfos; import com.powsybl.caseserver.elasticsearch.CaseInfosService; -import com.powsybl.caseserver.parsers.FileNameInfos; -import com.powsybl.caseserver.parsers.FileNameParser; -import com.powsybl.caseserver.parsers.FileNameParsers; import com.powsybl.caseserver.repository.CaseMetadataEntity; import com.powsybl.caseserver.repository.CaseMetadataRepository; import com.powsybl.computation.ComputationManager; @@ -31,14 +28,11 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.*; -import java.util.stream.Collectors; +import java.nio.file.*; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; import java.util.stream.Stream; import static com.powsybl.caseserver.CaseException.createDirectoryNotFound; @@ -99,6 +93,9 @@ public String getCaseName(UUID caseUuid) { throw createDirectoryNotFound(caseUuid); } CaseInfos caseInfos = getCaseInfos(file); + if (caseInfos == null) { + throw CaseException.getFileNameNotFound(caseUuid); + } return caseInfos.getName(); } @@ -249,25 +246,8 @@ private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); } - // TODO should not be duplicated with S3CaseService public List getCasesToReindex() { - Set casesToReindex = caseMetadataRepository.findAllByIndexedTrue() - .stream() - .map(CaseMetadataEntity::getId) - .collect(Collectors.toSet()); - return getCases(getStorageRootDir()).stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); - } - - @Override - public CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { - FileNameParser parser = FileNameParsers.findParser(fileBaseName); - if (parser != null) { - Optional fileNameInfos = parser.parse(fileBaseName); - if (fileNameInfos.isPresent()) { - return CaseInfos.create(fileBaseName, caseUuid, format, fileNameInfos.get()); - } - } - return CaseInfos.builder().name(fileBaseName).uuid(caseUuid).format(format).build(); + return getCasesToReindex(caseMetadataRepository); } @Transactional @@ -364,7 +344,7 @@ public List getCases() { return walk.filter(Files::isRegularFile) .map(this::getCaseInfos) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -377,26 +357,4 @@ public List searchCases(String query) { return caseInfosService.searchCaseInfos(query); } - public List getCases(Path directory) { - try (Stream walk = Files.walk(directory)) { - return walk.filter(Files::isRegularFile) - .map(file -> createInfos(file.getFileName().toString(), UUID.fromString(file.getParent().getFileName().toString()), getFormat(file))) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public List getMetadata(List ids) { - List cases = new ArrayList<>(); - ids.forEach(caseUuid -> { - Path file = getCaseFile(caseUuid); - if (file != null) { - CaseInfos caseInfos = getCaseInfos(file); - cases.add(caseInfos); - } - }); - return cases; - } - } diff --git a/src/main/java/com/powsybl/caseserver/service/S3CaseService.java b/src/main/java/com/powsybl/caseserver/service/S3CaseService.java index bd6b9be..4c9d431 100644 --- a/src/main/java/com/powsybl/caseserver/service/S3CaseService.java +++ b/src/main/java/com/powsybl/caseserver/service/S3CaseService.java @@ -60,6 +60,7 @@ public class S3CaseService implements CaseService { private static final Logger LOGGER = LoggerFactory.getLogger(S3CaseService.class); + public static final int MAX_SIZE = 500000000; private ComputationManager computationManager = LocalComputationManager.getDefault(); @@ -116,7 +117,7 @@ private R withTempCopy(UUID case } catch (CaseException e) { throw CaseException.initTempFile(caseUuid, e); } catch (Throwable ex) { - throw CaseException.initTempFile(caseUuid); + throw CaseException.initTempFile(caseUuid, ex); } // after this line, need to cleanup the file try { @@ -125,7 +126,7 @@ private R withTempCopy(UUID case } catch (CaseException e) { throw CaseException.createFileNotImportable(tempdirPath); } catch (Throwable t) { - throw CaseException.processTempFile(caseUuid); + throw CaseException.processTempFile(caseUuid, t); } } finally { try { @@ -353,7 +354,7 @@ public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIn importZipContent(mpf.getInputStream(), caseUuid); } } catch (IOException e) { - throw CaseException.createFileNotImportable(caseName); + throw CaseException.createFileNotImportable(caseName, e); } createCaseMetadataEntity(caseUuid, withExpiration, withIndexation, caseMetadataRepository); @@ -367,11 +368,14 @@ public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIn } private void importZipContent(InputStream inputStream, UUID caseUuid) throws IOException { - try (ZipInputStream zipInputStream = new SecuredZipInputStream(inputStream, 1000, 500000000)) { + try (ZipInputStream zipInputStream = new SecuredZipInputStream(inputStream, 1000, MAX_SIZE)) { ZipEntry entry; while ((entry = zipInputStream.getNextEntry()) != null) { if (!entry.isDirectory()) { + if (entry.getSize() > MAX_SIZE) { + throw new IOException("File is too large: " + entry.getName()); + } processEntry(caseUuid, zipInputStream, entry); } zipInputStream.closeEntry(); @@ -425,8 +429,8 @@ public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { if (caseBytes.isPresent()) { try { importZipContent(new ByteArrayInputStream(caseBytes.get()), newCaseUuid); - } catch (IOException ioException) { - throw new UncheckedIOException(ioException); + } catch (Exception e) { + throw CaseException.importZipContent(sourceCaseUuid, e); } } @@ -441,13 +445,8 @@ public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { return newCaseUuid; } - // TODO should not be duplicated with FSCaseService public List getCasesToReindex() { - Set casesToReindex = caseMetadataRepository.findAllByIndexedTrue() - .stream() - .map(CaseMetadataEntity::getId) - .collect(Collectors.toSet()); - return getCases().stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); + return getCasesToReindex(caseMetadataRepository); } @Transactional @@ -528,18 +527,6 @@ public List searchCases(String query) { return caseInfosService.searchCaseInfos(query); } - @Override - public List getMetadata(List ids) { - List cases = new ArrayList<>(); - ids.forEach(caseUuid -> { - CaseInfos caseInfos = getCaseInfos(caseUuid); - if (Objects.nonNull(caseInfos)) { - cases.add(caseInfos); - } - }); - return cases; - } - private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + NOT_FOUND)); } diff --git a/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java similarity index 71% rename from src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java rename to src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java index c652022..0128a68 100644 --- a/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/AbstractSupervisionControllerTest.java @@ -6,26 +6,16 @@ */ package com.powsybl.caseserver; -import com.google.common.jimfs.Configuration; -import com.google.common.jimfs.Jimfs; import com.powsybl.caseserver.repository.CaseMetadataRepository; -import com.powsybl.caseserver.service.FsCaseService; +import com.powsybl.caseserver.service.CaseService; import com.powsybl.caseserver.service.SupervisionService; -import com.powsybl.computation.ComputationManager; import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; import org.springframework.mock.web.MockMultipartFile; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import java.io.IOException; @@ -40,26 +30,22 @@ /** * @author Jamal KHEYYAD */ -@RunWith(SpringRunner.class) -@AutoConfigureMockMvc -@SpringBootTest(properties = {"case-store-directory=/cases", "storage.type=file"}) // TODO test with both impls or mock ? -@ContextConfiguration(classes = {CaseApplication.class}) -public class SupervisionControllerTest { + +public abstract class AbstractSupervisionControllerTest { @Autowired SupervisionService supervisionService; @Autowired CaseMetadataRepository caseMetadataRepository; - @Autowired - FsCaseService caseService; + CaseService caseService; @Autowired - private MockMvc mockMvc; + protected MockMvc mockMvc; - @Value("${case-store-directory}") - private String rootDirectory; + @Value("${case-store-directory:#{systemProperties['user.home'].concat(\"/cases\")}}") + String rootDirectory; private static final String TEST_CASE = "testCase.xiidm"; - private FileSystem fileSystem; + FileSystem fileSystem; @Test public void testGetCaseInfosCount() throws Exception { @@ -116,18 +102,11 @@ private void importCase(Boolean indexed) throws Exception { } private static MockMultipartFile createMockMultipartFile() throws IOException { - try (InputStream inputStream = SupervisionControllerTest.class.getResourceAsStream("/" + SupervisionControllerTest.TEST_CASE)) { - return new MockMultipartFile("file", SupervisionControllerTest.TEST_CASE, MediaType.TEXT_PLAIN_VALUE, inputStream); + try (InputStream inputStream = AbstractSupervisionControllerTest.class.getResourceAsStream("/" + AbstractSupervisionControllerTest.TEST_CASE)) { + return new MockMultipartFile("file", AbstractSupervisionControllerTest.TEST_CASE, MediaType.TEXT_PLAIN_VALUE, inputStream); } } - @Before - public void setUp() { - fileSystem = Jimfs.newFileSystem(Configuration.unix()); - caseService.setFileSystem(fileSystem); - caseService.setComputationManager(Mockito.mock(ComputationManager.class)); - } - @After public void tearDown() throws Exception { fileSystem.close(); diff --git a/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java new file mode 100644 index 0000000..d65467c --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/FsSupervisionControllerTest.java @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.service.FsCaseService; +import com.powsybl.computation.ComputationManager; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * @author Jamal KHEYYAD + */ +@RunWith(SpringRunner.class) +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"}) +@TestPropertySource(properties = {"storage.type=file"}) +public class FsSupervisionControllerTest extends AbstractSupervisionControllerTest { + + @Autowired + private FsCaseService fsCaseService; + + @Before + public void setUp() { + caseService = fsCaseService; + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + ((FsCaseService) caseService).setFileSystem(fileSystem); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + caseMetadataRepository.deleteAll(); + } +} diff --git a/src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java new file mode 100644 index 0000000..ea53cbc --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/S3SupervisionControllerTest.java @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.service.MinioContainerConfig; +import com.powsybl.caseserver.service.S3CaseService; +import com.powsybl.computation.ComputationManager; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * @author Jamal KHEYYAD + */ +@RunWith(SpringRunner.class) +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.MOCK, properties = {"case-store-directory=/cases"}) +@TestPropertySource(properties = {"storage.type=S3"}) +public class S3SupervisionControllerTest extends AbstractSupervisionControllerTest implements MinioContainerConfig { + + @Autowired + private S3CaseService s3CaseService; + + @Before + public void setUp() { + caseService = s3CaseService; + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + caseService.deleteAllCases(); + } +} diff --git a/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java b/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java index ef97945..e5c8ed0 100644 --- a/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/datasource/AbstractCaseDataSourceControllerTest.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.powsybl.caseserver.elasticsearch.CaseInfosRepository; +import com.powsybl.caseserver.elasticsearch.DisableElasticsearch; import com.powsybl.caseserver.repository.CaseMetadataRepository; import com.powsybl.commons.datasource.DataSource; import org.junit.Test; @@ -32,6 +33,7 @@ * @author Abdelsalem Hedhili */ +@DisableElasticsearch public abstract class AbstractCaseDataSourceControllerTest { @MockBean