Skip to content

Commit

Permalink
Add comments + clean
Browse files Browse the repository at this point in the history 
Signed-off-by: Etienne Homer <etiennehomer@gmail.com>
  • Loading branch information
@etiennehomer
etiennehomer committed Oct 31, 2024
1 parent 28cc495 commit 6b73714
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 42 deletions.
4 changes: 4 additions & 0 deletions src/main/java/com/powsybl/caseserver/CaseException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ public static CaseException importZipContent(UUID uuid, Exception e) {
return new CaseException(Type.ZIP_FILE_PROCESS, "Error processing zip content file: " + uuid, e);
}

public static CaseException copyZipContent(UUID uuid, Exception e) {
return new CaseException(Type.ZIP_FILE_PROCESS, "Error copying zip content file: " + uuid, e);
}

public static CaseException createUnsupportedFormat(String format) {
return new CaseException(Type.UNSUPPORTED_FORMAT, "The format: " + format + " is unsupported");
}
Expand Down
30 changes: 10 additions & 20 deletions src/main/java/com/powsybl/caseserver/datasource/S3CaseDataSourceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import java.util.Set;
import java.util.UUID;

import static com.powsybl.caseserver.service.S3CaseService.GZIP_EXTENSION;
import static com.powsybl.caseserver.service.S3CaseService.*;

/**
* @author Ghazwa Rehili <ghazwa.rehili at rte-france.com>
Expand Down Expand Up @@ -53,22 +53,18 @@ public Boolean datasourceExists(UUID caseUuid, String fileName) {

@Override
public byte[] getInputStream(UUID caseUuid, String fileName) {
String caseName = s3CaseService.getCaseName(caseUuid);
String caseFileKey;
if (S3CaseService.isArchivedCaseFile(s3CaseService.getCaseName(caseUuid))) {
caseFileKey = uuidToPrefixKey(caseUuid) + fileName + GZIP_EXTENSION;
} else {
caseFileKey = uuidToPrefixKey(caseUuid) + s3CaseService.getCaseName(caseUuid);
// For archived cases (.zip, .tar, ...), individual files are gzipped in S3 server.
// Here the requested file is decompressed and simply returned.
if (S3CaseService.isArchivedCaseFile(caseName)) {
caseFileKey = uuidToKeyWithFileName(caseUuid, fileName + GZIP_EXTENSION);
return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey,
file -> S3CaseService.decompress(Files.readAllBytes(file)));
}
if (S3CaseService.isArchivedCaseFile(s3CaseService.getCaseName(caseUuid))) {
return s3CaseService.withS3DownloadedTempPath(caseUuid, caseFileKey, file -> S3CaseService.decompress(Files.readAllBytes(file)));
} else {
return withS3DownloadedDataSource(caseUuid, caseFileKey,
caseFileKey = uuidToKeyWithFileName(caseUuid, caseName);
return withS3DownloadedDataSource(caseUuid, caseFileKey,
datasource -> IOUtils.toByteArray(datasource.newInputStream(Paths.get(fileName).getFileName().toString())));
}
}

private String uuidToPrefixKey(UUID uuid) {
return CASES_PREFIX + uuid.toString() + "/";
}

@Override
Expand All @@ -81,12 +77,6 @@ public Set<String> listName(UUID caseUuid, String regex) {
return s3CaseService.listName(caseUuid, regex);
}

public <R, T extends Throwable> R withS3DownloadedDataSource(UUID caseUuid, FailableFunction<DataSource, R, T> f) {
FailableFunction<Path, DataSource, T> pathToDataSource = DataSource::fromPath;
FailableFunction<Path, R, T> composedFunction = pathToDataSource.andThen(f);
return s3CaseService.withS3DownloadedTempPath(caseUuid, composedFunction);
}

public <R, T extends Throwable> R withS3DownloadedDataSource(UUID caseUuid, String caseFileKey, FailableFunction<DataSource, R, T> f) {
FailableFunction<Path, DataSource, T> pathToDataSource = DataSource::fromPath;
FailableFunction<Path, R, T> composedFunction = pathToDataSource.andThen(f);
Expand Down
58 changes: 36 additions & 22 deletions src/main/java/com/powsybl/caseserver/service/S3CaseService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public <R, T extends Throwable> R withS3DownloadedTempPath(UUID caseUuid, Failab
}

public <R, T extends Throwable> R withS3DownloadedTempPath(UUID caseUuid, String caseFileKey, FailableFunction<Path, R, T> f) {
String nonNullCaseFileKey = Objects.requireNonNullElse(caseFileKey, getCaseFileObjectKey(caseUuid));
String nonNullCaseFileKey = Objects.requireNonNullElse(caseFileKey, uuidToKeyWithOriginameFileName(caseUuid));
String filename = parseFilenameFromKey(nonNullCaseFileKey);
return withTempCopy(caseUuid, filename, path ->
s3Client.getObject(GetObjectRequest.builder().bucket(bucketName).key(nonNullCaseFileKey).build(), path), f);
Expand Down Expand Up @@ -189,12 +189,16 @@ private String parseFilenameFromKey(String key) {
return key.substring(secondSlash + 1);
}

private String uuidToPrefixKey(UUID uuid) {
public static String uuidToKeyPrefix(UUID uuid) {
return CASES_PREFIX + uuid.toString() + DELIMITER;
}

private String uuidAndFilenameToKey(UUID uuid, String filename) {
return uuidToPrefixKey(uuid) + filename;
public static String uuidToKeyWithFileName(UUID uuid, String filename) {
return uuidToKeyPrefix(uuid) + filename;
}

public String uuidToKeyWithOriginameFileName(UUID caseUuid) {
return uuidToKeyWithFileName(caseUuid, getOriginalFilename(caseUuid));
}

private List<S3Object> getCasesSummaries(String prefix) {
Expand All @@ -211,17 +215,13 @@ private ListObjectsV2Request getListObjectsV2Request(String prefix) {
}

private List<S3Object> getCaseFileSummaries(UUID caseUuid) {
List<S3Object> files = getCasesSummaries(uuidToPrefixKey(caseUuid));
List<S3Object> files = getCasesSummaries(uuidToKeyPrefix(caseUuid));
if (files.size() > 1) {
LOGGER.warn("Multiple files for case {}", caseUuid);
}
return files;
}

public String getCaseFileObjectKey(UUID caseUuid) {
return CASES_PREFIX + caseUuid + "/" + getOriginalFilename(caseUuid);
}

private List<CaseInfos> infosFromDownloadCaseFileSummaries(List<S3Object> objectSummaries) {
List<CaseInfos> caseInfosList = new ArrayList<>();
for (S3Object objectSummary : objectSummaries) {
Expand Down Expand Up @@ -251,7 +251,7 @@ public String getCaseName(UUID caseUuid) {
public Optional<byte[]> getCaseBytes(UUID caseUuid) {
String caseFileKey = null;
try {
caseFileKey = getCaseFileObjectKey(caseUuid);
caseFileKey = uuidToKeyWithOriginameFileName(caseUuid);
GetObjectRequest getObjectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(caseFileKey)
Expand All @@ -276,15 +276,15 @@ public List<CaseInfos> getCases() {

@Override
public boolean caseExists(UUID uuid) {
return !getCasesSummaries(uuidToPrefixKey(uuid)).isEmpty();
return !getCasesSummaries(uuidToKeyPrefix(uuid)).isEmpty();
}

public Boolean datasourceExists(UUID caseUuid, String fileName) {
if (getCaseFileSummaries(caseUuid).size() > 1 && fileName.equals(getCaseName(caseUuid))) {
return Boolean.FALSE;
}

String key = uuidToPrefixKey(caseUuid) + fileName;
String key = uuidToKeyWithFileName(caseUuid, fileName);
String caseName = getCaseName(caseUuid);
// For compressed cases, we append the compression extension to the case name as only the compressed file is stored in S3.
// i.e. : Assuming test.xml.gz is stored in S3. When you request datasourceExists(randomUUID, "test.xml"), you ask to S3 API ("test.xml" + ".gz") exists ? => true
Expand Down Expand Up @@ -317,12 +317,16 @@ public static boolean isArchivedCaseFile(String caseName) {
public Set<String> listName(UUID caseUuid, String regex) {
List<String> fileNames;
if (isCompressedCaseFile(getOriginalFilename(caseUuid))) {
// For a compressed file basename.xml.gz, listName() should return ['basename.xml']. That's why we remove the compression extension to the filename.
fileNames = List.of(getOriginalFilename(caseUuid).replace("." + getCompressionFormat(caseUuid), ""));
} else {
List<S3Object> s3Objects = getCaseFileSummaries(caseUuid);
fileNames = s3Objects.stream().map(obj -> Paths.get(obj.key()).toString().replace(CASES_PREFIX + caseUuid.toString() + DELIMITER, "")).toList();
// For archived cases :
if (isArchivedCaseFile(getOriginalFilename(caseUuid))) {
// the original archive name has to be filtered.
fileNames = fileNames.stream().filter(name -> !name.equals(getOriginalFilename(caseUuid))).toList();
// each subfile hase been gzipped -> we have to remove the gz extension.
fileNames = fileNames.stream().map(name -> name.replace(GZIP_EXTENSION, "")).toList();
}
}
Expand All @@ -340,8 +344,19 @@ public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIn
String compressionFormat = FileNameUtils.getExtension(Paths.get(caseName));

try (InputStream inputStream = mpf.getInputStream()) {
String key = uuidAndFilenameToKey(caseUuid, caseName);

String key = uuidToKeyWithFileName(caseUuid, caseName);

// We store archived cases in S3 in a specific way : in the caseUuid directory, we store :
// - the original archive
// - the extracted files are exploded in the caseUuid directory. This allows to use HeadObjectRequest for datasource/exists,
// to download subfiles separately, or to anwser to datasource/list with ListObjectV2.
// But this unarchived storage could increase tenfold used disk space: so each extracted file is gzipped to avoid increasing it.
// Compression of subfiles is done in a simple way: no matter if the subfile is compressed or not, it will be gzipped in the storage.
// example : archive.zip containing [file1.xml, file2.xml.gz]
// will be stored as :
// - archive.zip
// - file1.xml.gz
// - file2.xml.gz.gz
if (isArchivedCaseFile(caseName)) {
importZipContent(mpf.getInputStream(), caseUuid);
}
Expand Down Expand Up @@ -371,7 +386,6 @@ public UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIn
private void importZipContent(InputStream inputStream, UUID caseUuid) throws IOException {
try (ZipInputStream zipInputStream = new SecuredZipInputStream(inputStream, 1000, MAX_SIZE)) {
ZipEntry entry;

while ((entry = zipInputStream.getNextEntry()) != null) {
if (!entry.isDirectory()) {
processEntry(caseUuid, zipInputStream, entry);
Expand All @@ -394,6 +408,7 @@ private void copyZipContent(InputStream inputStream, UUID sourcecaseUuid, UUID c
}

private void copyEntry(UUID sourcecaseUuid, UUID caseUuid, String fileName) {
// To optimize copy, files to copy are not downloaded on the case-server. They are directly copied on the S3 server.
CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder()
.sourceBucket(bucketName)
.sourceKey(CASES_PREFIX + sourcecaseUuid + "/" + fileName)
Expand All @@ -409,7 +424,7 @@ private void copyEntry(UUID sourcecaseUuid, UUID caseUuid, String fileName) {

private void processEntry(UUID caseUuid, ZipInputStream zipInputStream, ZipEntry entry) throws IOException {
String fileName = entry.getName();
String extractedKey = uuidAndFilenameToKey(caseUuid, fileName);
String extractedKey = uuidToKeyWithFileName(caseUuid, fileName);
byte[] fileBytes = compress(ByteStreams.toByteArray(zipInputStream));

PutObjectRequest extractedFileRequest = PutObjectRequest.builder()
Expand Down Expand Up @@ -439,9 +454,10 @@ public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Source case " + sourceCaseUuid + NOT_FOUND);
}

String sourceKey = getCaseFileObjectKey(sourceCaseUuid);
String sourceKey = uuidToKeyWithOriginameFileName(sourceCaseUuid);
UUID newCaseUuid = UUID.randomUUID();
String targetKey = uuidAndFilenameToKey(newCaseUuid, parseFilenameFromKey(sourceKey));
String targetKey = uuidToKeyWithFileName(newCaseUuid, parseFilenameFromKey(sourceKey));
// To optimize copy, cases to copy are not downloaded on the case-server. They are directly copied on the S3 server.
CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder()
.sourceBucket(bucketName)
.sourceKey(sourceKey)
Expand All @@ -460,7 +476,7 @@ public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) {
try {
copyZipContent(new ByteArrayInputStream(caseBytes.get()), sourceCaseUuid, newCaseUuid);
} catch (Exception e) {
throw CaseException.importZipContent(sourceCaseUuid, e);
throw CaseException.copyZipContent(sourceCaseUuid, e);
}
}
CaseInfos existingCaseInfos = getCaseInfos(sourceCaseUuid);
Expand All @@ -474,11 +490,9 @@ public UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) {

@Override
public Optional<Network> loadNetwork(UUID caseUuid) {

if (!caseExists(caseUuid)) {
return Optional.empty();
}

return Optional.of(withS3DownloadedTempPath(caseUuid, path -> {
Network network = Network.read(path);
if (network == null) {
Expand All @@ -490,7 +504,7 @@ public Optional<Network> loadNetwork(UUID caseUuid) {

@Override
public void deleteCase(UUID caseUuid) {
String prefixKey = uuidToPrefixKey(caseUuid);
String prefixKey = uuidToKeyPrefix(caseUuid);
List<ObjectIdentifier> objectsToDelete = s3Client.listObjectsV2(builder -> builder.bucket(bucketName).prefix(prefixKey))
.contents()
.stream()
Expand Down

0 comments on commit 6b73714

Please sign in to comment.