From f485d8b0b83980efe8bfadb81034471709c7ca6d Mon Sep 17 00:00:00 2001 From: Anis Touri Date: Thu, 2 May 2024 17:21:20 +0200 Subject: [PATCH 01/11] Refactor duplication endpoint (#28) Signed-off-by: TOURI ANIS --- src/main/java/com/powsybl/caseserver/CaseController.java | 8 ++++---- .../java/com/powsybl/caseserver/CaseControllerTest.java | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/powsybl/caseserver/CaseController.java b/src/main/java/com/powsybl/caseserver/CaseController.java index ad410b9..29dee89 100644 --- a/src/main/java/com/powsybl/caseserver/CaseController.java +++ b/src/main/java/com/powsybl/caseserver/CaseController.java @@ -157,16 +157,16 @@ public ResponseEntity importCase(@RequestParam("file") MultipartFile file, return ResponseEntity.ok().body(caseUuid); } - @PostMapping(value = "/cases") + @PostMapping(value = "/cases", params = "duplicateFrom") @Operation(summary = "create a case from an existing one") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "The case has been duplicated"), @ApiResponse(responseCode = "404", description = "Source case not found"), @ApiResponse(responseCode = "500", description = "An error occurred during the case file duplication")}) public ResponseEntity duplicateCase( - @RequestParam("duplicateFrom") UUID sourceCaseUuid, + @RequestParam("duplicateFrom") UUID caseId, @RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration) { - LOGGER.debug("duplicateCase request received with parameter sourceCaseUuid = {}", sourceCaseUuid); - UUID newCaseUuid = caseService.duplicateCase(sourceCaseUuid, withExpiration); + LOGGER.debug("duplicateCase request received with parameter sourceCaseUuid = {}", caseId); + UUID newCaseUuid = caseService.duplicateCase(caseId, withExpiration); return ResponseEntity.ok().body(newCaseUuid); } diff --git a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java index b5f703e..8a45be2 100644 --- a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java @@ -312,7 +312,7 @@ public void test() throws Exception { assertNull(caseMetadataEntity.getExpirationDate()); //duplicate an existing case - MvcResult duplicateResult = mvc.perform(post("/v1/cases").param("duplicateFrom", caseUuid.toString())) + MvcResult duplicateResult = mvc.perform(post("/v1/cases?duplicateFrom=" + caseUuid)) .andExpect(status().isOk()) .andReturn(); @@ -353,8 +353,7 @@ public void test() throws Exception { assertNotNull(caseMetadataEntity.getExpirationDate()); //duplicate an existing case withExpiration - MvcResult duplicateResult2 = mvc.perform(post("/v1/cases") - .param("duplicateFrom", caseUuid.toString()) + MvcResult duplicateResult2 = mvc.perform(post("/v1/cases?duplicateFrom=" + caseUuid) .param("withExpiration", "true")) .andExpect(status().isOk()) .andReturn(); @@ -399,7 +398,7 @@ public void test() throws Exception { assertTrue(deleteExpirationResult.getResponse().getContentAsString().contains("case " + randomUuid + " not found")); // assert that duplicating a non existing case should return a 404 - mvc.perform(post("/v1/cases").param("duplicateFrom", UUID.randomUUID().toString())) + mvc.perform(post("/v1/cases?duplicateFrom=" + UUID.randomUUID())) .andExpect(status().isNotFound()) .andReturn(); From 4b72916c71579e835da5b58789efda0ba70429bb Mon Sep 17 00:00:00 2001 From: Anis Touri Date: Fri, 3 May 2024 16:25:34 +0200 Subject: [PATCH 02/11] Parent 19: update liquibase maven plugin (#32) Signed-off-by: TOURI ANIS --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e6553a3..94e7877 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ com.powsybl powsybl-parent-ws - 17 + 19 From 3713262a6ea0912e5523a0f619feeebdf3338a32 Mon Sep 17 00:00:00 2001 From: etiennehomer Date: Fri, 17 May 2024 17:06:29 +0200 Subject: [PATCH 03/11] Update to powsybl dependencies 2024.1.0 (#31) Signed-off-by: Etienne Homer --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 94e7877..28a51b6 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 3.11.1 0.0.2 - 2.9.0 + 2.10.0 1.18.3 From ce1679bff5b68a57dbd22bc53b7a3e213282473b Mon Sep 17 00:00:00 2001 From: Antoine Bouhours Date: Wed, 29 May 2024 16:35:59 +0200 Subject: [PATCH 04/11] Enable Prometheus in every microservice (#33) Signed-off-by: BOUHOURS Antoine --- pom.xml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 28a51b6..041d796 100644 --- a/pom.xml +++ b/pom.xml @@ -143,10 +143,6 @@ org.springframework.boot spring-boot-autoconfigure - - org.springframework.boot - spring-boot-starter-actuator - org.springframework.boot @@ -224,6 +220,16 @@ log4j-over-slf4j runtime + + org.springframework.boot + spring-boot-starter-actuator + runtime + + + io.micrometer + micrometer-registry-prometheus + runtime + com.google.guava From 312155e292671c0df114733469ab453bfeb1af15 Mon Sep 17 00:00:00 2001 From: Achour berrahma Date: Fri, 14 Jun 2024 11:27:42 +0200 Subject: [PATCH 05/11] Fix case names when exported (#34) --- src/main/java/com/powsybl/caseserver/CaseService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/CaseService.java index d051201..c237966 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/CaseService.java @@ -15,13 +15,13 @@ import com.powsybl.caseserver.repository.CaseMetadataEntity; import com.powsybl.caseserver.repository.CaseMetadataRepository; import com.powsybl.commons.datasource.DataSource; +import com.powsybl.commons.datasource.DataSourceUtil; import com.powsybl.commons.datasource.MemDataSource; import com.powsybl.computation.ComputationManager; import com.powsybl.computation.local.LocalComputationManager; import com.powsybl.iidm.network.Exporter; import com.powsybl.iidm.network.Importer; import com.powsybl.iidm.network.Network; -import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -426,7 +426,7 @@ public Optional exportCase(UUID caseUuid, String format, Map Date: Fri, 14 Jun 2024 11:37:35 +0200 Subject: [PATCH 06/11] update to powsybl-ws-dependencies v2.11.0 (#35) Signed-off-by: Abdelsalem --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 041d796..b58b170 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 3.11.1 0.0.2 - 2.10.0 + 2.11.0 1.18.3 From d7df58da9bd593928d4e398541b8690853412801 Mon Sep 17 00:00:00 2001 From: Anis Touri Date: Thu, 20 Jun 2024 17:53:45 +0200 Subject: [PATCH 07/11] replace OffsetDateTime with Instant (#36) Signed-off-by: TOURI ANIS --- .../java/com/powsybl/caseserver/CaseService.java | 8 ++++---- .../com/powsybl/caseserver/ScheduledCaseCleaner.java | 9 ++++----- .../caseserver/repository/CaseMetadataEntity.java | 6 +++--- .../changesets/changelog_20240620T100317Z.xml | 6 ++++++ .../resources/db/changelog/db.changelog-master.yaml | 4 ++++ .../com/powsybl/caseserver/CaseControllerTest.java | 8 ++++---- .../powsybl/caseserver/ScheduledCaseCleanerTest.java | 12 ++++++------ src/test/resources/application-default.yaml | 10 ++++++---- 8 files changed, 37 insertions(+), 26 deletions(-) create mode 100644 src/main/resources/db/changelog/changesets/changelog_20240620T100317Z.xml diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/CaseService.java index c237966..e4207ad 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/CaseService.java @@ -44,8 +44,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -249,9 +249,9 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { } private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration) { - LocalDateTime expirationTime = null; + Instant expirationTime = null; if (withExpiration) { - expirationTime = LocalDateTime.now(ZoneOffset.UTC).plusHours(1); + expirationTime = Instant.now().plus(1, ChronoUnit.HOURS); } caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime)); } diff --git a/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java b/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java index 0bd5389..011bdd3 100644 --- a/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java +++ b/src/main/java/com/powsybl/caseserver/ScheduledCaseCleaner.java @@ -12,8 +12,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.time.Instant; /** * @author Abdelsalem Hedhili @@ -34,10 +33,10 @@ public ScheduledCaseCleaner(CaseMetadataRepository caseMetadataRepository, CaseS @Scheduled(cron = "${cleaning-cases-cron}", zone = "UTC") public void deleteExpiredCases() { - LocalDateTime localDateTime = LocalDateTime.now(ZoneOffset.UTC); - LOGGER.info("Cleaning cases cron starting execution at {}", localDateTime); + Instant now = Instant.now(); + LOGGER.info("Cleaning cases cron starting execution at {}", now); caseMetadataRepository.findAll().stream().filter(caseMetadataEntity -> caseMetadataEntity.getExpirationDate() != null) - .filter(caseMetadataEntity -> localDateTime.isAfter(caseMetadataEntity.getExpirationDate())) + .filter(caseMetadataEntity -> now.isAfter(caseMetadataEntity.getExpirationDate())) .forEach(caseMetadataEntity -> { caseService.deleteCase(caseMetadataEntity.getId()); caseMetadataRepository.deleteById(caseMetadataEntity.getId()); diff --git a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java index a3efa34..ebacd0e 100644 --- a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java +++ b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java @@ -6,7 +6,7 @@ */ package com.powsybl.caseserver.repository; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.UUID; import lombok.*; @@ -31,6 +31,6 @@ public class CaseMetadataEntity { @Column(name = "id") private UUID id; - @Column(name = "expirationDate") - private LocalDateTime expirationDate; + @Column(name = "expirationDate", columnDefinition = "timestamptz") + private Instant expirationDate; } diff --git a/src/main/resources/db/changelog/changesets/changelog_20240620T100317Z.xml b/src/main/resources/db/changelog/changesets/changelog_20240620T100317Z.xml new file mode 100644 index 0000000..26f6fc1 --- /dev/null +++ b/src/main/resources/db/changelog/changesets/changelog_20240620T100317Z.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/main/resources/db/changelog/db.changelog-master.yaml b/src/main/resources/db/changelog/db.changelog-master.yaml index 8258ab0..7768178 100644 --- a/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/src/main/resources/db/changelog/db.changelog-master.yaml @@ -1,4 +1,8 @@ databaseChangeLog: - include: file: changesets/changelog_20221206T100716Z.xml + relativeToChangelogFile: true + + - include: + file: changesets/changelog_20240620T100317Z.xml relativeToChangelogFile: true \ No newline at end of file diff --git a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java index 8a45be2..f53be5e 100644 --- a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java @@ -41,9 +41,9 @@ import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.time.Instant; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.UUID; @@ -332,9 +332,9 @@ public void test() throws Exception { assertNull(caseMetadataEntity.getExpirationDate()); // import a case with expiration - LocalDateTime beforeImportDate = LocalDateTime.now(ZoneOffset.UTC).plusHours(1); + Instant beforeImportDate = Instant.now().plus(1, ChronoUnit.HOURS); UUID thirdCaseUuid = importCase(TEST_CASE, true); - LocalDateTime afterImportDate = LocalDateTime.now(ZoneOffset.UTC).plusHours(1); + Instant afterImportDate = Instant.now().plus(1, ChronoUnit.HOURS); // assert that the broker message has been sent messageImport = outputDestination.receive(1000, "case.import.destination"); diff --git a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java index a634e95..6408a1e 100644 --- a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java +++ b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java @@ -18,8 +18,8 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.junit4.SpringRunner; -import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.UUID; import static org.junit.Assert.assertEquals; @@ -58,10 +58,10 @@ private void cleanDB() { @Test public void test() { - LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); - LocalDateTime yesterday = now.minusDays(1); - CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plusHours(1)); - CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plusHours(1)); + Instant now = Instant.now(); + Instant yesterday = now.minus(1, ChronoUnit.DAYS); + CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS)); + CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS)); CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null); caseMetadataRepository.save(shouldExpireEntity); caseMetadataRepository.save(shouldNotExpireEntity); diff --git a/src/test/resources/application-default.yaml b/src/test/resources/application-default.yaml index e045393..9dba455 100644 --- a/src/test/resources/application-default.yaml +++ b/src/test/resources/application-default.yaml @@ -4,10 +4,12 @@ spring: properties: dialect: org.hibernate.dialect.H2Dialect hibernate.format_sql: true - hibernate: - format_sql: true - generate_statistics: true - dialect: org.hibernate.dialect.H2Dialect + hibernate: + format_sql: true + generate_statistics: true + dialect: org.hibernate.dialect.H2Dialect + #to turn off schema validation that fails (because of timestampz types) and blocks tests even if the schema is compatible + ddl-auto: none logging: level: From 1c3212e9269834c0b60bbcceda3bea8be7eb45f3 Mon Sep 17 00:00:00 2001 From: souissimai <133104748+souissimai@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:18:44 +0200 Subject: [PATCH 08/11] Rename file when converting case (#37) Signed-off-by: maissa SOUISSI --- .../java/com/powsybl/caseserver/CaseController.java | 3 ++- src/main/java/com/powsybl/caseserver/CaseService.java | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/powsybl/caseserver/CaseController.java b/src/main/java/com/powsybl/caseserver/CaseController.java index 29dee89..17776ad 100644 --- a/src/main/java/com/powsybl/caseserver/CaseController.java +++ b/src/main/java/com/powsybl/caseserver/CaseController.java @@ -123,9 +123,10 @@ public ResponseEntity downloadCase(@PathVariable("caseUuid") UUID caseUu public ResponseEntity exportCase( @PathVariable UUID caseUuid, @RequestParam String format, + @RequestParam(value = "fileName", required = false) String fileName, @RequestBody(required = false) Map formatParameters) throws IOException { LOGGER.debug("exportCase request received with parameter caseUuid = {}", caseUuid); - return caseService.exportCase(caseUuid, format, formatParameters).map(networkInfos -> { + return caseService.exportCase(caseUuid, format, fileName, formatParameters).map(networkInfos -> { var headers = new HttpHeaders(); headers.setContentDisposition( ContentDisposition.builder("attachment") diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/CaseService.java index e4207ad..27b9fdd 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/CaseService.java @@ -408,7 +408,7 @@ Optional getCaseBytes(UUID caseUuid) { return Optional.empty(); } - public Optional exportCase(UUID caseUuid, String format, Map formatParameters) throws IOException { + public Optional exportCase(UUID caseUuid, String format, String fileName, Map formatParameters) throws IOException { if (!Exporter.getFormats().contains(format)) { throw CaseException.createUnsupportedFormat(format); } @@ -426,17 +426,17 @@ public Optional exportCase(UUID caseUuid, String format, Map Date: Mon, 29 Jul 2024 11:59:41 +0200 Subject: [PATCH 09/11] upgrade to powsybl-dependencies 2024.2.0 (#38) Signed-off-by: Franck LECUYER --- pom.xml | 2 +- src/test/resources/testCase.xiidm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b58b170..17af863 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 3.11.1 0.0.2 - 2.11.0 + 2.12.0 1.18.3 diff --git a/src/test/resources/testCase.xiidm b/src/test/resources/testCase.xiidm index fa355fa..a97c9ca 100644 --- a/src/test/resources/testCase.xiidm +++ b/src/test/resources/testCase.xiidm @@ -1,2 +1,2 @@ - + From e9614cf78e870084403057ea1a8af1d34991d0ee Mon Sep 17 00:00:00 2001 From: jamal-khey Date: Wed, 21 Aug 2024 14:51:04 +0200 Subject: [PATCH 10/11] add indexed attribute index only elements marked for indexation (#39) Signed-off-by: jamal-khey --- .../powsybl/caseserver/CaseController.java | 13 +- .../com/powsybl/caseserver/CaseService.java | 42 ++--- .../caseserver/SupervisionController.java | 82 +++++++++ .../elasticsearch/CaseInfosService.java | 6 + .../caseserver/elasticsearch/ESConfig.java | 2 + .../repository/CaseMetadataEntity.java | 3 + .../repository/CaseMetadataRepository.java | 3 +- .../services/SupervisionService.java | 43 +++++ .../changesets/changelog_20240726T144717Z.xml | 10 ++ .../db/changelog/db.changelog-master.yaml | 7 +- .../caseserver/CaseControllerTest.java | 155 +++++++++++++----- .../caseserver/ScheduledCaseCleanerTest.java | 6 +- .../caseserver/SupervisionControllerTest.java | 146 +++++++++++++++++ .../powsybl/caseserver/utils/TestUtils.java | 36 ++++ 14 files changed, 475 insertions(+), 79 deletions(-) create mode 100644 src/main/java/com/powsybl/caseserver/SupervisionController.java create mode 100644 src/main/java/com/powsybl/caseserver/services/SupervisionService.java create mode 100644 src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml create mode 100644 src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/utils/TestUtils.java diff --git a/src/main/java/com/powsybl/caseserver/CaseController.java b/src/main/java/com/powsybl/caseserver/CaseController.java index 17776ad..0f480f9 100644 --- a/src/main/java/com/powsybl/caseserver/CaseController.java +++ b/src/main/java/com/powsybl/caseserver/CaseController.java @@ -152,9 +152,10 @@ public ResponseEntity exists(@PathVariable("caseUuid") UUID caseUuid) { @Operation(summary = "import a case") @SuppressWarnings("javasecurity:S5145") public ResponseEntity importCase(@RequestParam("file") MultipartFile file, - @RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration) { + @RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration, + @RequestParam(value = "withIndexation", required = false, defaultValue = "false") boolean withIndexation) { LOGGER.debug("importCase request received with file = {}", file.getName()); - UUID caseUuid = caseService.importCase(file, withExpiration); + UUID caseUuid = caseService.importCase(file, withExpiration, withIndexation); return ResponseEntity.ok().body(caseUuid); } @@ -206,14 +207,6 @@ public ResponseEntity> searchCases(@RequestParam(value = "q") St return ResponseEntity.ok().body(cases); } - @PostMapping(value = "/cases/reindex-all") - @Operation(summary = "reindex all cases") - public ResponseEntity reindexAllCases() { - LOGGER.debug("reindex all cases request received"); - caseService.reindexAllCases(); - return ResponseEntity.ok().build(); - } - @GetMapping(value = "/cases/metadata") @Operation(summary = "Get cases Metadata") public ResponseEntity> getMetadata(@RequestParam("ids") List ids) { diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/CaseService.java index 27b9fdd..29c8892 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/CaseService.java @@ -46,14 +46,7 @@ import java.nio.file.StandardCopyOption; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; @@ -179,7 +172,7 @@ boolean caseExists(UUID caseName) { return Files.exists(caseFile) && Files.isRegularFile(caseFile); } - UUID importCase(MultipartFile mpf, boolean withExpiration) { + UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation) { checkStorageInitialization(); UUID caseUuid = UUID.randomUUID(); @@ -214,9 +207,11 @@ UUID importCase(MultipartFile mpf, boolean withExpiration) { throw e; } - createCaseMetadataEntity(caseUuid, withExpiration); + createCaseMetadataEntity(caseUuid, withExpiration, withIndexation); CaseInfos caseInfos = createInfos(caseFile.getFileName().toString(), caseUuid, importer.getFormat()); - caseInfosService.addCaseInfos(caseInfos); + if (withIndexation) { + caseInfosService.addCaseInfos(caseInfos); + } sendImportMessage(caseInfos.createMessage()); return caseUuid; } @@ -238,8 +233,9 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { CaseInfos existingCaseInfos = caseInfosService.getCaseInfosByUuid(sourceCaseUuid.toString()).orElseThrow(); CaseInfos caseInfos = createInfos(existingCaseInfos.getName(), newCaseUuid, existingCaseInfos.getFormat()); caseInfosService.addCaseInfos(caseInfos); - createCaseMetadataEntity(newCaseUuid, withExpiration); + CaseMetadataEntity existingCase = getCaseMetaDataEntity(sourceCaseUuid); + createCaseMetadataEntity(newCaseUuid, withExpiration, existingCase.isIndexed()); sendImportMessage(caseInfos.createMessage()); return newCaseUuid; @@ -248,12 +244,24 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { } } - private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration) { + private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { + return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); + } + + private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, boolean withIndexation) { Instant expirationTime = null; if (withExpiration) { expirationTime = Instant.now().plus(1, ChronoUnit.HOURS); } - caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime)); + caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime, withIndexation)); + } + + public List getCasesToReindex() { + Set casesToReindex = caseMetadataRepository.findAllByIndexedTrue() + .stream() + .map(CaseMetadataEntity::getId) + .collect(Collectors.toSet()); + return getCases(getStorageRootDir()).stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); } CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { @@ -269,7 +277,7 @@ CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { @Transactional public void disableCaseExpiration(UUID caseUuid) { - CaseMetadataEntity caseMetadataEntity = caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); + CaseMetadataEntity caseMetadataEntity = getCaseMetaDataEntity(caseUuid); caseMetadataEntity.setExpirationDate(null); } @@ -373,10 +381,6 @@ private void sendImportMessage(Message message) { caseInfosPublisher.send("publishCaseImport-out-0", message); } - public void reindexAllCases() { - caseInfosService.recreateAllCaseInfos(getCases(getStorageRootDir())); - } - public List getMetadata(List ids) { List cases = new ArrayList<>(); ids.forEach(caseUuid -> { diff --git a/src/main/java/com/powsybl/caseserver/SupervisionController.java b/src/main/java/com/powsybl/caseserver/SupervisionController.java new file mode 100644 index 0000000..cf826a9 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/SupervisionController.java @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.powsybl.caseserver.elasticsearch.CaseInfosService; +import com.powsybl.caseserver.services.SupervisionService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +/** + * @author Jamal KHEYYAD + */ +@RestController +@RequestMapping(value = "/" + CaseConstants.API_VERSION + "/supervision") +@Tag(name = "case-server - Supervision") +public class SupervisionController { + private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionController.class); + + private final SupervisionService supervisionService; + private final CaseService caseService; + private final ClientConfiguration elasticsearchClientConfiguration; + private final CaseInfosService caseInfosService; + + public SupervisionController(SupervisionService supervisionService, CaseService caseService, ClientConfiguration elasticsearchClientConfiguration, CaseInfosService caseInfosService) { + this.supervisionService = supervisionService; + this.caseService = caseService; + this.elasticsearchClientConfiguration = elasticsearchClientConfiguration; + this.caseInfosService = caseInfosService; + } + + @GetMapping(value = "/elasticsearch-host") + @Operation(summary = "get the elasticsearch address") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "the elasticsearch address")}) + public ResponseEntity getElasticsearchHost() { + String host = elasticsearchClientConfiguration.getEndpoints().get(0).getHostName() + + ":" + + elasticsearchClientConfiguration.getEndpoints().get(0).getPort(); + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(host); + } + + @GetMapping(value = "/cases/index-name") + @Operation(summary = "get the indexed cases index name") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Indexed directory cases index name")}) + public ResponseEntity getIndexedCasesFIndexName() { + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(caseInfosService.getDirectoryCasesIndexName()); + } + + @PostMapping(value = "/cases/reindex") + @Operation(summary = "reindex all cases") + public ResponseEntity reindexAllCases() { + LOGGER.debug("reindex all cases request received"); + caseInfosService.recreateAllCaseInfos(caseService.getCasesToReindex()); + return ResponseEntity.ok().build(); + } + + @DeleteMapping(value = "/cases/indexation") + @Operation(summary = "delete indexed cases") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all indexed cases have been deleted")}) + public ResponseEntity deleteIndexedCases() { + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(Long.toString(supervisionService.deleteIndexedCases())); + } + + @GetMapping(value = "/cases/indexation-count") + @Operation(summary = "get indexed cases count") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Indexed cases count")}) + public ResponseEntity getIndexedCasesCount() { + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(Long.toString(supervisionService.getIndexedCasesCount())); + } + +} diff --git a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java index 7c539c1..3286410 100644 --- a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java +++ b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java @@ -9,7 +9,9 @@ import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery; import com.google.common.collect.Lists; import com.powsybl.caseserver.dto.CaseInfos; +import lombok.Getter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.ComponentScan; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -42,6 +44,10 @@ public class CaseInfosService { @Autowired private ElasticsearchOperations operations; + @Value(ESConfig.CASE_INFOS_INDEX_NAME) + @Getter + private String directoryCasesIndexName; + public CaseInfos addCaseInfos(@NonNull final CaseInfos ci) { caseInfosRepository.save(ci); return ci; diff --git a/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java b/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java index bc35d05..1972b2e 100644 --- a/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java +++ b/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java @@ -32,6 +32,8 @@ @EnableElasticsearchRepositories public class ESConfig extends ElasticsearchConfiguration { + public static final String CASE_INFOS_INDEX_NAME = "#{@environment.getProperty('powsybl-ws.elasticsearch.index.prefix')}cases"; + @Value("#{'${spring.data.elasticsearch.embedded:false}' ? 'localhost' : '${spring.data.elasticsearch.host}'}") private String esHost; diff --git a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java index ebacd0e..2bca67f 100644 --- a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java +++ b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java @@ -33,4 +33,7 @@ public class CaseMetadataEntity { @Column(name = "expirationDate", columnDefinition = "timestamptz") private Instant expirationDate; + + @Column(name = "indexed", columnDefinition = "boolean default false", nullable = false) + private boolean indexed = false; } diff --git a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java index 79c9cca..c44b383 100644 --- a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java +++ b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java @@ -9,6 +9,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.util.List; import java.util.UUID; /** @@ -16,5 +17,5 @@ */ @Repository public interface CaseMetadataRepository extends JpaRepository { - + List findAllByIndexedTrue(); } diff --git a/src/main/java/com/powsybl/caseserver/services/SupervisionService.java b/src/main/java/com/powsybl/caseserver/services/SupervisionService.java new file mode 100644 index 0000000..5a11ab6 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/services/SupervisionService.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.services; + +import com.powsybl.caseserver.elasticsearch.CaseInfosRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author Jamal KHEYYAD + */ +@Service +public class SupervisionService { + private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionService.class); + + private final CaseInfosRepository caseInfosRepository; + + public SupervisionService(CaseInfosRepository caseInfosRepository) { + this.caseInfosRepository = caseInfosRepository; + } + + public long deleteIndexedCases() { + AtomicReference startTime = new AtomicReference<>(); + startTime.set(System.nanoTime()); + + long nbIndexesToDelete = getIndexedCasesCount(); + caseInfosRepository.deleteAll(); + LOGGER.trace("Indexed cases deletion : {} seconds", TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime.get())); + return nbIndexesToDelete; + } + + public long getIndexedCasesCount() { + return caseInfosRepository.count(); + } +} diff --git a/src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml b/src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml new file mode 100644 index 0000000..3681251 --- /dev/null +++ b/src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/main/resources/db/changelog/db.changelog-master.yaml b/src/main/resources/db/changelog/db.changelog-master.yaml index 7768178..5ad8263 100644 --- a/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/src/main/resources/db/changelog/db.changelog-master.yaml @@ -5,4 +5,9 @@ databaseChangeLog: - include: file: changesets/changelog_20240620T100317Z.xml - relativeToChangelogFile: true \ No newline at end of file + relativeToChangelogFile: true + + - include: + file: changesets/changelog_20240726T144717Z.xml + relativeToChangelogFile: true + diff --git a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java index f53be5e..e92b746 100644 --- a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java @@ -15,6 +15,7 @@ import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser; import com.powsybl.caseserver.repository.CaseMetadataEntity; import com.powsybl.caseserver.repository.CaseMetadataRepository; +import com.powsybl.caseserver.utils.TestUtils; import com.powsybl.computation.ComputationManager; import org.junit.After; import org.junit.Before; @@ -105,12 +106,15 @@ public class CaseControllerTest { private FileSystem fileSystem; + private final String caseImportDestination = "case.import.destination"; + @Before public void setUp() { fileSystem = Jimfs.newFileSystem(Configuration.unix()); caseService.setFileSystem(fileSystem); caseService.setComputationManager(Mockito.mock(ComputationManager.class)); cleanDB(); + outputDestination.clear(); } private void cleanDB() { @@ -120,6 +124,8 @@ private void cleanDB() { @After public void tearDown() throws Exception { fileSystem.close(); + List destinations = List.of(caseImportDestination); + TestUtils.assertQueuesEmptyThenClear(destinations, outputDestination); } private void createStorageDir() throws IOException { @@ -136,29 +142,42 @@ private static MockMultipartFile createMockMultipartFile(String fileName) throws } @Test - public void test() throws Exception { + public void testStorageNotCreated() throws Exception { // expect a fail since the storage dir. is not created mvc.perform(delete("/v1/cases")) .andExpect(status().isUnprocessableEntity()); + } + @Test + public void testDeleteCases() throws Exception { // create the storage dir createStorageDir(); - // now it must work mvc.perform(delete("/v1/cases")) .andExpect(status().isOk()); + } + + @Test + public void testCheckNonExistingCase() throws Exception { + // create the storage dir + createStorageDir(); // check if the case exists (except a false) mvc.perform(get("/v1/cases/{caseUuid}/exists", RANDOM_UUID)) .andExpect(status().isOk()) .andExpect(content().string("false")) .andReturn(); + } + + @Test + public void testImportValidCase() throws Exception { + createStorageDir(); // import a case UUID firstCaseUuid = importCase(TEST_CASE, false); // assert that the broker message has been sent - Message messageImport = outputDestination.receive(1000, "case.import.destination"); + Message messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); MessageHeaders headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -203,20 +222,76 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().string("true")) .andReturn(); + } + + @Test + public void testImportInvalidFile() throws Exception { + createStorageDir(); // import a non valid case and expect a fail mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile(NOT_A_NETWORK))) + .file(createMockMultipartFile(NOT_A_NETWORK))) .andExpect(status().isUnprocessableEntity()) .andExpect(content().string(startsWith("This file cannot be imported"))) .andReturn(); // import a non valid case with a valid extension and expect a fail mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile(STILL_NOT_A_NETWORK))) + .file(createMockMultipartFile(STILL_NOT_A_NETWORK))) .andExpect(status().isUnprocessableEntity()) .andExpect(content().string(startsWith("This file cannot be imported"))) .andReturn(); + } + + @Test + public void testDownloadNonExistingCase() throws Exception { + createStorageDir(); + + // download a non existing case + mvc.perform(get(GET_CASE_URL, UUID.randomUUID())) + .andExpect(status().isNoContent()) + .andReturn(); + } + + @Test + public void testExportNonExistingCaseFromat() throws Exception { + createStorageDir(); + + // import a case + UUID firstCaseUuid = importCase(TEST_CASE, false); + + // export a case in a non-existing format + mvc.perform(post(GET_CASE_URL, firstCaseUuid).param("format", "JPEG")) + .andExpect(status().isUnprocessableEntity()); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); + } + + @Test + public void deleteNonExistingCase() throws Exception { + createStorageDir(); + + // import a case + UUID caseaseUuid = importCase(TEST_CASE, false); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); + + // delete the case + mvc.perform(delete(GET_CASE_URL, caseaseUuid)) + .andExpect(status().isOk()); + + // delete non existing file + mvc.perform(delete(GET_CASE_URL, caseaseUuid)) + .andExpect(content().string(startsWith("The directory with the following uuid doesn't exist:"))) + .andReturn(); + + } + + @Test + public void test() throws Exception { + // create the storage dir + createStorageDir(); + + // import a case + UUID firstCaseUuid = importCase(TEST_CASE, false); // list the cases and expect the one imported before mvc.perform(get("/v1/cases")) @@ -236,18 +311,14 @@ public void test() throws Exception { .andExpect(content().contentTypeCompatibleWith(MediaType.APPLICATION_OCTET_STREAM)) .andReturn(); assertThat(mvcResult.getResponse().getHeader("content-disposition")).contains("attachment;"); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); - // downlaod a case + // download a case mvc.perform(get(GET_CASE_URL, firstCaseUuid)) .andExpect(status().isOk()) .andExpect(content().xml(testCaseContent)) .andReturn(); - // downlaod a non existing case - mvc.perform(get(GET_CASE_URL, UUID.randomUUID())) - .andExpect(status().isNoContent()) - .andReturn(); - // export a case in CGMES format mvcResult = mvc.perform(post(GET_CASE_URL, firstCaseUuid).param("format", "CGMES")) .andExpect(status().isOk()) @@ -255,29 +326,15 @@ public void test() throws Exception { .andReturn(); assertThat(mvcResult.getResponse().getHeader("content-disposition")).contains("attachment;"); - // export a non-existing case - mvc.perform(post(GET_CASE_URL, UUID.randomUUID()).param("format", "XIIDM")) - .andExpect(status().isNoContent()) - .andReturn(); - - // export a case in a non-existing format - mvc.perform(post(GET_CASE_URL, firstCaseUuid).param("format", "JPEG")) - .andExpect(status().isUnprocessableEntity()); - // delete the case mvc.perform(delete(GET_CASE_URL, firstCaseUuid)) .andExpect(status().isOk()); - // delete non existing file - mvc.perform(delete(GET_CASE_URL, firstCaseUuid)) - .andExpect(content().string(startsWith("The directory with the following uuid doesn't exist:"))) - .andReturn(); - // import a case to delete it UUID secondCaseUuid = importCase(TEST_CASE, false); // assert that the broker message has been sent - Message messageImportPrivate2 = outputDestination.receive(1000, "case.import.destination"); + Message messageImportPrivate2 = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImportPrivate2.getPayload())); MessageHeaders headersPrivateCase2 = messageImportPrivate2.getHeaders(); assertEquals("testCase.xiidm", headersPrivateCase2.get(CaseInfos.NAME_HEADER_KEY)); @@ -285,7 +342,7 @@ public void test() throws Exception { assertEquals("XIIDM", headersPrivateCase2.get(CaseInfos.FORMAT_HEADER_KEY)); //check that the case doesn't have an expiration date - caseMetadataEntity = caseMetadataRepository.findById(secondCaseUuid).orElseThrow(); + CaseMetadataEntity caseMetadataEntity = caseMetadataRepository.findById(secondCaseUuid).orElseThrow(); assertEquals(secondCaseUuid, caseMetadataEntity.getId()); assertNull(caseMetadataEntity.getExpirationDate()); @@ -299,9 +356,9 @@ public void test() throws Exception { UUID caseUuid = importCase(TEST_CASE, false); // assert that the broker message has been sent - messageImport = outputDestination.receive(1000, "case.import.destination"); + Message messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); - headersCase = messageImport.getHeaders(); + MessageHeaders headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); assertEquals(caseUuid, headersCase.get(CaseInfos.UUID_HEADER_KEY)); assertEquals("XIIDM", headersCase.get(CaseInfos.FORMAT_HEADER_KEY)); @@ -319,7 +376,7 @@ public void test() throws Exception { String duplicateCaseUuid = duplicateResult.getResponse().getContentAsString().replace("\"", ""); // assert that broker message has been sent after duplication - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals(UUID.fromString(duplicateCaseUuid), headersCase.get(CaseInfos.UUID_HEADER_KEY)); @@ -337,7 +394,7 @@ public void test() throws Exception { Instant afterImportDate = Instant.now().plus(1, ChronoUnit.HOURS); // assert that the broker message has been sent - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -362,7 +419,7 @@ public void test() throws Exception { assertNotEquals(caseUuid.toString(), duplicateCaseUuid2); // assert that broker message has been sent after duplication - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals(UUID.fromString(duplicateCaseUuid2), headersCase.get(CaseInfos.UUID_HEADER_KEY)); @@ -422,12 +479,14 @@ private UUID importCase(String testCase, Boolean withExpiration) throws Exceptio if (withExpiration) { importedCase = mvc.perform(multipart("/v1/cases") .file(createMockMultipartFile(testCase)) - .param("withExpiration", withExpiration.toString())) + .param("withExpiration", withExpiration.toString()) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); } else { importedCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile(testCase))) + .file(createMockMultipartFile(testCase)) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); } @@ -470,14 +529,15 @@ public void searchCaseTest() throws Exception { // import IIDM test case String aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("testCase.xiidm"))) + .file(createMockMultipartFile("testCase.xiidm")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); UUID aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - Message messageImport = outputDestination.receive(1000, "case.import.destination"); + Message messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); MessageHeaders headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -486,14 +546,15 @@ public void searchCaseTest() throws Exception { // import CGMES french file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200424T1330Z_2D_RTEFRANCE_001.zip"))) + .file(createMockMultipartFile("20200424T1330Z_2D_RTEFRANCE_001.zip")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200424T1330Z_2D_RTEFRANCE_001.zip", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -502,14 +563,15 @@ public void searchCaseTest() throws Exception { // import UCTE french file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200103_0915_FO5_FR0.UCT"))) + .file(createMockMultipartFile("20200103_0915_FO5_FR0.UCT")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200103_0915_FO5_FR0.UCT", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -518,14 +580,15 @@ public void searchCaseTest() throws Exception { // import UCTE german file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200103_0915_SN5_D80.UCT"))) + .file(createMockMultipartFile("20200103_0915_SN5_D80.UCT")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200103_0915_SN5_D80.UCT", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -534,14 +597,15 @@ public void searchCaseTest() throws Exception { // import UCTE swiss file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200103_0915_135_CH2.UCT"))) + .file(createMockMultipartFile("20200103_0915_135_CH2.UCT")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200103_0915_135_CH2.UCT", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -656,7 +720,7 @@ public void searchCaseTest() throws Exception { assertFalse(response.contains("\"name\":\"20200103_0915_135_CH2.UCT\"")); // reindex all cases - mvc.perform(post("/v1/cases/reindex-all")) + mvc.perform(post("/v1/supervision/cases/reindex")) .andExpect(status().isOk()); mvcResult = mvc.perform(get("/v1/cases/search") @@ -708,5 +772,6 @@ public void invalidFileInCaseDirectoryShouldBeIgnored() throws Exception { Files.delete(filePath); mvc.perform(delete("/v1/cases")) .andExpect(status().isOk()); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); } } diff --git a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java index 6408a1e..23e7724 100644 --- a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java +++ b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java @@ -60,9 +60,9 @@ private void cleanDB() { public void test() { Instant now = Instant.now(); Instant yesterday = now.minus(1, ChronoUnit.DAYS); - CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS)); - CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS)); - CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null); + CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS), false); + CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS), false); + CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null, false); caseMetadataRepository.save(shouldExpireEntity); caseMetadataRepository.save(shouldNotExpireEntity); caseMetadataRepository.save(noExpireDateEntity); diff --git a/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java new file mode 100644 index 0000000..454aba8 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.repository.CaseMetadataRepository; +import com.powsybl.caseserver.services.SupervisionService; +import com.powsybl.computation.ComputationManager; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * @author Jamal KHEYYAD + */ +@RunWith(SpringRunner.class) +@AutoConfigureMockMvc +@SpringBootTest(properties = {"case-store-directory=/cases"}) +@ContextConfiguration(classes = {CaseApplication.class}) +public class SupervisionControllerTest { + @Autowired + SupervisionService supervisionService; + @Autowired + CaseMetadataRepository caseMetadataRepository; + @Autowired + CaseService caseService; + + @Autowired + private MockMvc mockMvc; + + @Value("${case-store-directory}") + private String rootDirectory; + + private static final String TEST_CASE = "testCase.xiidm"; + private FileSystem fileSystem; + + @Test + public void testGetCaseInfosCount() throws Exception { + createStorageDir(); + importCase(true); + importCase(true); + importCase(false); + + mockMvc.perform(post("/v1/supervision/cases/reindex")) + .andExpect(status().isOk()); + + Assert.assertEquals(2, supervisionService.getIndexedCasesCount()); + + } + + @Test + public void testReindexAll() throws Exception { + createStorageDir(); + importCase(true); + importCase(true); + importCase(false); + + mockMvc.perform(delete("/v1/supervision/cases/indexation")) + .andExpect(status().isOk()); + + Assert.assertEquals(0, supervisionService.getIndexedCasesCount()); + + //reindex + mockMvc.perform(post("/v1/supervision/cases/reindex")) + .andExpect(status().isOk()); + + String countStr = mockMvc.perform(get("/v1/supervision/cases/indexation-count")) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + Assert.assertEquals("2", countStr); + Assert.assertEquals(2, supervisionService.getIndexedCasesCount()); + + } + + @Test + public void testGetIndexName() throws Exception { + String result = mockMvc.perform(get("/v1/supervision/cases/index-name")) + .andReturn().getResponse().getContentAsString(); + + Assert.assertEquals("cases", result); + } + + private void importCase(Boolean indexed) throws Exception { + mockMvc.perform(multipart("/v1/cases") + .file(createMockMultipartFile()) + .param("withIndexation", indexed.toString())) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + } + + private static MockMultipartFile createMockMultipartFile() throws IOException { + try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + SupervisionControllerTest.TEST_CASE)) { + return new MockMultipartFile("file", SupervisionControllerTest.TEST_CASE, MediaType.TEXT_PLAIN_VALUE, inputStream); + } + } + + @Before + public void setUp() { + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + caseService.setFileSystem(fileSystem); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + } + + @After + public void tearDown() throws Exception { + fileSystem.close(); + cleanDB(); + } + + private void cleanDB() { + caseMetadataRepository.deleteAll(); + } + + private void createStorageDir() throws IOException { + Path path = fileSystem.getPath(rootDirectory); + if (!Files.exists(path)) { + Files.createDirectories(path); + } + } +} diff --git a/src/test/java/com/powsybl/caseserver/utils/TestUtils.java b/src/test/java/com/powsybl/caseserver/utils/TestUtils.java new file mode 100644 index 0000000..f4fd835 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/utils/TestUtils.java @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package com.powsybl.caseserver.utils; + +import org.springframework.cloud.stream.binder.test.OutputDestination; + +import java.util.List; + +import static org.junit.Assert.assertNull; + +/** + * @author Jamal KHEYYAD + */ +public final class TestUtils { + + private static final long TIMEOUT = 100; + + private TestUtils() { + + } + + public static void assertQueuesEmptyThenClear(List destinations, OutputDestination output) { + try { + destinations.forEach(destination -> assertNull("Should not be any messages in queue " + destination + " : ", output.receive(TIMEOUT, destination))); + } catch (NullPointerException e) { + // Ignoring + } finally { + output.clear(); // purge in order to not fail the other tests + } + } +} From 1b6db241a397ba7a7750d90df56127e4604b8133 Mon Sep 17 00:00:00 2001 From: jamal-khey Date: Thu, 22 Aug 2024 10:07:54 +0200 Subject: [PATCH 11/11] fix a bug affecting duplication of non indexed cases (#40) Signed-off-by: jamal-khey --- .../com/powsybl/caseserver/CaseService.java | 17 +++++++++----- .../caseserver/CaseControllerTest.java | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/CaseService.java index 29c8892..cf2ddf5 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/CaseService.java @@ -53,6 +53,7 @@ import java.util.zip.ZipOutputStream; import static com.powsybl.caseserver.CaseException.createDirectoryNotFound; +import static com.powsybl.caseserver.dto.CaseInfos.*; /** * @author Abdelsalem Hedhili @@ -114,7 +115,7 @@ public List getCases(Path directory) { private CaseInfos getCaseInfos(Path file) { try { - return createInfos(file.getFileName().toString(), UUID.fromString(file.getParent().getFileName().toString()), getFormat(file)); + return createInfos(file, UUID.fromString(file.getParent().getFileName().toString())); } catch (Exception e) { LOGGER.error("Error processing file {}: {}", file.getFileName(), e.getMessage(), e); return null; @@ -230,12 +231,14 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { newCaseFile = newCaseUuidDirectory.resolve(existingCaseFile.getFileName()); Files.copy(existingCaseFile, newCaseFile, StandardCopyOption.COPY_ATTRIBUTES); - CaseInfos existingCaseInfos = caseInfosService.getCaseInfosByUuid(sourceCaseUuid.toString()).orElseThrow(); - CaseInfos caseInfos = createInfos(existingCaseInfos.getName(), newCaseUuid, existingCaseInfos.getFormat()); - caseInfosService.addCaseInfos(caseInfos); - CaseMetadataEntity existingCase = getCaseMetaDataEntity(sourceCaseUuid); + CaseInfos caseInfos = createInfos(newCaseFile, newCaseUuid); + if (existingCase.isIndexed()) { + caseInfosService.addCaseInfos(caseInfos); + } + createCaseMetadataEntity(newCaseUuid, withExpiration, existingCase.isIndexed()); + sendImportMessage(caseInfos.createMessage()); return newCaseUuid; @@ -244,6 +247,10 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { } } + private CaseInfos createInfos(Path caseFile, UUID caseUuid) { + return createInfos(caseFile.getFileName().toString(), caseUuid, getFormat(caseFile)); + } + private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); } diff --git a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java index e92b746..bdcfb5d 100644 --- a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java @@ -474,6 +474,28 @@ public void test() throws Exception { assertTrue(response.contains("\"format\":\"XIIDM\"")); } + @Test + public void testDuplicateNonIndexedCase() throws Exception { + // create the storage dir + createStorageDir(); + + // import IIDM test case + String caseUuid = mvc.perform(multipart("/v1/cases") + .file(createMockMultipartFile(TEST_CASE)) + .param("withExpiration", "false") + .param("withIndexation", "false")) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); + //duplicate an existing case + String duplicateCaseStr = mvc.perform(post("/v1/cases?duplicateFrom=" + caseUuid.substring(1, caseUuid.length() - 1))) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + UUID duplicateCaseUuid = UUID.fromString(duplicateCaseStr.substring(1, duplicateCaseStr.length() - 1)); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); + assertFalse(caseMetadataRepository.findById(duplicateCaseUuid).get().isIndexed()); + } + private UUID importCase(String testCase, Boolean withExpiration) throws Exception { String importedCase; if (withExpiration) {