From d27e42de57f45740f6a19b854a4b7d09e9b1a34a Mon Sep 17 00:00:00 2001 From: HARPER Jon Date: Mon, 16 Sep 2024 13:38:36 +0200 Subject: [PATCH] add indexed attribute index only elements marked for indexation (#39) Signed-off-by: jamal-khey --- .../powsybl/caseserver/CaseController.java | 13 +- .../com/powsybl/caseserver/CaseService.java | 42 ++--- .../caseserver/SupervisionController.java | 82 +++++++++ .../elasticsearch/CaseInfosService.java | 6 + .../caseserver/elasticsearch/ESConfig.java | 2 + .../repository/CaseMetadataEntity.java | 3 + .../repository/CaseMetadataRepository.java | 3 +- .../services/SupervisionService.java | 43 +++++ .../changesets/changelog_20240726T144717Z.xml | 10 ++ .../db/changelog/db.changelog-master.yaml | 7 +- .../caseserver/CaseControllerTest.java | 155 +++++++++++++----- .../caseserver/ScheduledCaseCleanerTest.java | 6 +- .../caseserver/SupervisionControllerTest.java | 146 +++++++++++++++++ .../powsybl/caseserver/utils/TestUtils.java | 36 ++++ 14 files changed, 475 insertions(+), 79 deletions(-) create mode 100644 src/main/java/com/powsybl/caseserver/SupervisionController.java create mode 100644 src/main/java/com/powsybl/caseserver/services/SupervisionService.java create mode 100644 src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml create mode 100644 src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java create mode 100644 src/test/java/com/powsybl/caseserver/utils/TestUtils.java diff --git a/src/main/java/com/powsybl/caseserver/CaseController.java b/src/main/java/com/powsybl/caseserver/CaseController.java index 17776ad..0f480f9 100644 --- a/src/main/java/com/powsybl/caseserver/CaseController.java +++ b/src/main/java/com/powsybl/caseserver/CaseController.java @@ -152,9 +152,10 @@ public ResponseEntity exists(@PathVariable("caseUuid") UUID caseUuid) { @Operation(summary = "import a case") @SuppressWarnings("javasecurity:S5145") public ResponseEntity importCase(@RequestParam("file") MultipartFile file, - @RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration) { + @RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration, + @RequestParam(value = "withIndexation", required = false, defaultValue = "false") boolean withIndexation) { LOGGER.debug("importCase request received with file = {}", file.getName()); - UUID caseUuid = caseService.importCase(file, withExpiration); + UUID caseUuid = caseService.importCase(file, withExpiration, withIndexation); return ResponseEntity.ok().body(caseUuid); } @@ -206,14 +207,6 @@ public ResponseEntity> searchCases(@RequestParam(value = "q") St return ResponseEntity.ok().body(cases); } - @PostMapping(value = "/cases/reindex-all") - @Operation(summary = "reindex all cases") - public ResponseEntity reindexAllCases() { - LOGGER.debug("reindex all cases request received"); - caseService.reindexAllCases(); - return ResponseEntity.ok().build(); - } - @GetMapping(value = "/cases/metadata") @Operation(summary = "Get cases Metadata") public ResponseEntity> getMetadata(@RequestParam("ids") List ids) { diff --git a/src/main/java/com/powsybl/caseserver/CaseService.java b/src/main/java/com/powsybl/caseserver/CaseService.java index 27b9fdd..29c8892 100644 --- a/src/main/java/com/powsybl/caseserver/CaseService.java +++ b/src/main/java/com/powsybl/caseserver/CaseService.java @@ -46,14 +46,7 @@ import java.nio.file.StandardCopyOption; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; @@ -179,7 +172,7 @@ boolean caseExists(UUID caseName) { return Files.exists(caseFile) && Files.isRegularFile(caseFile); } - UUID importCase(MultipartFile mpf, boolean withExpiration) { + UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation) { checkStorageInitialization(); UUID caseUuid = UUID.randomUUID(); @@ -214,9 +207,11 @@ UUID importCase(MultipartFile mpf, boolean withExpiration) { throw e; } - createCaseMetadataEntity(caseUuid, withExpiration); + createCaseMetadataEntity(caseUuid, withExpiration, withIndexation); CaseInfos caseInfos = createInfos(caseFile.getFileName().toString(), caseUuid, importer.getFormat()); - caseInfosService.addCaseInfos(caseInfos); + if (withIndexation) { + caseInfosService.addCaseInfos(caseInfos); + } sendImportMessage(caseInfos.createMessage()); return caseUuid; } @@ -238,8 +233,9 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { CaseInfos existingCaseInfos = caseInfosService.getCaseInfosByUuid(sourceCaseUuid.toString()).orElseThrow(); CaseInfos caseInfos = createInfos(existingCaseInfos.getName(), newCaseUuid, existingCaseInfos.getFormat()); caseInfosService.addCaseInfos(caseInfos); - createCaseMetadataEntity(newCaseUuid, withExpiration); + CaseMetadataEntity existingCase = getCaseMetaDataEntity(sourceCaseUuid); + createCaseMetadataEntity(newCaseUuid, withExpiration, existingCase.isIndexed()); sendImportMessage(caseInfos.createMessage()); return newCaseUuid; @@ -248,12 +244,24 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) { } } - private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration) { + private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) { + return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); + } + + private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, boolean withIndexation) { Instant expirationTime = null; if (withExpiration) { expirationTime = Instant.now().plus(1, ChronoUnit.HOURS); } - caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime)); + caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime, withIndexation)); + } + + public List getCasesToReindex() { + Set casesToReindex = caseMetadataRepository.findAllByIndexedTrue() + .stream() + .map(CaseMetadataEntity::getId) + .collect(Collectors.toSet()); + return getCases(getStorageRootDir()).stream().filter(c -> casesToReindex.contains(c.getUuid())).toList(); } CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { @@ -269,7 +277,7 @@ CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) { @Transactional public void disableCaseExpiration(UUID caseUuid) { - CaseMetadataEntity caseMetadataEntity = caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found")); + CaseMetadataEntity caseMetadataEntity = getCaseMetaDataEntity(caseUuid); caseMetadataEntity.setExpirationDate(null); } @@ -373,10 +381,6 @@ private void sendImportMessage(Message message) { caseInfosPublisher.send("publishCaseImport-out-0", message); } - public void reindexAllCases() { - caseInfosService.recreateAllCaseInfos(getCases(getStorageRootDir())); - } - public List getMetadata(List ids) { List cases = new ArrayList<>(); ids.forEach(caseUuid -> { diff --git a/src/main/java/com/powsybl/caseserver/SupervisionController.java b/src/main/java/com/powsybl/caseserver/SupervisionController.java new file mode 100644 index 0000000..cf826a9 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/SupervisionController.java @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.powsybl.caseserver.elasticsearch.CaseInfosService; +import com.powsybl.caseserver.services.SupervisionService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +/** + * @author Jamal KHEYYAD + */ +@RestController +@RequestMapping(value = "/" + CaseConstants.API_VERSION + "/supervision") +@Tag(name = "case-server - Supervision") +public class SupervisionController { + private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionController.class); + + private final SupervisionService supervisionService; + private final CaseService caseService; + private final ClientConfiguration elasticsearchClientConfiguration; + private final CaseInfosService caseInfosService; + + public SupervisionController(SupervisionService supervisionService, CaseService caseService, ClientConfiguration elasticsearchClientConfiguration, CaseInfosService caseInfosService) { + this.supervisionService = supervisionService; + this.caseService = caseService; + this.elasticsearchClientConfiguration = elasticsearchClientConfiguration; + this.caseInfosService = caseInfosService; + } + + @GetMapping(value = "/elasticsearch-host") + @Operation(summary = "get the elasticsearch address") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "the elasticsearch address")}) + public ResponseEntity getElasticsearchHost() { + String host = elasticsearchClientConfiguration.getEndpoints().get(0).getHostName() + + ":" + + elasticsearchClientConfiguration.getEndpoints().get(0).getPort(); + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(host); + } + + @GetMapping(value = "/cases/index-name") + @Operation(summary = "get the indexed cases index name") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Indexed directory cases index name")}) + public ResponseEntity getIndexedCasesFIndexName() { + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(caseInfosService.getDirectoryCasesIndexName()); + } + + @PostMapping(value = "/cases/reindex") + @Operation(summary = "reindex all cases") + public ResponseEntity reindexAllCases() { + LOGGER.debug("reindex all cases request received"); + caseInfosService.recreateAllCaseInfos(caseService.getCasesToReindex()); + return ResponseEntity.ok().build(); + } + + @DeleteMapping(value = "/cases/indexation") + @Operation(summary = "delete indexed cases") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all indexed cases have been deleted")}) + public ResponseEntity deleteIndexedCases() { + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(Long.toString(supervisionService.deleteIndexedCases())); + } + + @GetMapping(value = "/cases/indexation-count") + @Operation(summary = "get indexed cases count") + @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Indexed cases count")}) + public ResponseEntity getIndexedCasesCount() { + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(Long.toString(supervisionService.getIndexedCasesCount())); + } + +} diff --git a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java index 7c539c1..3286410 100644 --- a/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java +++ b/src/main/java/com/powsybl/caseserver/elasticsearch/CaseInfosService.java @@ -9,7 +9,9 @@ import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery; import com.google.common.collect.Lists; import com.powsybl.caseserver.dto.CaseInfos; +import lombok.Getter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.ComponentScan; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -42,6 +44,10 @@ public class CaseInfosService { @Autowired private ElasticsearchOperations operations; + @Value(ESConfig.CASE_INFOS_INDEX_NAME) + @Getter + private String directoryCasesIndexName; + public CaseInfos addCaseInfos(@NonNull final CaseInfos ci) { caseInfosRepository.save(ci); return ci; diff --git a/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java b/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java index bc35d05..1972b2e 100644 --- a/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java +++ b/src/main/java/com/powsybl/caseserver/elasticsearch/ESConfig.java @@ -32,6 +32,8 @@ @EnableElasticsearchRepositories public class ESConfig extends ElasticsearchConfiguration { + public static final String CASE_INFOS_INDEX_NAME = "#{@environment.getProperty('powsybl-ws.elasticsearch.index.prefix')}cases"; + @Value("#{'${spring.data.elasticsearch.embedded:false}' ? 'localhost' : '${spring.data.elasticsearch.host}'}") private String esHost; diff --git a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java index ebacd0e..2bca67f 100644 --- a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java +++ b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataEntity.java @@ -33,4 +33,7 @@ public class CaseMetadataEntity { @Column(name = "expirationDate", columnDefinition = "timestamptz") private Instant expirationDate; + + @Column(name = "indexed", columnDefinition = "boolean default false", nullable = false) + private boolean indexed = false; } diff --git a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java index 79c9cca..c44b383 100644 --- a/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java +++ b/src/main/java/com/powsybl/caseserver/repository/CaseMetadataRepository.java @@ -9,6 +9,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.util.List; import java.util.UUID; /** @@ -16,5 +17,5 @@ */ @Repository public interface CaseMetadataRepository extends JpaRepository { - + List findAllByIndexedTrue(); } diff --git a/src/main/java/com/powsybl/caseserver/services/SupervisionService.java b/src/main/java/com/powsybl/caseserver/services/SupervisionService.java new file mode 100644 index 0000000..5a11ab6 --- /dev/null +++ b/src/main/java/com/powsybl/caseserver/services/SupervisionService.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver.services; + +import com.powsybl.caseserver.elasticsearch.CaseInfosRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author Jamal KHEYYAD + */ +@Service +public class SupervisionService { + private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionService.class); + + private final CaseInfosRepository caseInfosRepository; + + public SupervisionService(CaseInfosRepository caseInfosRepository) { + this.caseInfosRepository = caseInfosRepository; + } + + public long deleteIndexedCases() { + AtomicReference startTime = new AtomicReference<>(); + startTime.set(System.nanoTime()); + + long nbIndexesToDelete = getIndexedCasesCount(); + caseInfosRepository.deleteAll(); + LOGGER.trace("Indexed cases deletion : {} seconds", TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime.get())); + return nbIndexesToDelete; + } + + public long getIndexedCasesCount() { + return caseInfosRepository.count(); + } +} diff --git a/src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml b/src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml new file mode 100644 index 0000000..3681251 --- /dev/null +++ b/src/main/resources/db/changelog/changesets/changelog_20240726T144717Z.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/main/resources/db/changelog/db.changelog-master.yaml b/src/main/resources/db/changelog/db.changelog-master.yaml index 7768178..5ad8263 100644 --- a/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/src/main/resources/db/changelog/db.changelog-master.yaml @@ -5,4 +5,9 @@ databaseChangeLog: - include: file: changesets/changelog_20240620T100317Z.xml - relativeToChangelogFile: true \ No newline at end of file + relativeToChangelogFile: true + + - include: + file: changesets/changelog_20240726T144717Z.xml + relativeToChangelogFile: true + diff --git a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java index f53be5e..e92b746 100644 --- a/src/test/java/com/powsybl/caseserver/CaseControllerTest.java +++ b/src/test/java/com/powsybl/caseserver/CaseControllerTest.java @@ -15,6 +15,7 @@ import com.powsybl.caseserver.parsers.entsoe.EntsoeFileNameParser; import com.powsybl.caseserver.repository.CaseMetadataEntity; import com.powsybl.caseserver.repository.CaseMetadataRepository; +import com.powsybl.caseserver.utils.TestUtils; import com.powsybl.computation.ComputationManager; import org.junit.After; import org.junit.Before; @@ -105,12 +106,15 @@ public class CaseControllerTest { private FileSystem fileSystem; + private final String caseImportDestination = "case.import.destination"; + @Before public void setUp() { fileSystem = Jimfs.newFileSystem(Configuration.unix()); caseService.setFileSystem(fileSystem); caseService.setComputationManager(Mockito.mock(ComputationManager.class)); cleanDB(); + outputDestination.clear(); } private void cleanDB() { @@ -120,6 +124,8 @@ private void cleanDB() { @After public void tearDown() throws Exception { fileSystem.close(); + List destinations = List.of(caseImportDestination); + TestUtils.assertQueuesEmptyThenClear(destinations, outputDestination); } private void createStorageDir() throws IOException { @@ -136,29 +142,42 @@ private static MockMultipartFile createMockMultipartFile(String fileName) throws } @Test - public void test() throws Exception { + public void testStorageNotCreated() throws Exception { // expect a fail since the storage dir. is not created mvc.perform(delete("/v1/cases")) .andExpect(status().isUnprocessableEntity()); + } + @Test + public void testDeleteCases() throws Exception { // create the storage dir createStorageDir(); - // now it must work mvc.perform(delete("/v1/cases")) .andExpect(status().isOk()); + } + + @Test + public void testCheckNonExistingCase() throws Exception { + // create the storage dir + createStorageDir(); // check if the case exists (except a false) mvc.perform(get("/v1/cases/{caseUuid}/exists", RANDOM_UUID)) .andExpect(status().isOk()) .andExpect(content().string("false")) .andReturn(); + } + + @Test + public void testImportValidCase() throws Exception { + createStorageDir(); // import a case UUID firstCaseUuid = importCase(TEST_CASE, false); // assert that the broker message has been sent - Message messageImport = outputDestination.receive(1000, "case.import.destination"); + Message messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); MessageHeaders headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -203,20 +222,76 @@ public void test() throws Exception { .andExpect(status().isOk()) .andExpect(content().string("true")) .andReturn(); + } + + @Test + public void testImportInvalidFile() throws Exception { + createStorageDir(); // import a non valid case and expect a fail mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile(NOT_A_NETWORK))) + .file(createMockMultipartFile(NOT_A_NETWORK))) .andExpect(status().isUnprocessableEntity()) .andExpect(content().string(startsWith("This file cannot be imported"))) .andReturn(); // import a non valid case with a valid extension and expect a fail mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile(STILL_NOT_A_NETWORK))) + .file(createMockMultipartFile(STILL_NOT_A_NETWORK))) .andExpect(status().isUnprocessableEntity()) .andExpect(content().string(startsWith("This file cannot be imported"))) .andReturn(); + } + + @Test + public void testDownloadNonExistingCase() throws Exception { + createStorageDir(); + + // download a non existing case + mvc.perform(get(GET_CASE_URL, UUID.randomUUID())) + .andExpect(status().isNoContent()) + .andReturn(); + } + + @Test + public void testExportNonExistingCaseFromat() throws Exception { + createStorageDir(); + + // import a case + UUID firstCaseUuid = importCase(TEST_CASE, false); + + // export a case in a non-existing format + mvc.perform(post(GET_CASE_URL, firstCaseUuid).param("format", "JPEG")) + .andExpect(status().isUnprocessableEntity()); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); + } + + @Test + public void deleteNonExistingCase() throws Exception { + createStorageDir(); + + // import a case + UUID caseaseUuid = importCase(TEST_CASE, false); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); + + // delete the case + mvc.perform(delete(GET_CASE_URL, caseaseUuid)) + .andExpect(status().isOk()); + + // delete non existing file + mvc.perform(delete(GET_CASE_URL, caseaseUuid)) + .andExpect(content().string(startsWith("The directory with the following uuid doesn't exist:"))) + .andReturn(); + + } + + @Test + public void test() throws Exception { + // create the storage dir + createStorageDir(); + + // import a case + UUID firstCaseUuid = importCase(TEST_CASE, false); // list the cases and expect the one imported before mvc.perform(get("/v1/cases")) @@ -236,18 +311,14 @@ public void test() throws Exception { .andExpect(content().contentTypeCompatibleWith(MediaType.APPLICATION_OCTET_STREAM)) .andReturn(); assertThat(mvcResult.getResponse().getHeader("content-disposition")).contains("attachment;"); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); - // downlaod a case + // download a case mvc.perform(get(GET_CASE_URL, firstCaseUuid)) .andExpect(status().isOk()) .andExpect(content().xml(testCaseContent)) .andReturn(); - // downlaod a non existing case - mvc.perform(get(GET_CASE_URL, UUID.randomUUID())) - .andExpect(status().isNoContent()) - .andReturn(); - // export a case in CGMES format mvcResult = mvc.perform(post(GET_CASE_URL, firstCaseUuid).param("format", "CGMES")) .andExpect(status().isOk()) @@ -255,29 +326,15 @@ public void test() throws Exception { .andReturn(); assertThat(mvcResult.getResponse().getHeader("content-disposition")).contains("attachment;"); - // export a non-existing case - mvc.perform(post(GET_CASE_URL, UUID.randomUUID()).param("format", "XIIDM")) - .andExpect(status().isNoContent()) - .andReturn(); - - // export a case in a non-existing format - mvc.perform(post(GET_CASE_URL, firstCaseUuid).param("format", "JPEG")) - .andExpect(status().isUnprocessableEntity()); - // delete the case mvc.perform(delete(GET_CASE_URL, firstCaseUuid)) .andExpect(status().isOk()); - // delete non existing file - mvc.perform(delete(GET_CASE_URL, firstCaseUuid)) - .andExpect(content().string(startsWith("The directory with the following uuid doesn't exist:"))) - .andReturn(); - // import a case to delete it UUID secondCaseUuid = importCase(TEST_CASE, false); // assert that the broker message has been sent - Message messageImportPrivate2 = outputDestination.receive(1000, "case.import.destination"); + Message messageImportPrivate2 = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImportPrivate2.getPayload())); MessageHeaders headersPrivateCase2 = messageImportPrivate2.getHeaders(); assertEquals("testCase.xiidm", headersPrivateCase2.get(CaseInfos.NAME_HEADER_KEY)); @@ -285,7 +342,7 @@ public void test() throws Exception { assertEquals("XIIDM", headersPrivateCase2.get(CaseInfos.FORMAT_HEADER_KEY)); //check that the case doesn't have an expiration date - caseMetadataEntity = caseMetadataRepository.findById(secondCaseUuid).orElseThrow(); + CaseMetadataEntity caseMetadataEntity = caseMetadataRepository.findById(secondCaseUuid).orElseThrow(); assertEquals(secondCaseUuid, caseMetadataEntity.getId()); assertNull(caseMetadataEntity.getExpirationDate()); @@ -299,9 +356,9 @@ public void test() throws Exception { UUID caseUuid = importCase(TEST_CASE, false); // assert that the broker message has been sent - messageImport = outputDestination.receive(1000, "case.import.destination"); + Message messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); - headersCase = messageImport.getHeaders(); + MessageHeaders headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); assertEquals(caseUuid, headersCase.get(CaseInfos.UUID_HEADER_KEY)); assertEquals("XIIDM", headersCase.get(CaseInfos.FORMAT_HEADER_KEY)); @@ -319,7 +376,7 @@ public void test() throws Exception { String duplicateCaseUuid = duplicateResult.getResponse().getContentAsString().replace("\"", ""); // assert that broker message has been sent after duplication - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals(UUID.fromString(duplicateCaseUuid), headersCase.get(CaseInfos.UUID_HEADER_KEY)); @@ -337,7 +394,7 @@ public void test() throws Exception { Instant afterImportDate = Instant.now().plus(1, ChronoUnit.HOURS); // assert that the broker message has been sent - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -362,7 +419,7 @@ public void test() throws Exception { assertNotEquals(caseUuid.toString(), duplicateCaseUuid2); // assert that broker message has been sent after duplication - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals(UUID.fromString(duplicateCaseUuid2), headersCase.get(CaseInfos.UUID_HEADER_KEY)); @@ -422,12 +479,14 @@ private UUID importCase(String testCase, Boolean withExpiration) throws Exceptio if (withExpiration) { importedCase = mvc.perform(multipart("/v1/cases") .file(createMockMultipartFile(testCase)) - .param("withExpiration", withExpiration.toString())) + .param("withExpiration", withExpiration.toString()) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); } else { importedCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile(testCase))) + .file(createMockMultipartFile(testCase)) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); } @@ -470,14 +529,15 @@ public void searchCaseTest() throws Exception { // import IIDM test case String aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("testCase.xiidm"))) + .file(createMockMultipartFile("testCase.xiidm")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); UUID aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - Message messageImport = outputDestination.receive(1000, "case.import.destination"); + Message messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); MessageHeaders headersCase = messageImport.getHeaders(); assertEquals("testCase.xiidm", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -486,14 +546,15 @@ public void searchCaseTest() throws Exception { // import CGMES french file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200424T1330Z_2D_RTEFRANCE_001.zip"))) + .file(createMockMultipartFile("20200424T1330Z_2D_RTEFRANCE_001.zip")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200424T1330Z_2D_RTEFRANCE_001.zip", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -502,14 +563,15 @@ public void searchCaseTest() throws Exception { // import UCTE french file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200103_0915_FO5_FR0.UCT"))) + .file(createMockMultipartFile("20200103_0915_FO5_FR0.UCT")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200103_0915_FO5_FR0.UCT", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -518,14 +580,15 @@ public void searchCaseTest() throws Exception { // import UCTE german file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200103_0915_SN5_D80.UCT"))) + .file(createMockMultipartFile("20200103_0915_SN5_D80.UCT")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200103_0915_SN5_D80.UCT", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -534,14 +597,15 @@ public void searchCaseTest() throws Exception { // import UCTE swiss file aCase = mvc.perform(multipart("/v1/cases") - .file(createMockMultipartFile("20200103_0915_135_CH2.UCT"))) + .file(createMockMultipartFile("20200103_0915_135_CH2.UCT")) + .param("withIndexation", "true")) .andExpect(status().isOk()) .andReturn().getResponse().getContentAsString(); aCaseUuid = UUID.fromString(aCase.substring(1, aCase.length() - 1)); // assert that broker message has been sent and properties are the right ones - messageImport = outputDestination.receive(1000, "case.import.destination"); + messageImport = outputDestination.receive(1000, caseImportDestination); assertEquals("", new String(messageImport.getPayload())); headersCase = messageImport.getHeaders(); assertEquals("20200103_0915_135_CH2.UCT", headersCase.get(CaseInfos.NAME_HEADER_KEY)); @@ -656,7 +720,7 @@ public void searchCaseTest() throws Exception { assertFalse(response.contains("\"name\":\"20200103_0915_135_CH2.UCT\"")); // reindex all cases - mvc.perform(post("/v1/cases/reindex-all")) + mvc.perform(post("/v1/supervision/cases/reindex")) .andExpect(status().isOk()); mvcResult = mvc.perform(get("/v1/cases/search") @@ -708,5 +772,6 @@ public void invalidFileInCaseDirectoryShouldBeIgnored() throws Exception { Files.delete(filePath); mvc.perform(delete("/v1/cases")) .andExpect(status().isOk()); + assertNotNull(outputDestination.receive(1000, caseImportDestination)); } } diff --git a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java index 6408a1e..23e7724 100644 --- a/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java +++ b/src/test/java/com/powsybl/caseserver/ScheduledCaseCleanerTest.java @@ -60,9 +60,9 @@ private void cleanDB() { public void test() { Instant now = Instant.now(); Instant yesterday = now.minus(1, ChronoUnit.DAYS); - CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS)); - CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS)); - CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null); + CaseMetadataEntity shouldNotExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), now.plus(1, ChronoUnit.HOURS), false); + CaseMetadataEntity shouldExpireEntity = new CaseMetadataEntity(UUID.randomUUID(), yesterday.plus(1, ChronoUnit.HOURS), false); + CaseMetadataEntity noExpireDateEntity = new CaseMetadataEntity(UUID.randomUUID(), null, false); caseMetadataRepository.save(shouldExpireEntity); caseMetadataRepository.save(shouldNotExpireEntity); caseMetadataRepository.save(noExpireDateEntity); diff --git a/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java b/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java new file mode 100644 index 0000000..454aba8 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/SupervisionControllerTest.java @@ -0,0 +1,146 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.caseserver; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import com.powsybl.caseserver.repository.CaseMetadataRepository; +import com.powsybl.caseserver.services.SupervisionService; +import com.powsybl.computation.ComputationManager; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * @author Jamal KHEYYAD + */ +@RunWith(SpringRunner.class) +@AutoConfigureMockMvc +@SpringBootTest(properties = {"case-store-directory=/cases"}) +@ContextConfiguration(classes = {CaseApplication.class}) +public class SupervisionControllerTest { + @Autowired + SupervisionService supervisionService; + @Autowired + CaseMetadataRepository caseMetadataRepository; + @Autowired + CaseService caseService; + + @Autowired + private MockMvc mockMvc; + + @Value("${case-store-directory}") + private String rootDirectory; + + private static final String TEST_CASE = "testCase.xiidm"; + private FileSystem fileSystem; + + @Test + public void testGetCaseInfosCount() throws Exception { + createStorageDir(); + importCase(true); + importCase(true); + importCase(false); + + mockMvc.perform(post("/v1/supervision/cases/reindex")) + .andExpect(status().isOk()); + + Assert.assertEquals(2, supervisionService.getIndexedCasesCount()); + + } + + @Test + public void testReindexAll() throws Exception { + createStorageDir(); + importCase(true); + importCase(true); + importCase(false); + + mockMvc.perform(delete("/v1/supervision/cases/indexation")) + .andExpect(status().isOk()); + + Assert.assertEquals(0, supervisionService.getIndexedCasesCount()); + + //reindex + mockMvc.perform(post("/v1/supervision/cases/reindex")) + .andExpect(status().isOk()); + + String countStr = mockMvc.perform(get("/v1/supervision/cases/indexation-count")) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + Assert.assertEquals("2", countStr); + Assert.assertEquals(2, supervisionService.getIndexedCasesCount()); + + } + + @Test + public void testGetIndexName() throws Exception { + String result = mockMvc.perform(get("/v1/supervision/cases/index-name")) + .andReturn().getResponse().getContentAsString(); + + Assert.assertEquals("cases", result); + } + + private void importCase(Boolean indexed) throws Exception { + mockMvc.perform(multipart("/v1/cases") + .file(createMockMultipartFile()) + .param("withIndexation", indexed.toString())) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + } + + private static MockMultipartFile createMockMultipartFile() throws IOException { + try (InputStream inputStream = CaseControllerTest.class.getResourceAsStream("/" + SupervisionControllerTest.TEST_CASE)) { + return new MockMultipartFile("file", SupervisionControllerTest.TEST_CASE, MediaType.TEXT_PLAIN_VALUE, inputStream); + } + } + + @Before + public void setUp() { + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + caseService.setFileSystem(fileSystem); + caseService.setComputationManager(Mockito.mock(ComputationManager.class)); + } + + @After + public void tearDown() throws Exception { + fileSystem.close(); + cleanDB(); + } + + private void cleanDB() { + caseMetadataRepository.deleteAll(); + } + + private void createStorageDir() throws IOException { + Path path = fileSystem.getPath(rootDirectory); + if (!Files.exists(path)) { + Files.createDirectories(path); + } + } +} diff --git a/src/test/java/com/powsybl/caseserver/utils/TestUtils.java b/src/test/java/com/powsybl/caseserver/utils/TestUtils.java new file mode 100644 index 0000000..f4fd835 --- /dev/null +++ b/src/test/java/com/powsybl/caseserver/utils/TestUtils.java @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package com.powsybl.caseserver.utils; + +import org.springframework.cloud.stream.binder.test.OutputDestination; + +import java.util.List; + +import static org.junit.Assert.assertNull; + +/** + * @author Jamal KHEYYAD + */ +public final class TestUtils { + + private static final long TIMEOUT = 100; + + private TestUtils() { + + } + + public static void assertQueuesEmptyThenClear(List destinations, OutputDestination output) { + try { + destinations.forEach(destination -> assertNull("Should not be any messages in queue " + destination + " : ", output.receive(TIMEOUT, destination))); + } catch (NullPointerException e) { + // Ignoring + } finally { + output.clear(); // purge in order to not fail the other tests + } + } +}