Skip to content

Commit

Permalink
add indexed attribute index only elements marked for indexation (#39)
Browse files Browse the repository at this point in the history
Signed-off-by: jamal-khey <myjamal89@gmail.com>
  • Loading branch information
jamal-khey authored and jonenst committed Sep 16, 2024
1 parent 75d27fe commit e9614cf
Show file tree
Hide file tree
Showing 14 changed files with 475 additions and 79 deletions.
13 changes: 3 additions & 10 deletions src/main/java/com/powsybl/caseserver/CaseController.java
Original file line number Diff line number Diff line change
Expand Up @@ -152,9 +152,10 @@ public ResponseEntity<Boolean> exists(@PathVariable("caseUuid") UUID caseUuid) {
@Operation(summary = "import a case")
@SuppressWarnings("javasecurity:S5145")
public ResponseEntity<UUID> importCase(@RequestParam("file") MultipartFile file,
@RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration) {
@RequestParam(value = "withExpiration", required = false, defaultValue = "false") boolean withExpiration,
@RequestParam(value = "withIndexation", required = false, defaultValue = "false") boolean withIndexation) {
LOGGER.debug("importCase request received with file = {}", file.getName());
UUID caseUuid = caseService.importCase(file, withExpiration);
UUID caseUuid = caseService.importCase(file, withExpiration, withIndexation);
return ResponseEntity.ok().body(caseUuid);
}

Expand Down Expand Up @@ -206,14 +207,6 @@ public ResponseEntity<List<CaseInfos>> searchCases(@RequestParam(value = "q") St
return ResponseEntity.ok().body(cases);
}

@PostMapping(value = "/cases/reindex-all")
@Operation(summary = "reindex all cases")
public ResponseEntity<Void> reindexAllCases() {
LOGGER.debug("reindex all cases request received");
caseService.reindexAllCases();
return ResponseEntity.ok().build();
}

@GetMapping(value = "/cases/metadata")
@Operation(summary = "Get cases Metadata")
public ResponseEntity<List<CaseInfos>> getMetadata(@RequestParam("ids") List<UUID> ids) {
Expand Down
42 changes: 23 additions & 19 deletions src/main/java/com/powsybl/caseserver/CaseService.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,7 @@
import java.nio.file.StandardCopyOption;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.UUID;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
Expand Down Expand Up @@ -179,7 +172,7 @@ boolean caseExists(UUID caseName) {
return Files.exists(caseFile) && Files.isRegularFile(caseFile);
}

UUID importCase(MultipartFile mpf, boolean withExpiration) {
UUID importCase(MultipartFile mpf, boolean withExpiration, boolean withIndexation) {
checkStorageInitialization();

UUID caseUuid = UUID.randomUUID();
Expand Down Expand Up @@ -214,9 +207,11 @@ UUID importCase(MultipartFile mpf, boolean withExpiration) {
throw e;
}

createCaseMetadataEntity(caseUuid, withExpiration);
createCaseMetadataEntity(caseUuid, withExpiration, withIndexation);
CaseInfos caseInfos = createInfos(caseFile.getFileName().toString(), caseUuid, importer.getFormat());
caseInfosService.addCaseInfos(caseInfos);
if (withIndexation) {
caseInfosService.addCaseInfos(caseInfos);
}
sendImportMessage(caseInfos.createMessage());
return caseUuid;
}
Expand All @@ -238,8 +233,9 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) {
CaseInfos existingCaseInfos = caseInfosService.getCaseInfosByUuid(sourceCaseUuid.toString()).orElseThrow();
CaseInfos caseInfos = createInfos(existingCaseInfos.getName(), newCaseUuid, existingCaseInfos.getFormat());
caseInfosService.addCaseInfos(caseInfos);
createCaseMetadataEntity(newCaseUuid, withExpiration);

CaseMetadataEntity existingCase = getCaseMetaDataEntity(sourceCaseUuid);
createCaseMetadataEntity(newCaseUuid, withExpiration, existingCase.isIndexed());
sendImportMessage(caseInfos.createMessage());
return newCaseUuid;

Expand All @@ -248,12 +244,24 @@ UUID duplicateCase(UUID sourceCaseUuid, boolean withExpiration) {
}
}

private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration) {
private CaseMetadataEntity getCaseMetaDataEntity(UUID caseUuid) {
return caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found"));
}

private void createCaseMetadataEntity(UUID newCaseUuid, boolean withExpiration, boolean withIndexation) {
Instant expirationTime = null;
if (withExpiration) {
expirationTime = Instant.now().plus(1, ChronoUnit.HOURS);
}
caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime));
caseMetadataRepository.save(new CaseMetadataEntity(newCaseUuid, expirationTime, withIndexation));
}

public List<CaseInfos> getCasesToReindex() {
Set<UUID> casesToReindex = caseMetadataRepository.findAllByIndexedTrue()
.stream()
.map(CaseMetadataEntity::getId)
.collect(Collectors.toSet());
return getCases(getStorageRootDir()).stream().filter(c -> casesToReindex.contains(c.getUuid())).toList();
}

CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) {
Expand All @@ -269,7 +277,7 @@ CaseInfos createInfos(String fileBaseName, UUID caseUuid, String format) {

@Transactional
public void disableCaseExpiration(UUID caseUuid) {
CaseMetadataEntity caseMetadataEntity = caseMetadataRepository.findById(caseUuid).orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "case " + caseUuid + " not found"));
CaseMetadataEntity caseMetadataEntity = getCaseMetaDataEntity(caseUuid);
caseMetadataEntity.setExpirationDate(null);
}

Expand Down Expand Up @@ -373,10 +381,6 @@ private void sendImportMessage(Message<String> message) {
caseInfosPublisher.send("publishCaseImport-out-0", message);
}

public void reindexAllCases() {
caseInfosService.recreateAllCaseInfos(getCases(getStorageRootDir()));
}

public List<CaseInfos> getMetadata(List<UUID> ids) {
List<CaseInfos> cases = new ArrayList<>();
ids.forEach(caseUuid -> {
Expand Down
82 changes: 82 additions & 0 deletions src/main/java/com/powsybl/caseserver/SupervisionController.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/**
* Copyright (c) 2024, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package com.powsybl.caseserver;

import com.powsybl.caseserver.elasticsearch.CaseInfosService;
import com.powsybl.caseserver.services.SupervisionService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

/**
* @author Jamal KHEYYAD <jamal.kheyyad at rte-international.com>
*/
@RestController
@RequestMapping(value = "/" + CaseConstants.API_VERSION + "/supervision")
@Tag(name = "case-server - Supervision")
public class SupervisionController {
private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionController.class);

private final SupervisionService supervisionService;
private final CaseService caseService;
private final ClientConfiguration elasticsearchClientConfiguration;
private final CaseInfosService caseInfosService;

public SupervisionController(SupervisionService supervisionService, CaseService caseService, ClientConfiguration elasticsearchClientConfiguration, CaseInfosService caseInfosService) {
this.supervisionService = supervisionService;
this.caseService = caseService;
this.elasticsearchClientConfiguration = elasticsearchClientConfiguration;
this.caseInfosService = caseInfosService;
}

@GetMapping(value = "/elasticsearch-host")
@Operation(summary = "get the elasticsearch address")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "the elasticsearch address")})
public ResponseEntity<String> getElasticsearchHost() {
String host = elasticsearchClientConfiguration.getEndpoints().get(0).getHostName()
+ ":"
+ elasticsearchClientConfiguration.getEndpoints().get(0).getPort();
return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(host);
}

@GetMapping(value = "/cases/index-name")
@Operation(summary = "get the indexed cases index name")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Indexed directory cases index name")})
public ResponseEntity<String> getIndexedCasesFIndexName() {
return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(caseInfosService.getDirectoryCasesIndexName());
}

@PostMapping(value = "/cases/reindex")
@Operation(summary = "reindex all cases")
public ResponseEntity<Void> reindexAllCases() {
LOGGER.debug("reindex all cases request received");
caseInfosService.recreateAllCaseInfos(caseService.getCasesToReindex());
return ResponseEntity.ok().build();
}

@DeleteMapping(value = "/cases/indexation")
@Operation(summary = "delete indexed cases")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all indexed cases have been deleted")})
public ResponseEntity<String> deleteIndexedCases() {
return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(Long.toString(supervisionService.deleteIndexedCases()));
}

@GetMapping(value = "/cases/indexation-count")
@Operation(summary = "get indexed cases count")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Indexed cases count")})
public ResponseEntity<String> getIndexedCasesCount() {
return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN).body(Long.toString(supervisionService.getIndexedCasesCount()));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
import com.google.common.collect.Lists;
import com.powsybl.caseserver.dto.CaseInfos;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
Expand Down Expand Up @@ -42,6 +44,10 @@ public class CaseInfosService {
@Autowired
private ElasticsearchOperations operations;

@Value(ESConfig.CASE_INFOS_INDEX_NAME)
@Getter
private String directoryCasesIndexName;

public CaseInfos addCaseInfos(@NonNull final CaseInfos ci) {
caseInfosRepository.save(ci);
return ci;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
@EnableElasticsearchRepositories
public class ESConfig extends ElasticsearchConfiguration {

public static final String CASE_INFOS_INDEX_NAME = "#{@environment.getProperty('powsybl-ws.elasticsearch.index.prefix')}cases";

@Value("#{'${spring.data.elasticsearch.embedded:false}' ? 'localhost' : '${spring.data.elasticsearch.host}'}")
private String esHost;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,7 @@ public class CaseMetadataEntity {

@Column(name = "expirationDate", columnDefinition = "timestamptz")
private Instant expirationDate;

@Column(name = "indexed", columnDefinition = "boolean default false", nullable = false)
private boolean indexed = false;
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

import java.util.List;
import java.util.UUID;

/**
* @author Abdelsalem Hedhili <abdelsalem.hedhili at rte-france.com>
*/
@Repository
public interface CaseMetadataRepository extends JpaRepository<CaseMetadataEntity, UUID> {

List<CaseMetadataEntity> findAllByIndexedTrue();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/**
* Copyright (c) 2024, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package com.powsybl.caseserver.services;

import com.powsybl.caseserver.elasticsearch.CaseInfosRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;

/**
* @author Jamal KHEYYAD <jamal.kheyyad at rte-international.com>
*/
@Service
public class SupervisionService {
private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionService.class);

private final CaseInfosRepository caseInfosRepository;

public SupervisionService(CaseInfosRepository caseInfosRepository) {
this.caseInfosRepository = caseInfosRepository;
}

public long deleteIndexedCases() {
AtomicReference<Long> startTime = new AtomicReference<>();
startTime.set(System.nanoTime());

long nbIndexesToDelete = getIndexedCasesCount();
caseInfosRepository.deleteAll();
LOGGER.trace("Indexed cases deletion : {} seconds", TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime.get()));
return nbIndexesToDelete;
}

public long getIndexedCasesCount() {
return caseInfosRepository.count();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext" xmlns:pro="http://www.liquibase.org/xml/ns/pro" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/pro http://www.liquibase.org/xml/ns/pro/liquibase-pro-latest.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
<changeSet author="jamalk (generated)" id="1722005243703-1">
<addColumn tableName="case_metadata">
<column defaultValueBoolean="false" name="indexed" type="boolean">
<constraints nullable="false"/>
</column>
</addColumn>
</changeSet>
</databaseChangeLog>
7 changes: 6 additions & 1 deletion src/main/resources/db/changelog/db.changelog-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,9 @@ databaseChangeLog:

- include:
file: changesets/changelog_20240620T100317Z.xml
relativeToChangelogFile: true
relativeToChangelogFile: true

- include:
file: changesets/changelog_20240726T144717Z.xml
relativeToChangelogFile: true

Loading

0 comments on commit e9614cf

Please sign in to comment.