diff --git a/pom.xml b/pom.xml index 48fd04b6f..8f0eec112 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ fr.insee.rmes Bauhaus-BO jar - 4.1.7-beta3 + 4.1.7-beta4 Bauhaus-Back-Office Back-office services for Bauhaus https://github.com/InseeFr/Bauhaus-Back-Office @@ -87,6 +87,7 @@ 1.17 12.4 8.5.11 + 1.20.4 @@ -100,6 +101,7 @@ org.springframework.boot spring-boot-starter-web + io.minio minio @@ -242,15 +244,30 @@ ${flexmark.version} + + org.testcontainers + minio + test + + org.testcontainers testcontainers + test org.testcontainers junit-jupiter + test + + + org.slf4j + slf4j-simple + test + + diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java b/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java index 8e43e57c0..a8769180e 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java @@ -27,7 +27,7 @@ public void delete(Path absolutePath) { } @Override - public InputStream read(String fileName) { + public InputStream readInDirectoryGestion(String fileName) { try { return Files.newInputStream(Paths.get(config.getDocumentsStorageGestion()).resolve(fileName)); } catch (IOException e) { @@ -36,12 +36,12 @@ public InputStream read(String fileName) { } @Override - public boolean existsInStorage(String filename) { + public boolean existsInStorageGestion(String filename) { return Files.exists(Paths.get(config.getDocumentsStorageGestion()).resolve(filename)); } @Override - public void write(InputStream content, Path destPath) { + public void writeToDirectoryGestion(InputStream content, Path destPath) { try { Files.copy(content, destPath, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { @@ -50,7 +50,7 @@ public void write(InputStream content, Path destPath) { } @Override - public void copy(String srcPath, String destPath) { + public void copyFromGestionToPublication(String srcPath, String destPath) { Path file = Paths.get(srcPath); Path targetPath = Paths.get(destPath); try { diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java b/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java index 77d138715..d067c126c 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java @@ -4,12 +4,14 @@ import java.nio.file.Path; public interface FilesOperations { - void delete(Path absolutePath); - InputStream read(String filename); - void write(InputStream content, Path destPath); - void copy(String srcPath, String destPath); + default void delete(Path absolutePath){ + throw new UnsupportedOperationException("Not implemented yet."); + } + InputStream readInDirectoryGestion(String filename); + void writeToDirectoryGestion(InputStream content, Path destPath); + void copyFromGestionToPublication(String srcPath, String destPath); boolean dirExists(Path gestionStorageFolder); - boolean existsInStorage(String filename); + boolean existsInStorageGestion(String filename); } diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java b/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java index 33074b365..577389403 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java @@ -19,11 +19,10 @@ public record MinioFilesOperation(MinioClient minioClient, String bucketName, St static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class); @Override - public InputStream read(String pathFile){ - String fileName= extractFileName(pathFile); - String objectName = directoryGestion + "/" + fileName; + public InputStream readInDirectoryGestion(String filename){ + String objectName = directoryGestion + "/" + filename; - logger.debug("Reading file with name {} from path {} as object {} in bucket {}", fileName, pathFile, objectName, bucketName); + logger.debug("Reading file with name {} from path {} as object {} in bucket {}", filename, filename, objectName, bucketName); try { return minioClient.getObject(GetObjectArgs.builder() @@ -31,7 +30,7 @@ public InputStream read(String pathFile){ .object(objectName) .build()); } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { - throw new RmesFileException(fileName, "Error reading file: " + fileName+" as object `"+objectName+"` in bucket "+bucketName, e); + throw new RmesFileException(filename, "Error reading file: " + filename+" as object `"+objectName+"` in bucket "+bucketName, e); } } private static String extractFileName(String filePath) { @@ -42,8 +41,8 @@ private static String extractFileName(String filePath) { } @Override - public boolean existsInStorage(String filename) { - var objectName = extractFileName(requireNonNull(filename)); + public boolean existsInStorageGestion(String filename) { + String objectName = directoryGestion + "/" + filename; logger.debug("Check existence of file with name {} as object {} in bucket {}", filename, objectName, bucketName); try { return minioClient.statObject(StatObjectArgs.builder() @@ -56,7 +55,7 @@ public boolean existsInStorage(String filename) { } @Override - public void write(InputStream content, Path filePath) { + public void writeToDirectoryGestion(InputStream content, Path filePath) { String filename = filePath.getFileName().toString(); String objectName = directoryGestion + "/" + filename; logger.debug("Writing to file with name {} from path {} as object {} in bucket {}", filename, filePath, objectName, bucketName); @@ -74,7 +73,7 @@ public void write(InputStream content, Path filePath) { } @Override - public void copy(String srcObjectName, String destObjectName) { + public void copyFromGestionToPublication(String srcObjectName, String destObjectName) { String srcObject = directoryGestion + "/" + extractFileName(srcObjectName); String destObject = directoryPublication + "/" + extractFileName(srcObjectName); diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java index 1486c7e10..83aec577f 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java @@ -44,7 +44,7 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException { JSONArray listDoc = docUtils.getListDocumentSims(idSims); Map mapIdUrls = new HashMap<>(); - listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), docUtils.getDocumentUrlFromDocument((JSONObject) doc))); + listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), DocumentsUtils.getDocumentUrlFromDocument((JSONObject) doc))); for (Map.Entry doc : mapIdUrls.entrySet()) { String docId = doc.getKey().toString(); @@ -68,10 +68,8 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException { } private void copyFileInPublicationFolders(String originalPath){ - String documentsStoragePublicationInterne = config.getDocumentsStoragePublicationInterne(); String documentsStoragePublicationExterne = config.getDocumentsStoragePublicationExterne(); - filesOperations.copy(originalPath, documentsStoragePublicationInterne); - filesOperations.copy(originalPath, documentsStoragePublicationExterne); + filesOperations.copyFromGestionToPublication(originalPath, documentsStoragePublicationExterne); } diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java index b43e92e07..8f2979606 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java @@ -496,7 +496,7 @@ private void uploadFile(InputStream documentFile, String documentName, String ur throw new RmesBadRequestException(ErrorCodes.DOCUMENT_CREATION_EXISTING_FILE, "There is already a document with that name.", documentName); } - filesOperations.write(documentFile, path); + filesOperations.writeToDirectoryGestion(documentFile, path); // don't throw an error if a file already exists under this name } @@ -662,7 +662,7 @@ protected String getDocumentFilename(String id) throws RmesException { public ResponseEntity downloadDocumentFile(String id) throws RmesException { String filePath = getDocumentFilename(id); - try (InputStream inputStream = filesOperations.read(filePath)) { // Lire via l'abstraction et utiliser try-with-resources + try (InputStream inputStream = filesOperations.readInDirectoryGestion(filePath)) { // Lire via l'abstraction et utiliser try-with-resources byte[] data = StreamUtils.copyToByteArray(inputStream); // Convertir InputStream en byte[] HttpHeaders headers = new HttpHeaders(); @@ -689,11 +689,11 @@ private String getFileName(String path) { } public InputStream retrieveDocumentFromStorage(String filename) { - return filesOperations.read(filename); + return filesOperations.readInDirectoryGestion(filename); } public boolean existsInStorage(String filename) { - return filesOperations.existsInStorage(filename); + return filesOperations.existsInStorageGestion(filename); } } diff --git a/src/main/java/fr/insee/rmes/config/PropertiesLogger.java b/src/main/java/fr/insee/rmes/config/PropertiesLogger.java index 93b717a56..6ee0ef291 100644 --- a/src/main/java/fr/insee/rmes/config/PropertiesLogger.java +++ b/src/main/java/fr/insee/rmes/config/PropertiesLogger.java @@ -21,7 +21,7 @@ public class PropertiesLogger implements ApplicationListener baseMotsCaches = Set.of("password", "pwd", "jeton", "token", "secret", "credential", "pw"); - private static final Set prefixesAffichesParDefaut= Set.of("fr.insee","logging","keycloak","spring","application","server","springdoc","management"); + private static final Set prefixesAffichesParDefaut= Set.of("fr.insee","logging","keycloak","spring","application","server","springdoc","management","minio"); private static final Set propertySourcesIgnoreesParDefaut = Set.of("systemProperties", "systemEnvironment"); private static final PropertySelectorEnum PROPERTY_SELECTOR_PAR_DEFAUT = PropertySelectorEnum.PREFIX; diff --git a/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java b/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java index 07cde8904..5db3be9e7 100644 --- a/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java +++ b/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java @@ -95,18 +95,18 @@ public void delete(Path absolutePath) { } @Override - public InputStream read(String path) { + public InputStream readInDirectoryGestion(String path) { throw new RmesFileException(nomFichier, "Error reading file: " + nomFichier+ " as object `"+objectName+"` in bucket "+bucketName, new MinioException()); } @Override - public void write(InputStream content, Path destPath) { + public void writeToDirectoryGestion(InputStream content, Path destPath) { } @Override - public void copy(String srcPath, String destPath) { + public void copyFromGestionToPublication(String srcPath, String destPath) { } @@ -116,7 +116,7 @@ public boolean dirExists(Path gestionStorageFolder) { } @Override - public boolean existsInStorage(String filename) { + public boolean existsInStorageGestion(String filename) { return false; } } diff --git a/src/test/java/fr/insee/rmes/testcontainers/minio/TestMinioFilesOperation.java b/src/test/java/fr/insee/rmes/testcontainers/minio/TestMinioFilesOperation.java new file mode 100644 index 000000000..dff9b1a27 --- /dev/null +++ b/src/test/java/fr/insee/rmes/testcontainers/minio/TestMinioFilesOperation.java @@ -0,0 +1,74 @@ +package fr.insee.rmes.testcontainers.minio; + +import fr.insee.rmes.bauhaus_services.MinioFilesOperation; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.StatObjectArgs; +import io.minio.errors.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MinIOContainer; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import static org.assertj.core.api.Assertions.assertThat; + +class TestMinioFilesOperation { + + MinIOContainer container = new MinIOContainer("minio/minio:RELEASE.2024-11-07T00-52-20Z"); + + @BeforeAll + public static void configureSlf4j() { + System.setProperty("org.slf4j.simpleLogger.log."+MinioFilesOperation.class.getName(), "debug"); + System.setProperty("slf4j.provider", "org.slf4j.simple.SimpleServiceProvider"); + } + + @Test + void testWritingThenCheckExistThenCopyThenRead_shouldBeOK() throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException { + container.start(); + var nomFichier = "test.txt"; + MinioClient minioClient = MinioClient + .builder() + .endpoint(container.getS3URL()) + .credentials(container.getUserName(), container.getPassword()) + .build(); + MinioFilesOperation minioFilesOperation = new MinioFilesOperation(minioClient,"metadata", "gestion", "publication"); + createBucket(minioFilesOperation.bucketName(), minioClient); + + Path absolutePathInGestion = Path.of("/mnt/applishare/rmes/data/storage/documents").resolve(nomFichier); + String contenuFichier = "Test"; + minioFilesOperation.writeToDirectoryGestion(new ByteArrayInputStream(contenuFichier.getBytes()), absolutePathInGestion); + assertThat(minioFilesOperation.dirExists(Path.of(minioFilesOperation.directoryGestion()))).isTrue(); + assertThat(minioFilesOperation.existsInStorageGestion(nomFichier)).isTrue(); + + minioFilesOperation.copyFromGestionToPublication(String.valueOf(absolutePathInGestion), "/mnt/applishare/rmes/data/storage/documents/tempPub1"); + assertThat(minioFilesOperation.dirExists(Path.of(minioFilesOperation.directoryPublication()))).isTrue(); + assertThat(fileExistsInPublication(minioClient, minioFilesOperation, nomFichier) + ).isTrue(); + + assertThat(new String(minioFilesOperation.readInDirectoryGestion(nomFichier).readAllBytes())).isEqualTo(contenuFichier); + } + + @AfterEach + void tearDown() { + container.stop(); + } + + private static boolean fileExistsInPublication(MinioClient minioClient, MinioFilesOperation minioFilesOperation, String nomFichier) throws ErrorResponseException, InsufficientDataException, InternalException, InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, XmlParserException { + return minioClient.statObject( + StatObjectArgs.builder() + .bucket(minioFilesOperation.bucketName()) + .object(minioFilesOperation.directoryPublication() + "/" + nomFichier).build() + ).size() > 0; + } + + private void createBucket(String bucketName, MinioClient minioClient) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); + } + +}