From 4ec05457bc85e9cfa279d9e7d2ca0b5bc4d9105c Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:08:05 +0000 Subject: [PATCH 01/71] Refactor FolderManager to allow custom date for archive folder --- .../data/managment/savers/FolderManager.java | 6 +++++- .../data/managment/savers/FolderManagerTest.java | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java index b100f9d0c..2894f90b5 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java @@ -18,9 +18,13 @@ static File createArchiveFolder() { } static File getOrCreateArchiveFolderWithCurrentDate() { + return getOrCreateArchiveFolderWithGivenDate(OffsetDateTime.now()); + } + + static File getOrCreateArchiveFolderWithGivenDate(OffsetDateTime date) { LOGGER.info(() -> "Getting archive folder for current date"); File archiveFolder = createArchiveFolder(); - String archiveFolderName = OffsetDateTime.now().toLocalDate().toString(); + String archiveFolderName = date.format(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE); File archiveFile = new File(archiveFolder, archiveFolderName); if (!archiveFile.exists()) { archiveFile.mkdir(); diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java index 025e13346..67d505d93 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java @@ -4,10 +4,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; +import java.time.OffsetDateTime; import org.junit.jupiter.api.Test; -public class FolderManagerTest { +class FolderManagerTest { @Test void testCreateArchiveFolder() { @@ -18,4 +19,13 @@ void testCreateArchiveFolder() { assertEquals("archive", archiveFolder.getName()); archiveFolder.delete(); } -} + + @Test + void testGetOrCreateArchiveFolderWithGivenDate() { + OffsetDateTime date = OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, OffsetDateTime.now().getOffset()); + + File actualArchiveFile = FolderManager.getOrCreateArchiveFolderWithGivenDate(date); + + assertEquals("2024-01-01", actualArchiveFile.getName()); + } +} \ No newline at end of file From 86caf484e98fb9d8d88c562bd91878399ebbc99c Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:48:14 +0000 Subject: [PATCH 02/71] Add test for FileManager's getOrCreateArchiveFile method --- .../data/managment/savers/FileManagerTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java new file mode 100644 index 000000000..8298cb642 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java @@ -0,0 +1,18 @@ +package io.github.mathieusoysal.data.managment.savers; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; + +import org.junit.jupiter.api.Test; + + +class FileManagerTest { + + @Test + void testGetOrCreateArchiveFile() { + // Assert that the file has the correct name + assertEquals(OffsetDateTime.now().format(DateTimeFormatter.ofPattern("HH")), ArchiveName.HOUR.getName()); + } +} \ No newline at end of file From a31e46c725d3f6635be6c898a9c41d83a779d344 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Thu, 11 Jan 2024 17:39:17 +0000 Subject: [PATCH 03/71] Add date formatting and save day sum up --- .../data/managment/savers/DataSaverTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java index 1a0c69855..f570a9e84 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.IOException; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.List; import org.junit.jupiter.api.AfterEach; @@ -24,7 +25,6 @@ void tearDown() { archiveFolder.delete(); } - @Test void testCreateArchiveLogements() throws ApiRequestFailedException, IOException { List logements = DataCollectorFromCrous.getAllLogementsWithoutConnection().stream().limit(2).toList(); @@ -34,8 +34,11 @@ void testCreateArchiveLogements() throws ApiRequestFailedException, IOException @Test void testCreateArchiveLogementsForDay() throws ApiRequestFailedException, IOException { - var dataCollector = new DataCollectorFromArchive("https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"); - assertDoesNotThrow(() -> dataCollector.getSumUpOfDay(LocalDate.of(2024, 1, 3))); + System.out.println(LocalDate.of(2024, 1, 11).format(DateTimeFormatter.ISO_LOCAL_DATE)); + var dataCollector = new DataCollectorFromArchive( + "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"); + String data = assertDoesNotThrow(() -> dataCollector.getSumUpOfDay(LocalDate.of(2024, 1, 10))); + assertDoesNotThrow(() -> DataSaver.save(ArchiveName.DAY_SUM_UP, data)); } } From 5777f78166df1e6d70fa7d330a30727c4ad14e45 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Thu, 11 Jan 2024 17:54:24 +0000 Subject: [PATCH 04/71] Add archiving workflow for data backup of day --- .github/workflows/archiving-day.yml | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/archiving-day.yml diff --git a/.github/workflows/archiving-day.yml b/.github/workflows/archiving-day.yml new file mode 100644 index 000000000..fd48ff137 --- /dev/null +++ b/.github/workflows/archiving-day.yml @@ -0,0 +1,39 @@ +name: Archiving data + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +jobs: + creating-archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + - uses: JamesIves/github-pages-deploy-action@v4.4.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + clean: false + branch: api + folder: archive + target-folder: v1/logements-crous/available + + + + + \ No newline at end of file From 73f0462117fc34a39b3e979d4bc052686342e63c Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 12 Jan 2024 09:24:38 +0000 Subject: [PATCH 05/71] Fix instantiation to use linkToData variable --- src/main/java/io/github/mathieusoysal/App.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index 933166f95..663e7fa03 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -28,7 +28,8 @@ public static void main(String[] args) } private static void createArchiveSumUpForThisDay() { - var dataCollector = new DataCollectorFromArchive(LINK_TO_DATA_PROPERTIE_NAME); + String linkToData = getPropertie(LINK_TO_DATA_PROPERTIE_NAME); + var dataCollector = new DataCollectorFromArchive(linkToData); var sumUpOfTheDay = dataCollector.getSumUpOfDay(LocalDate.now()); DataSaver.save(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); } From 7a4621ecc63f8942a1390a5bd206d9e38220e3af Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 13 Jan 2024 22:56:02 +0000 Subject: [PATCH 06/71] Add logements classification feature --- .../java/io/github/mathieusoysal/App.java | 21 +++++--- .../java/io/github/mathieusoysal/Utils.java | 10 ++++ .../collectors/DataCollectorFromArchive.java | 6 +++ .../collectors/RequestorToGetSumUpOfDay.java | 7 ++- .../mathieusoysal/logement/Logement.java | 54 +++++++++++++++---- .../logement/LogementsClassifier.java | 36 +++++++++++++ 6 files changed, 115 insertions(+), 19 deletions(-) create mode 100644 src/main/java/io/github/mathieusoysal/Utils.java create mode 100644 src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index 663e7fa03..5a3785244 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -2,7 +2,9 @@ import java.io.IOException; import java.time.LocalDate; +import java.util.stream.IntStream; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.forax.beautifullogger.Logger; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; @@ -10,6 +12,8 @@ import io.github.mathieusoysal.data.managment.savers.ArchiveName; import io.github.mathieusoysal.data.managment.savers.DataSaver; import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; +import io.github.mathieusoysal.logement.Logement; +import io.github.mathieusoysal.logement.LogementsClassifier; public class App { private static final Logger LOGGER = Logger.getLogger(); @@ -17,14 +21,15 @@ public class App { private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD"; private static final String LINK_TO_DATA_PROPERTIE_NAME = "LINK_TO_DATA"; - public static void main(String[] args) - throws IOException { - LOGGER.info(() -> "Starting application"); - if (sumupdayModIsActivated()) - createArchiveSumUpForThisDay(); - else - createArchiveForThisHour(); - LOGGER.info(() -> "Application finished"); + public static void main(String[] args) { + String linkToData = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"; + var dataCollector = new DataCollectorFromArchive(linkToData); + var collectors = new LogementsClassifier(); + IntStream.range(1, 12) + .mapToObj(i -> dataCollector.getSumUpConvertedOfDay(LocalDate.of(2024, 1, i))) + .forEach(collectors::addLogements); + var result = collectors.classify(); + createArchiveSumUpForThisDay(); } private static void createArchiveSumUpForThisDay() { diff --git a/src/main/java/io/github/mathieusoysal/Utils.java b/src/main/java/io/github/mathieusoysal/Utils.java new file mode 100644 index 000000000..4fb73b1cb --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/Utils.java @@ -0,0 +1,10 @@ +package io.github.mathieusoysal; + +import java.util.HashSet; +import java.util.List; + +public class Utils { + public static boolean listEqualsIgnoreOrder(List list1, List list2) { + return new HashSet<>(list1).equals(new HashSet<>(list2)); + } +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java index a9e5124ec..2495f173b 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java @@ -2,6 +2,8 @@ import java.time.LocalDate; +import io.github.mathieusoysal.logement.Logement; + public class DataCollectorFromArchive { private final String archiveUrl; @@ -12,5 +14,9 @@ public DataCollectorFromArchive(String archiveUrl) { public String getSumUpOfDay(LocalDate day) { return new RequestorToGetSumUpOfDay(day).requestWitGet(archiveUrl); } + + public Logement[][] getSumUpConvertedOfDay(LocalDate day) { + return new RequestorToGetSumUpOfDay(day).getSumUpOfDay(archiveUrl); + } } diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java index 669e81257..82c40455f 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java @@ -30,6 +30,11 @@ public RequestorToGetSumUpOfDay(String date) { @Override public String requestWitGet(String url) { + var sumUp = getSumUpOfDay(url); + return Convertor.convertLogementMatrixToJson(sumUp); + } + + public Logement[][] getSumUpOfDay(String url) { LOGGER.info(() -> "Creating sum up of the day: " + date); String linkToDataForTheDay = url + "/" + date; Logement[][] sumUp; @@ -44,7 +49,7 @@ public String requestWitGet(String url) { LOGGER.info(() -> "Logements received"); } LOGGER.info(() -> "profil closed"); - return Convertor.convertLogementMatrixToJson(sumUp); + return sumUp; } private static Logement[] getFromUrl(String url, APIRequestContext context) { diff --git a/src/main/java/io/github/mathieusoysal/logement/Logement.java b/src/main/java/io/github/mathieusoysal/logement/Logement.java index 8566558fd..9c3f0417d 100644 --- a/src/main/java/io/github/mathieusoysal/logement/Logement.java +++ b/src/main/java/io/github/mathieusoysal/logement/Logement.java @@ -2,6 +2,8 @@ import java.util.List; +import io.github.mathieusoysal.Utils; + /** * Logement * @@ -29,10 +31,9 @@ public class Logement { private List occupationMods; private List transports; - // Create constructor - public Logement() - {} + public Logement() { + } /** * Constructs a new Logement object with the specified ID. @@ -56,12 +57,6 @@ public int hashCode() { return result; } - /** - * Indicates whether some other object is "equal to" this one. - * - * @param obj the reference object with which to compare - * @return true if this object is the same as the obj argument; false otherwise - */ @Override public boolean equals(Object obj) { if (this == obj) @@ -69,7 +64,46 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; Logement other = (Logement) obj; - return id == other.id; + if (address == null) { + if (other.address != null) + return false; + } else if (!address.equals(other.address)) + return false; + if (areaMax != other.areaMax) + return false; + if (areaMin != other.areaMin) + return false; + if (available != other.available) + return false; + if (bedCount != other.bedCount) + return false; + if (bedKind != other.bedKind) + return false; + if (bedroomCount != other.bedroomCount) + return false; + if (description == null) { + if (other.description != null) + return false; + } else if (!description.equals(other.description)) + return false; + if (highDemand != other.highDemand) + return false; + if (id != other.id) + return false; + if (inUnavailabilityPeriod != other.inUnavailabilityPeriod) + return false; + if (label == null) { + if (other.label != null) + return false; + } else if (!label.equals(other.label)) + return false; + if (lowStock != other.lowStock) + return false; + if (!Utils.listEqualsIgnoreOrder(occupationMods, other.occupationMods)) + return false; + if (roomCount != other.roomCount) + return false; + return Utils.listEqualsIgnoreOrder(transports, other.transports); } /** diff --git a/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java b/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java new file mode 100644 index 000000000..a6dad2afe --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java @@ -0,0 +1,36 @@ +package io.github.mathieusoysal.logement; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class LogementsClassifier { + + private List logements; + + public LogementsClassifier() { + this.logements = new ArrayList<>(); + } + + public void addLogement(Logement logement) { + this.logements.add(logement); + } + + public void addLogements(Logement[][] logements) { + for (Logement[] logements1 : logements) + for (Logement logement : logements1) + this.logements.add(logement); + } + + public Map> classify() { + var mapped = logements.stream().collect( + Collectors.groupingBy( + Logement::getId, + Collectors.toList())); + var result = new HashMap>(); + mapped.forEach((key, value) -> result.put(key, value.stream().unordered().distinct().toList())); + return result; + } +} \ No newline at end of file From a619baad7ef4a769f725014f279068b9f574a29b Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 14 Jan 2024 22:57:30 +0000 Subject: [PATCH 07/71] Add new ArchiveName enum value and update LogementsClassifier --- .../java/io/github/mathieusoysal/App.java | 5 +- .../data/managment/savers/ArchiveName.java | 1 + .../mathieusoysal/logement/Logement.java | 84 ++++++++----------- .../logement/LogementsClassifier.java | 39 +++++---- .../logement/OccupationMode.java | 22 +++++ 5 files changed, 85 insertions(+), 66 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index 5a3785244..c613f1724 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -4,7 +4,6 @@ import java.time.LocalDate; import java.util.stream.IntStream; -import com.fasterxml.jackson.databind.ObjectMapper; import com.github.forax.beautifullogger.Logger; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; @@ -12,7 +11,6 @@ import io.github.mathieusoysal.data.managment.savers.ArchiveName; import io.github.mathieusoysal.data.managment.savers.DataSaver; import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; -import io.github.mathieusoysal.logement.Logement; import io.github.mathieusoysal.logement.LogementsClassifier; public class App { @@ -28,8 +26,7 @@ public static void main(String[] args) { IntStream.range(1, 12) .mapToObj(i -> dataCollector.getSumUpConvertedOfDay(LocalDate.of(2024, 1, i))) .forEach(collectors::addLogements); - var result = collectors.classify(); - createArchiveSumUpForThisDay(); + DataSaver.save(ArchiveName.ALL_LOGEMENTS, collectors.getLogements()); } private static void createArchiveSumUpForThisDay() { diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java index 5f5e2e379..45813150d 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java @@ -5,6 +5,7 @@ import java.util.function.Supplier; public enum ArchiveName { + ALL_LOGEMENTS(() -> "all_logements"), DAY_SUM_UP(() -> "sum-up"), HOUR(() -> OffsetDateTime.now().toLocalTime().format(DateTimeFormatter.ofPattern("HH"))); diff --git a/src/main/java/io/github/mathieusoysal/logement/Logement.java b/src/main/java/io/github/mathieusoysal/logement/Logement.java index 9c3f0417d..69bfb1fb5 100644 --- a/src/main/java/io/github/mathieusoysal/logement/Logement.java +++ b/src/main/java/io/github/mathieusoysal/logement/Logement.java @@ -1,8 +1,9 @@ package io.github.mathieusoysal.logement; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; - -import io.github.mathieusoysal.Utils; +import java.util.Set; /** * Logement @@ -28,7 +29,7 @@ public class Logement { private List equipements; private double areaMin; private double areaMax; - private List occupationMods; + private Set occupationMods; private List transports; // Create constructor @@ -59,51 +60,38 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - Logement other = (Logement) obj; - if (address == null) { - if (other.address != null) - return false; - } else if (!address.equals(other.address)) - return false; - if (areaMax != other.areaMax) - return false; - if (areaMin != other.areaMin) - return false; - if (available != other.available) - return false; - if (bedCount != other.bedCount) - return false; - if (bedKind != other.bedKind) - return false; - if (bedroomCount != other.bedroomCount) - return false; - if (description == null) { - if (other.description != null) - return false; - } else if (!description.equals(other.description)) - return false; - if (highDemand != other.highDemand) + if (obj == null) return false; - if (id != other.id) + if (!(obj instanceof Logement)) return false; - if (inUnavailabilityPeriod != other.inUnavailabilityPeriod) - return false; - if (label == null) { - if (other.label != null) - return false; - } else if (!label.equals(other.label)) - return false; - if (lowStock != other.lowStock) - return false; - if (!Utils.listEqualsIgnoreOrder(occupationMods, other.occupationMods)) - return false; - if (roomCount != other.roomCount) - return false; - return Utils.listEqualsIgnoreOrder(transports, other.transports); + Logement other = (Logement) obj; + return id == other.id; + } + + /** + * Adds the specified Logement object to this Logement object. + * + * @param logement the Logement object to be added + */ + public void addLogement(Logement logement) { + if (logement == null) + return; + label = logement.getLabel(); + address = logement.getAddress(); + bedCount = logement.getBedCount(); + bedKind = logement.getBedKind(); + bedroomCount = logement.getBedroomCount(); + roomCount = logement.getRoomCount(); + inUnavailabilityPeriod = logement.isInUnavailabilityPeriod(); + description = logement.getDescription(); + available = logement.isAvailable(); + highDemand = logement.isHighDemand(); + lowStock = logement.isLowStock(); + equipements = logement.getEquipements(); + areaMin = logement.getAreaMin(); + areaMax = logement.getAreaMax(); + occupationMods.addAll(logement.occupationMods); + transports = logement.getTransports(); } /** @@ -248,7 +236,7 @@ public double getAreaMax() { * @return the list of occupation modifications for this Logement object */ public List getOccupationMods() { - return occupationMods; + return new ArrayList<>(occupationMods); } /** @@ -331,7 +319,7 @@ public void setAreaMax(double areaMax) { } public void setOccupationMods(List occupationMods) { - this.occupationMods = occupationMods; + this.occupationMods = new HashSet<>(occupationMods); } public void setTransports(List transports) { diff --git a/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java b/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java index a6dad2afe..50cbeb489 100644 --- a/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java +++ b/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java @@ -1,36 +1,47 @@ package io.github.mathieusoysal.logement; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class LogementsClassifier { - private List logements; + private Map logements; + + public LogementsClassifier(Set logements) { + this.logements = logements + .stream() + .collect(Collectors.toMap(Logement::getId, logement -> logement)); + } public LogementsClassifier() { - this.logements = new ArrayList<>(); + this.logements = new HashMap<>(); } public void addLogement(Logement logement) { - this.logements.add(logement); + if (!logements.containsKey(logement.getId())) + logements.put(logement.getId(), logement); + else { + Logement existingLogement = logements.get(logement.getId()); + existingLogement.addLogement(logement); + } + } + + public void addLogements(Collection logements) { + logements.forEach(this::addLogement); } public void addLogements(Logement[][] logements) { - for (Logement[] logements1 : logements) - for (Logement logement : logements1) - this.logements.add(logement); + for (Logement[] logements2 : logements) + for (Logement logement : logements2) + addLogement(logement); } - public Map> classify() { - var mapped = logements.stream().collect( - Collectors.groupingBy( - Logement::getId, - Collectors.toList())); - var result = new HashMap>(); - mapped.forEach((key, value) -> result.put(key, value.stream().unordered().distinct().toList())); - return result; + public List getLogements() { + return new ArrayList<>(logements.values()); } } \ No newline at end of file diff --git a/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java b/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java index 77ec30c70..345db7e49 100644 --- a/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java +++ b/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java @@ -62,4 +62,26 @@ public int getRentMin() { public int getRentMax() { return rentMax; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((occupationKind == null) ? 0 : occupationKind.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + OccupationMode other = (OccupationMode) obj; + return occupationKind == other.occupationKind; + } + + } From 935e63b5ee315e30da9bc7e50e71b98c3d75725c Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:36:25 +0000 Subject: [PATCH 08/71] Add method to get all logements --- .../collectors/DataCollectorFromArchive.java | 9 +++++- .../RequestorToGetAllLogements.java | 31 +++++++++++++++++++ .../data/managment/savers/FileManager.java | 10 ++++-- .../data/managment/savers/FolderManager.java | 4 +-- .../collectors/DataCollectorTest.java | 7 +++++ .../managment/savers/FolderManagerTest.java | 2 +- 6 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java index 2495f173b..abdd189c6 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java @@ -1,7 +1,9 @@ package io.github.mathieusoysal.data.managment.collectors; import java.time.LocalDate; +import java.util.List; +import io.github.mathieusoysal.data.managment.convertors.Convertor; import io.github.mathieusoysal.logement.Logement; public class DataCollectorFromArchive { @@ -11,11 +13,16 @@ public DataCollectorFromArchive(String archiveUrl) { this.archiveUrl = archiveUrl; } + public List getAllLogements() { + String json = new RequestorToGetAllLogements().requestWitGet(archiveUrl); + return Convertor.convertJsonToListOfLogements(json); + } + public String getSumUpOfDay(LocalDate day) { return new RequestorToGetSumUpOfDay(day).requestWitGet(archiveUrl); } - public Logement[][] getSumUpConvertedOfDay(LocalDate day) { + public Logement[][] getConvertedSumUpOfDay(LocalDate day) { return new RequestorToGetSumUpOfDay(day).getSumUpOfDay(archiveUrl); } diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java new file mode 100644 index 000000000..ac561402f --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java @@ -0,0 +1,31 @@ +package io.github.mathieusoysal.data.managment.collectors; + +import com.github.forax.beautifullogger.Logger; +import com.microsoft.playwright.Playwright; + +import io.github.mathieusoysal.exceptions.ApiRequestFailedRuntimeException; + +class RequestorToGetAllLogements implements Requestor { + private static final String ALL_LOGEMENTS = "/all_logements"; + private static final Logger LOGGER = Logger.getLogger(); + + @Override + public String requestWitGet(String url) { + LOGGER.info(() -> "Getting all logements from archive"); + String linkToDataForTheDay = url + ALL_LOGEMENTS; + String result; + LOGGER.info(() -> "Creating profil to request logements"); + try (Playwright playwright = Playwright.create()) { + LOGGER.info(() -> "profil created"); + var context = playwright.request().newContext(); + var respons = context.get(linkToDataForTheDay); + if (!respons.ok()) + throw new ApiRequestFailedRuntimeException(respons); + result = respons.text(); + LOGGER.info(() -> "Logements received"); + } + LOGGER.info(() -> "profil closed"); + return result; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java index d61a4c61a..01157cad6 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java @@ -11,10 +11,14 @@ class FileManager { private static final Logger LOGGER = Logger.getLogger(); - static File getOrCreateArchiveFile(ArchiveName archiveName) { + static File getOrCreateArchiveFileForCurrentDay(ArchiveName archiveName) { var archiveFolder = FolderManager.getOrCreateArchiveFolderWithCurrentDate(); - return getArchiveFile(archiveFolder, - archiveName.getName()); + return getArchiveFile(archiveFolder, archiveName.getName()); + } + + static File getOrCreateArchiveFileForGlobal(ArchiveName archiveName) { + var archiveFolder = FolderManager.getOrCreateArchiveFolder(); + return getArchiveFile(archiveFolder, archiveName.getName()); } private static File getArchiveFile(File archiveFolder, String archiveFileName) throws DateTimeException { diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java index 2894f90b5..8e86e942f 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java @@ -7,7 +7,7 @@ class FolderManager { private static final java.util.logging.Logger LOGGER = java.util.logging.Logger .getLogger(FolderManager.class.getName()); - static File createArchiveFolder() { + static File getOrCreateArchiveFolder() { LOGGER.info(() -> "getting archive folder"); File archiveFolder = new File("archive"); if (!archiveFolder.exists()) { @@ -23,7 +23,7 @@ static File getOrCreateArchiveFolderWithCurrentDate() { static File getOrCreateArchiveFolderWithGivenDate(OffsetDateTime date) { LOGGER.info(() -> "Getting archive folder for current date"); - File archiveFolder = createArchiveFolder(); + File archiveFolder = getOrCreateArchiveFolder(); String archiveFolderName = date.format(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE); File archiveFile = new File(archiveFolder, archiveFolderName); if (!archiveFile.exists()) { diff --git a/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java b/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java index ae2a7292d..2b1649284 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java @@ -93,4 +93,11 @@ void testCreateSumUpOfTheDay() throws JsonProcessingException { dataCollectorFromArchive.getSumUpOfDay(date);}); } + @Test + void testGetAllLogements() + { + String linkToData = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available"; + assertDoesNotThrow(() -> new DataCollectorFromArchive(linkToData).getAllLogements()); + } + } \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java index 67d505d93..63768d72a 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java @@ -12,7 +12,7 @@ class FolderManagerTest { @Test void testCreateArchiveFolder() { - File archiveFolder = FolderManager.createArchiveFolder(); + File archiveFolder = FolderManager.getOrCreateArchiveFolder(); assertTrue(archiveFolder.exists()); assertTrue(archiveFolder.isDirectory()); From 83806016ec82f36d3c4ae62953a242c2533b092a Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 21 Jan 2024 13:28:43 +0000 Subject: [PATCH 09/71] Add new archiver classes and properties --- .../java/io/github/mathieusoysal/App.java | 60 ++----------------- .../io/github/mathieusoysal/ArchiveMod.java | 38 ++++++++++++ .../io/github/mathieusoysal/Properties.java | 42 +++++++++++++ .../archivers/ArchiveAllLogements.java | 19 ++++++ .../data/managment/archivers/ArchiveDay.java | 17 ++++++ .../data/managment/archivers/ArchiveHour.java | 17 ++++++ .../data/managment/archivers/Archiver.java | 26 ++++++++ .../collectors/DataCollectorFromCrous.java | 10 +--- .../data/managment/savers/DataSaver.java | 15 +++-- .../logement/pojo/Convertor.java | 40 +++++++++---- .../data/managment/savers/DataSaverTest.java | 4 +- 11 files changed, 207 insertions(+), 81 deletions(-) create mode 100644 src/main/java/io/github/mathieusoysal/ArchiveMod.java create mode 100644 src/main/java/io/github/mathieusoysal/Properties.java create mode 100644 src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java create mode 100644 src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java create mode 100644 src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java create mode 100644 src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index c613f1724..5a2678a89 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -1,67 +1,15 @@ package io.github.mathieusoysal; -import java.io.IOException; -import java.time.LocalDate; -import java.util.stream.IntStream; - import com.github.forax.beautifullogger.Logger; -import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; -import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; -import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.DataSaver; -import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; -import io.github.mathieusoysal.logement.LogementsClassifier; - public class App { private static final Logger LOGGER = Logger.getLogger(); - private static final String MAIL_PROPERTIES_NAME = "MAIL"; - private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD"; - private static final String LINK_TO_DATA_PROPERTIE_NAME = "LINK_TO_DATA"; public static void main(String[] args) { - String linkToData = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"; - var dataCollector = new DataCollectorFromArchive(linkToData); - var collectors = new LogementsClassifier(); - IntStream.range(1, 12) - .mapToObj(i -> dataCollector.getSumUpConvertedOfDay(LocalDate.of(2024, 1, i))) - .forEach(collectors::addLogements); - DataSaver.save(ArchiveName.ALL_LOGEMENTS, collectors.getLogements()); - } - - private static void createArchiveSumUpForThisDay() { - String linkToData = getPropertie(LINK_TO_DATA_PROPERTIE_NAME); - var dataCollector = new DataCollectorFromArchive(linkToData); - var sumUpOfTheDay = dataCollector.getSumUpOfDay(LocalDate.now()); - DataSaver.save(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); - } - - private static boolean sumupdayModIsActivated() { - return System.getenv(LINK_TO_DATA_PROPERTIE_NAME) != null; - } - - private static void createArchiveForThisHour() - throws IOException { - var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(getEmail(), getPassword()); - DataSaver.save(ArchiveName.HOUR, logements); - } - - private static String getPropertie(final String propertieName) { - LOGGER.info(() -> "Getting " + propertieName + " from environment variables"); - String propertie = System.getenv(propertieName); - if (propertie == null) { - LOGGER.error(() -> propertieName + " not found in environment variables"); - throw new PropertiesNotFoundRuntimeException(propertieName); - } - return propertie; - } - - private static String getEmail() { - return getPropertie(MAIL_PROPERTIES_NAME); - } - - private static String getPassword() { - return getPropertie(PASSWORD_PROPERTIES_NAME); + LOGGER.info(() -> "Starting application"); + ArchiveMod archiveMod = ArchiveMod.getArchiveModFromEnvironmentVariables(); + archiveMod.archive(); + LOGGER.info(() -> "Application successfully finished"); } } diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMod.java b/src/main/java/io/github/mathieusoysal/ArchiveMod.java new file mode 100644 index 000000000..6a922ffbe --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/ArchiveMod.java @@ -0,0 +1,38 @@ +package io.github.mathieusoysal; + +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.data.managment.archivers.ArchiveAllLogements; +import io.github.mathieusoysal.data.managment.archivers.ArchiveDay; +import io.github.mathieusoysal.data.managment.archivers.ArchiveHour; +import io.github.mathieusoysal.data.managment.archivers.Archiver; + +public enum ArchiveMod { + DAY_SUM_UP(ArchiveDay::new), + ALL_LOGEMENTS(ArchiveAllLogements::new), + HOUR(ArchiveHour::new); + + private static final Logger LOGGER = Logger.getLogger(); + private final Archiver archiver; + + ArchiveMod(Archiver archiver) { + this.archiver = archiver; + } + + public void archive() { + archiver.archive(); + } + + public static ArchiveMod getArchiveMod(String archiveMod) { + try { + return ArchiveMod.valueOf(archiveMod); + } catch (IllegalArgumentException e) { + LOGGER.warning(() -> archiveMod + " is not a valid archive mod"); + return HOUR; + } + } + + public static ArchiveMod getArchiveModFromEnvironmentVariables() { + return getArchiveMod(Properties.ARCHIVE_MOD.getValue()); + } +} diff --git a/src/main/java/io/github/mathieusoysal/Properties.java b/src/main/java/io/github/mathieusoysal/Properties.java new file mode 100644 index 000000000..c91595795 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/Properties.java @@ -0,0 +1,42 @@ +package io.github.mathieusoysal; + +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; + +public enum Properties { + LINK_TO_ARCHIVE("LINK_TO_ARCHIVE"), + MAIL("MAIL"), + PASSWORD("PASSWORD"), + ARCHIVE_MOD("ARCHIVE_MOD"), + SPECIFIC_DAY("DAY_TO_SUM_UP"); + + private static final Logger LOGGER = Logger.getLogger(); + private final String name; + + Properties(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public boolean isPresent() { + return System.getenv(name) != null; + } + + public String getValue() { + LOGGER.info(() -> "Getting " + name + " from environment variables"); + String propertie = System.getenv(name); + if (propertie == null) { + LOGGER.error(() -> name + " not found in environment variables"); + throw new PropertiesNotFoundRuntimeException(name); + } + return propertie; + } + + public static boolean modToArchiveIsActivated() { + return System.getenv(ARCHIVE_MOD.getName()) != null; + } +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java b/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java new file mode 100644 index 000000000..7412356d3 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java @@ -0,0 +1,19 @@ +package io.github.mathieusoysal.data.managment.archivers; + +import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; +import io.github.mathieusoysal.data.managment.savers.ArchiveName; +import io.github.mathieusoysal.data.managment.savers.DataSaver; +import io.github.mathieusoysal.logement.LogementsClassifier; + +public class ArchiveAllLogements implements Archiver { + + @Override + public void archive() { + var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); + var logements = new LogementsClassifier(); + logements.addLogements(dataCollector.getAllLogements()); + logements.addLogements(dataCollector.getConvertedSumUpOfDay(Archiver.getDayToArchive())); + DataSaver.saveForGlobalArchive(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java b/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java new file mode 100644 index 000000000..a2bdd03cd --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java @@ -0,0 +1,17 @@ +package io.github.mathieusoysal.data.managment.archivers; + +import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; +import io.github.mathieusoysal.data.managment.savers.ArchiveName; +import io.github.mathieusoysal.data.managment.savers.DataSaver; + +public class ArchiveDay implements Archiver { + + @Override + public void archive() { + var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); + var sumUpOfTheDay = dataCollector.getSumUpOfDay(Archiver.getDayToArchive()); + DataSaver.saveForCurrentDay(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); + } + + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java b/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java new file mode 100644 index 000000000..0b59da5d4 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java @@ -0,0 +1,17 @@ +package io.github.mathieusoysal.data.managment.archivers; + +import io.github.mathieusoysal.Properties; +import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; +import io.github.mathieusoysal.data.managment.savers.ArchiveName; +import io.github.mathieusoysal.data.managment.savers.DataSaver; + +public class ArchiveHour implements Archiver { + + @Override + public void archive() { + var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(Properties.MAIL.getValue(), + Properties.PASSWORD.getValue()); + DataSaver.saveForCurrentDay(ArchiveName.HOUR, logements); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java new file mode 100644 index 000000000..b05230e21 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java @@ -0,0 +1,26 @@ +package io.github.mathieusoysal.data.managment.archivers; + +import java.time.LocalDate; + +import io.github.mathieusoysal.Properties; + +@FunctionalInterface +public interface Archiver { + public static final String DEFAULT_LINK_TO_ARCHIVE = "https://raw.githubusercontent.com/mathieusoysal/LogementsCrous/main/data/data.csv"; + + static String getLinkToArchive() { + if (Properties.LINK_TO_ARCHIVE.isPresent()) + return Properties.LINK_TO_ARCHIVE.getValue(); + else + return DEFAULT_LINK_TO_ARCHIVE; + } + + static LocalDate getDayToArchive() { + if(Properties.SPECIFIC_DAY.isPresent()) + return LocalDate.parse(Properties.SPECIFIC_DAY.getValue()); + else + return LocalDate.now(); + } + + void archive(); +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java index ddb7ac3bb..1e8cdeb73 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java @@ -1,6 +1,5 @@ package io.github.mathieusoysal.data.managment.collectors; -import java.io.IOException; import java.util.List; import com.github.forax.beautifullogger.Logger; @@ -12,23 +11,20 @@ public class DataCollectorFromCrous { private static final Logger LOGGER = Logger.getLogger(); private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32"; - public static List getAvailableLogementsWithoutConnection() - throws IOException { + public static List getAvailableLogementsWithoutConnection() { Requestor requestor = new RequestorWithoutConnection(); String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS); return Convertor.getLogementsFromBruteJsonString(jsonLogements); } - public static List getAllLogementsWithoutConnection() - throws IOException { + public static List getAllLogementsWithoutConnection() { LOGGER.info(() -> "Getting all logements"); Requestor requestor = new RequestorWithoutConnection(); String jsonLogements = requestor.requestWitGet("https://trouverunlogement.lescrous.fr/api/fr/search/29"); return Convertor.getLogementsFromBruteJsonString(jsonLogements); } - public static List getAvailableLogementsWithConnection(String email, String password) - throws IOException { + public static List getAvailableLogementsWithConnection(String email, String password) { Requestor requestor = new RequestorWithConnection(email, password); String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS); return Convertor.getLogementsFromBruteJsonString(jsonLogements); diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java index 27e416bc4..a0db38715 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java @@ -14,15 +14,22 @@ public class DataSaver { private static final Logger LOGGER = Logger.getLogger(); - public static File save(ArchiveName archiveName, String content) { - File archiveFile = FileManager.getOrCreateArchiveFile(archiveName); + public static File saveForCurrentDay(ArchiveName archiveName, String content) { + File archiveFile = FileManager.getOrCreateArchiveFileForCurrentDay(archiveName); writeDataInsideArchiveFile(content, archiveFile); return archiveFile; } - public static File save(ArchiveName archiveName, List logements) { + public static File saveForCurrentDay(ArchiveName archiveName, List logements) { String logementsJson = Convertor.convertLogementsToJson(logements); - return save(archiveName, logementsJson); + return saveForCurrentDay(archiveName, logementsJson); + } + + public static File saveForGlobalArchive(ArchiveName archiveName, List logements) { + File archiveFile = FileManager.getOrCreateArchiveFileForGlobal(archiveName); + String logementsJson = Convertor.convertLogementsToJson(logements); + writeDataInsideArchiveFile(logementsJson, archiveFile); + return archiveFile; } private static void writeDataInsideArchiveFile(String logementsJson, File archiveFile) { diff --git a/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java b/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java index a9ed3d126..7cea44fbe 100644 --- a/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java +++ b/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java @@ -21,18 +21,30 @@ public class Convertor { private Convertor() { } - static List getItemsFromJsonFile(File file) throws IOException { - LOGGER.info(() -> "Reading json file for convertion to java object"); - ObjectMapper objectMapper = new ObjectMapper(); - Input results = objectMapper.readValue(file, Input.class); + static List getItemsFromJsonFile(File file) { + Input results; + try { + LOGGER.info(() -> "Reading json file for convertion to java object"); + ObjectMapper objectMapper = new ObjectMapper(); + results = objectMapper.readValue(file, Input.class); + } catch (IOException e) { + LOGGER.error(() -> "Error while reading json file"); + throw new ConvertorException("Error while reading json file", e); + } LOGGER.info(() -> "Json file converted to java object"); return results.getResults().getItems(); } - static List getItemsFromJsonString(String json) throws IOException { - LOGGER.info(() -> "Reading json string for convertion to java object"); - ObjectMapper objectMapper = new ObjectMapper(); - Input results = objectMapper.readValue(json, Input.class); + static List getItemsFromJsonString(String json) { + Input results; + try { + LOGGER.info(() -> "Reading json string for convertion to java object"); + ObjectMapper objectMapper = new ObjectMapper(); + results = objectMapper.readValue(json, Input.class); + } catch (IOException e) { + LOGGER.error(() -> "Error while reading json file"); + throw new ConvertorException("Error while reading json file", e); + } LOGGER.info(() -> "Json string converted to java object"); return results.getResults().getItems(); } @@ -44,14 +56,12 @@ static List convertItemsToLogements(List items) { return result; } - public static List getLogementsFromBruteJsonFile(File file) - throws IOException { + public static List getLogementsFromBruteJsonFile(File file) { List items = getItemsFromJsonFile(file); return convertItemsToLogements(items); } - public static List getLogementsFromBruteJsonString(String json) - throws IOException { + public static List getLogementsFromBruteJsonString(String json) { List items = getItemsFromJsonString(json); return convertItemsToLogements(items); } @@ -107,4 +117,10 @@ private static List getTransports(It .toList(); } + static class ConvertorException extends RuntimeException { + public ConvertorException(String message, Throwable cause) { + super(message, cause); + } + } + } diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java index f570a9e84..f43c643d0 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java @@ -28,7 +28,7 @@ void tearDown() { @Test void testCreateArchiveLogements() throws ApiRequestFailedException, IOException { List logements = DataCollectorFromCrous.getAllLogementsWithoutConnection().stream().limit(2).toList(); - var file = assertDoesNotThrow(() -> DataSaver.save(ArchiveName.HOUR, logements)); + var file = assertDoesNotThrow(() -> DataSaver.saveForCurrentDay(ArchiveName.HOUR, logements)); file.delete(); } @@ -38,7 +38,7 @@ void testCreateArchiveLogementsForDay() throws ApiRequestFailedException, IOExce var dataCollector = new DataCollectorFromArchive( "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"); String data = assertDoesNotThrow(() -> dataCollector.getSumUpOfDay(LocalDate.of(2024, 1, 10))); - assertDoesNotThrow(() -> DataSaver.save(ArchiveName.DAY_SUM_UP, data)); + assertDoesNotThrow(() -> DataSaver.saveForCurrentDay(ArchiveName.DAY_SUM_UP, data)); } } From 748d5fc7bd77b8e7cffb1eea9f427b525d7253fa Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 21 Jan 2024 13:31:15 +0000 Subject: [PATCH 10/71] Update workflow name for archiving data --- .github/workflows/archiving-day.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/archiving-day.yml b/.github/workflows/archiving-day.yml index fd48ff137..3e58051f3 100644 --- a/.github/workflows/archiving-day.yml +++ b/.github/workflows/archiving-day.yml @@ -1,4 +1,4 @@ -name: Archiving data +name: Archiving data for day summary on: schedule: From 54e8401f47a6e70551dc8fe78117e2087c1f97e0 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:20:49 +0000 Subject: [PATCH 11/71] Add integration test workflow --- .github/workflows/Integration-test.yml | 93 ++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/Integration-test.yml diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml new file mode 100644 index 000000000..5b7d5084c --- /dev/null +++ b/.github/workflows/Integration-test.yml @@ -0,0 +1,93 @@ +name: Integration Tests + +on: + push: + branches: [ master, beta ] + pull_request: + branches: [ master, beta ] + +jobs: + unit-test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Build & Install + run: mvn -B install -D skipTests --no-transfer-progress + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: Run tests + run: mvn test + env: + ARCHIVE_MOD: HOUR + TEST_MAIL : ${{ secrets.TEST_MAIL }} + TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} + creating-archive-with-connection: + needs: unit-test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + MAIL : ${{ secrets.TEST_MAIL }} + PASSWORD : ${{ secrets.TEST_PASSWORD }} + creating-archive-day-sum-up: + needs: unit-test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + ARCHIVE_MOD: DAY_SUM_UP + SPESIFIC_DAY: "2024-01-03" + LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + creating-archive-all-logement: + needs: unit-test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + ARCHIVE_MOD: ALL_LOGEMENT + SPESIFIC_DAY: "2024-01-03" + LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available \ No newline at end of file From cc332a42f49dc1ff60a0af776f43cf5c9b1ab812 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:22:35 +0000 Subject: [PATCH 12/71] Fix typo in environment variable name --- .github/workflows/Integration-test.yml | 6 +++--- src/main/java/io/github/mathieusoysal/App.java | 2 +- .../{ArchiveMod.java => ArchiveMode.java} | 12 ++++++------ .../java/io/github/mathieusoysal/Properties.java | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) rename src/main/java/io/github/mathieusoysal/{ArchiveMod.java => ArchiveMode.java} (73%) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 5b7d5084c..d220f4234 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -23,7 +23,7 @@ jobs: - name: Run tests run: mvn test env: - ARCHIVE_MOD: HOUR + ARCHIVE_MODE: HOUR TEST_MAIL : ${{ secrets.TEST_MAIL }} TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} creating-archive-with-connection: @@ -66,7 +66,7 @@ jobs: (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ (echo "==== Maven Deploy Step Failed ====" && exit 1) env: - ARCHIVE_MOD: DAY_SUM_UP + ARCHIVE_MODE: DAY_SUM_UP SPESIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available creating-archive-all-logement: @@ -88,6 +88,6 @@ jobs: (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ (echo "==== Maven Deploy Step Failed ====" && exit 1) env: - ARCHIVE_MOD: ALL_LOGEMENT + ARCHIVE_MODE: ALL_LOGEMENT SPESIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available \ No newline at end of file diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index 5a2678a89..2e42765fe 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -7,7 +7,7 @@ public class App { public static void main(String[] args) { LOGGER.info(() -> "Starting application"); - ArchiveMod archiveMod = ArchiveMod.getArchiveModFromEnvironmentVariables(); + ArchiveMode archiveMod = ArchiveMode.getArchiveModFromEnvironmentVariables(); archiveMod.archive(); LOGGER.info(() -> "Application successfully finished"); } diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMod.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java similarity index 73% rename from src/main/java/io/github/mathieusoysal/ArchiveMod.java rename to src/main/java/io/github/mathieusoysal/ArchiveMode.java index 6a922ffbe..fdb142f91 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMod.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -7,7 +7,7 @@ import io.github.mathieusoysal.data.managment.archivers.ArchiveHour; import io.github.mathieusoysal.data.managment.archivers.Archiver; -public enum ArchiveMod { +public enum ArchiveMode { DAY_SUM_UP(ArchiveDay::new), ALL_LOGEMENTS(ArchiveAllLogements::new), HOUR(ArchiveHour::new); @@ -15,7 +15,7 @@ public enum ArchiveMod { private static final Logger LOGGER = Logger.getLogger(); private final Archiver archiver; - ArchiveMod(Archiver archiver) { + ArchiveMode(Archiver archiver) { this.archiver = archiver; } @@ -23,16 +23,16 @@ public void archive() { archiver.archive(); } - public static ArchiveMod getArchiveMod(String archiveMod) { + public static ArchiveMode getArchiveMod(String archiveMod) { try { - return ArchiveMod.valueOf(archiveMod); + return ArchiveMode.valueOf(archiveMod); } catch (IllegalArgumentException e) { LOGGER.warning(() -> archiveMod + " is not a valid archive mod"); return HOUR; } } - public static ArchiveMod getArchiveModFromEnvironmentVariables() { - return getArchiveMod(Properties.ARCHIVE_MOD.getValue()); + public static ArchiveMode getArchiveModFromEnvironmentVariables() { + return getArchiveMod(Properties.ARCHIVE_MODE.getValue()); } } diff --git a/src/main/java/io/github/mathieusoysal/Properties.java b/src/main/java/io/github/mathieusoysal/Properties.java index c91595795..e9ead20e6 100644 --- a/src/main/java/io/github/mathieusoysal/Properties.java +++ b/src/main/java/io/github/mathieusoysal/Properties.java @@ -8,7 +8,7 @@ public enum Properties { LINK_TO_ARCHIVE("LINK_TO_ARCHIVE"), MAIL("MAIL"), PASSWORD("PASSWORD"), - ARCHIVE_MOD("ARCHIVE_MOD"), + ARCHIVE_MODE("ARCHIVE_MODE"), SPECIFIC_DAY("DAY_TO_SUM_UP"); private static final Logger LOGGER = Logger.getLogger(); @@ -37,6 +37,6 @@ public String getValue() { } public static boolean modToArchiveIsActivated() { - return System.getenv(ARCHIVE_MOD.getName()) != null; + return System.getenv(ARCHIVE_MODE.getName()) != null; } } From 383b0ae5e002562ad06932b0924f4dd5e1fff281 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:26:26 +0000 Subject: [PATCH 13/71] Refactor archiver package --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 8 ++++---- .../managment => }/archivers/ArchiveAllLogements.java | 2 +- .../{data/managment => }/archivers/ArchiveDay.java | 2 +- .../{data/managment => }/archivers/ArchiveHour.java | 2 +- .../{data/managment => }/archivers/Archiver.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename src/main/java/io/github/mathieusoysal/{data/managment => }/archivers/ArchiveAllLogements.java (93%) rename src/main/java/io/github/mathieusoysal/{data/managment => }/archivers/ArchiveDay.java (90%) rename src/main/java/io/github/mathieusoysal/{data/managment => }/archivers/ArchiveHour.java (90%) rename src/main/java/io/github/mathieusoysal/{data/managment => }/archivers/Archiver.java (92%) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index fdb142f91..ecef1e97d 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -2,10 +2,10 @@ import com.github.forax.beautifullogger.Logger; -import io.github.mathieusoysal.data.managment.archivers.ArchiveAllLogements; -import io.github.mathieusoysal.data.managment.archivers.ArchiveDay; -import io.github.mathieusoysal.data.managment.archivers.ArchiveHour; -import io.github.mathieusoysal.data.managment.archivers.Archiver; +import io.github.mathieusoysal.archivers.ArchiveAllLogements; +import io.github.mathieusoysal.archivers.ArchiveDay; +import io.github.mathieusoysal.archivers.ArchiveHour; +import io.github.mathieusoysal.archivers.Archiver; public enum ArchiveMode { DAY_SUM_UP(ArchiveDay::new), diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiveAllLogements.java similarity index 93% rename from src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java rename to src/main/java/io/github/mathieusoysal/archivers/ArchiveAllLogements.java index 7412356d3..7c92836a6 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiveAllLogements.java @@ -1,4 +1,4 @@ -package io.github.mathieusoysal.data.managment.archivers; +package io.github.mathieusoysal.archivers; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiveDay.java similarity index 90% rename from src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java rename to src/main/java/io/github/mathieusoysal/archivers/ArchiveDay.java index a2bdd03cd..7edc4a393 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveDay.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiveDay.java @@ -1,4 +1,4 @@ -package io.github.mathieusoysal.data.managment.archivers; +package io.github.mathieusoysal.archivers; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiveHour.java similarity index 90% rename from src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java rename to src/main/java/io/github/mathieusoysal/archivers/ArchiveHour.java index 0b59da5d4..02d7dc764 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/archivers/ArchiveHour.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiveHour.java @@ -1,4 +1,4 @@ -package io.github.mathieusoysal.data.managment.archivers; +package io.github.mathieusoysal.archivers; import io.github.mathieusoysal.Properties; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; diff --git a/src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java similarity index 92% rename from src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java rename to src/main/java/io/github/mathieusoysal/archivers/Archiver.java index b05230e21..362dc821b 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/archivers/Archiver.java +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -1,4 +1,4 @@ -package io.github.mathieusoysal.data.managment.archivers; +package io.github.mathieusoysal.archivers; import java.time.LocalDate; From a2504f357557b7389ab6f3111a6cedecf0ee9f58 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:37:23 +0000 Subject: [PATCH 14/71] Fix access modifier in archive method --- src/main/java/io/github/mathieusoysal/archivers/Archiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java index 362dc821b..7c386c0dc 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -22,5 +22,5 @@ static LocalDate getDayToArchive() { return LocalDate.now(); } - void archive(); + public void archive(); } From a3154689ae45cd682da9334df495d6e58a533f36 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:38:05 +0000 Subject: [PATCH 15/71] Refactor archiver class names --- .../java/io/github/mathieusoysal/ArchiveMode.java | 12 ++++++------ ...veAllLogements.java => ArchiverAllLogements.java} | 2 +- .../archivers/{ArchiveDay.java => ArchiverDay.java} | 2 +- .../{ArchiveHour.java => ArchiverHour.java} | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) rename src/main/java/io/github/mathieusoysal/archivers/{ArchiveAllLogements.java => ArchiverAllLogements.java} (93%) rename src/main/java/io/github/mathieusoysal/archivers/{ArchiveDay.java => ArchiverDay.java} (92%) rename src/main/java/io/github/mathieusoysal/archivers/{ArchiveHour.java => ArchiverHour.java} (92%) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index ecef1e97d..c0f85cde3 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -2,15 +2,15 @@ import com.github.forax.beautifullogger.Logger; -import io.github.mathieusoysal.archivers.ArchiveAllLogements; -import io.github.mathieusoysal.archivers.ArchiveDay; -import io.github.mathieusoysal.archivers.ArchiveHour; +import io.github.mathieusoysal.archivers.ArchiverAllLogements; +import io.github.mathieusoysal.archivers.ArchiverDay; +import io.github.mathieusoysal.archivers.ArchiverHour; import io.github.mathieusoysal.archivers.Archiver; public enum ArchiveMode { - DAY_SUM_UP(ArchiveDay::new), - ALL_LOGEMENTS(ArchiveAllLogements::new), - HOUR(ArchiveHour::new); + DAY_SUM_UP(ArchiverDay::new), + ALL_LOGEMENTS(ArchiverAllLogements::new), + HOUR(ArchiverHour::new); private static final Logger LOGGER = Logger.getLogger(); private final Archiver archiver; diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiveAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java similarity index 93% rename from src/main/java/io/github/mathieusoysal/archivers/ArchiveAllLogements.java rename to src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java index 7c92836a6..375eb7c69 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiveAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -5,7 +5,7 @@ import io.github.mathieusoysal.data.managment.savers.DataSaver; import io.github.mathieusoysal.logement.LogementsClassifier; -public class ArchiveAllLogements implements Archiver { +public class ArchiverAllLogements implements Archiver { @Override public void archive() { diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiveDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java similarity index 92% rename from src/main/java/io/github/mathieusoysal/archivers/ArchiveDay.java rename to src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java index 7edc4a393..13d861369 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiveDay.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java @@ -4,7 +4,7 @@ import io.github.mathieusoysal.data.managment.savers.ArchiveName; import io.github.mathieusoysal.data.managment.savers.DataSaver; -public class ArchiveDay implements Archiver { +public class ArchiverDay implements Archiver { @Override public void archive() { diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiveHour.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java similarity index 92% rename from src/main/java/io/github/mathieusoysal/archivers/ArchiveHour.java rename to src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java index 02d7dc764..928810465 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiveHour.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java @@ -5,7 +5,7 @@ import io.github.mathieusoysal.data.managment.savers.ArchiveName; import io.github.mathieusoysal.data.managment.savers.DataSaver; -public class ArchiveHour implements Archiver { +public class ArchiverHour implements Archiver { @Override public void archive() { From 6dc4972990b0e5f397b1d614153d8df3d5327450 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:44:11 +0000 Subject: [PATCH 16/71] Fix access modifier of ArchiveMode constructor --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index c0f85cde3..33a3a56ba 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -15,7 +15,7 @@ public enum ArchiveMode { private static final Logger LOGGER = Logger.getLogger(); private final Archiver archiver; - ArchiveMode(Archiver archiver) { + private ArchiveMode(Archiver archiver) { this.archiver = archiver; } From cf98c017f8d1c14f7e02bcf76f82374157f9ba00 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:45:31 +0000 Subject: [PATCH 17/71] Remove unused method modToArchiveIsActivated() --- src/main/java/io/github/mathieusoysal/Properties.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/Properties.java b/src/main/java/io/github/mathieusoysal/Properties.java index e9ead20e6..c6edb8cf7 100644 --- a/src/main/java/io/github/mathieusoysal/Properties.java +++ b/src/main/java/io/github/mathieusoysal/Properties.java @@ -35,8 +35,4 @@ public String getValue() { } return propertie; } - - public static boolean modToArchiveIsActivated() { - return System.getenv(ARCHIVE_MODE.getName()) != null; - } } From 656597c15c3fa337346f214ade4a2890beeed3e4 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:54:21 +0000 Subject: [PATCH 18/71] Fix getArchiveModFromEnvironmentVariables method --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index 33a3a56ba..daceba31c 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -33,6 +33,9 @@ public static ArchiveMode getArchiveMod(String archiveMod) { } public static ArchiveMode getArchiveModFromEnvironmentVariables() { - return getArchiveMod(Properties.ARCHIVE_MODE.getValue()); + if (Properties.ARCHIVE_MODE.isPresent()) + return getArchiveMod(Properties.ARCHIVE_MODE.getValue()); + else + return HOUR; } } From 7fe13a982d6ab239cbdd648d4c0ce6a74e2e6498 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 10:55:25 +0100 Subject: [PATCH 19/71] Update src/main/java/io/github/mathieusoysal/ArchiveMode.java Signed-off-by: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index daceba31c..a14eaa3d8 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -27,7 +27,8 @@ public static ArchiveMode getArchiveMod(String archiveMod) { try { return ArchiveMode.valueOf(archiveMod); } catch (IllegalArgumentException e) { - LOGGER.warning(() -> archiveMod + " is not a valid archive mod"); + LOGGER.error(() -> archiveMod + " is not a valid archive mod"); + System.exit(1); return HOUR; } } From 772ffce3cdaf7bf38169d45b2b3182e5fd174881 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:56:22 +0000 Subject: [PATCH 20/71] Fix typo in ARCHIVE_MODE variable --- .github/workflows/Integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index d220f4234..e606a69e9 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -88,6 +88,6 @@ jobs: (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ (echo "==== Maven Deploy Step Failed ====" && exit 1) env: - ARCHIVE_MODE: ALL_LOGEMENT + ARCHIVE_MODE: ALL_LOGEMENTS SPESIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available \ No newline at end of file From b6b3e0d59776cbbd4e486bd4ee93236d88d60212 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:57:13 +0000 Subject: [PATCH 21/71] Add archiving workflow for hourly data --- .github/workflows/{archiving.yml => archiving-hour.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{archiving.yml => archiving-hour.yml} (97%) diff --git a/.github/workflows/archiving.yml b/.github/workflows/archiving-hour.yml similarity index 97% rename from .github/workflows/archiving.yml rename to .github/workflows/archiving-hour.yml index 2ff6593a1..76fbe0c8a 100644 --- a/.github/workflows/archiving.yml +++ b/.github/workflows/archiving-hour.yml @@ -1,4 +1,4 @@ -name: Archiving data +name: Archiving data for hour on: schedule: From 72e02d80b63b2e12a3e6482acf7c5fbf66957b5a Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 10:45:21 +0000 Subject: [PATCH 22/71] Add hash calculation for archived files --- .../archivers/ArchiverAllLogements.java | 38 +++++++++++++++++++ .../archivers/ArchiverAllLogementsTest.java | 33 ++++++++++++++++ .../mathieusoysal/resources/test-hash1.txt | 1 + .../mathieusoysal/resources/test-hash2.txt | 1 + .../mathieusoysal/resources/test-hash3.txt | 1 + 5 files changed, 74 insertions(+) create mode 100644 src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java create mode 100644 src/test/java/io/github/mathieusoysal/resources/test-hash1.txt create mode 100644 src/test/java/io/github/mathieusoysal/resources/test-hash2.txt create mode 100644 src/test/java/io/github/mathieusoysal/resources/test-hash3.txt diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java index 375eb7c69..1ec0a4266 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -1,5 +1,13 @@ package io.github.mathieusoysal.archivers; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import com.github.forax.beautifullogger.Logger; + import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; import io.github.mathieusoysal.data.managment.savers.DataSaver; @@ -7,6 +15,8 @@ public class ArchiverAllLogements implements Archiver { + private static final Logger LOGGER = Logger.getLogger(); + @Override public void archive() { var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); @@ -16,4 +26,32 @@ public void archive() { DataSaver.saveForGlobalArchive(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); } + String getHashOfArchivedFile(File archivedFile) { + String result = ""; + try { + byte[] digest = getHashedBytesFrom(archivedFile); + result = convertBytesToString(digest); + } catch (NoSuchAlgorithmException e) { + LOGGER.error("Error while getting hash of archived file the MD5 algorithm doesn't found in system", e); + System.exit(1); + } catch (IOException e) { + LOGGER.error("Error while reading archived file", e); + System.exit(1); + } + return result; + } + + private String convertBytesToString(byte[] digest) { + StringBuilder sb = new StringBuilder(); + for (byte b : digest) + sb.append(String.format("%02x", b)); + return sb.toString(); + } + + private byte[] getHashedBytesFrom(File archivedFile) throws NoSuchAlgorithmException, IOException { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(archivedFile.toPath())); + return md.digest(); + } + } diff --git a/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java b/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java new file mode 100644 index 000000000..505445f70 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java @@ -0,0 +1,33 @@ +package io.github.mathieusoysal.archivers; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; + +import org.junit.jupiter.api.Test; + +class ArchiverAllLogementsTest { + private static File jsonTestFile1 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash1.txt"); + private static File jsonTestFile2 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash2.txt"); + private static File jsonTestFile3 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash3.txt"); + + + @Test + void testGetHashOfArchivedFile_shouldBeEqual_IfContentIsEquals() throws NoSuchAlgorithmException, IOException { + ArchiverAllLogements archiverAllLogements = new ArchiverAllLogements(); + String hash1 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile3); + String hash2 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile2); + assertEquals(hash1, hash2); + } + + @Test + void testGetHashOfArchivedFile_shouldBeDifferent_IfContentIsDifferent() throws NoSuchAlgorithmException, IOException { + ArchiverAllLogements archiverAllLogements = new ArchiverAllLogements(); + String hash1 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile1); + String hash2 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile2); + assertEquals(false, hash1.equals(hash2)); + } + +} \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/resources/test-hash1.txt b/src/test/java/io/github/mathieusoysal/resources/test-hash1.txt new file mode 100644 index 000000000..16cbe371e --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/resources/test-hash1.txt @@ -0,0 +1 @@ +TESTTest \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/resources/test-hash2.txt b/src/test/java/io/github/mathieusoysal/resources/test-hash2.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/resources/test-hash2.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/resources/test-hash3.txt b/src/test/java/io/github/mathieusoysal/resources/test-hash3.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/resources/test-hash3.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file From 454925796ae11b587cc0ca49c8ffd4e658a281eb Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:07:13 +0000 Subject: [PATCH 23/71] Update link to archive and add specific day environment variable --- .github/workflows/Integration-test.yml | 1 + .../io/github/mathieusoysal/archivers/Archiver.java | 2 +- .../archivers/ArchiverAllLogements.java | 12 +++++++++++- .../data/managment/savers/ArchiveName.java | 1 + .../data/managment/savers/DataSaver.java | 8 ++++++-- .../archivers/ArchiverAllLogementsTest.java | 13 ++++++++++++- 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index e606a69e9..ad2ce3865 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -23,6 +23,7 @@ jobs: - name: Run tests run: mvn test env: + SPESIFIC_DAY: "2024-01-03" ARCHIVE_MODE: HOUR TEST_MAIL : ${{ secrets.TEST_MAIL }} TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} diff --git a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java index 7c386c0dc..dc4a2a07a 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -6,7 +6,7 @@ @FunctionalInterface public interface Archiver { - public static final String DEFAULT_LINK_TO_ARCHIVE = "https://raw.githubusercontent.com/mathieusoysal/LogementsCrous/main/data/data.csv"; + public static final String DEFAULT_LINK_TO_ARCHIVE = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"; static String getLinkToArchive() { if (Properties.LINK_TO_ARCHIVE.isPresent()) diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java index 1ec0a4266..aaff14ce2 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -19,11 +19,21 @@ public class ArchiverAllLogements implements Archiver { @Override public void archive() { + var archivedFile = archiveAllLogements(); + updateHashOfAllLogement(archivedFile); + } + + private void updateHashOfAllLogement(File archivedFile) { + var hash = getHashOfArchivedFile(archivedFile); + DataSaver.saveForGlobalArchive(ArchiveName.HASH_ALL_LOGEMENTS, hash); + } + + private File archiveAllLogements() { var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); var logements = new LogementsClassifier(); logements.addLogements(dataCollector.getAllLogements()); logements.addLogements(dataCollector.getConvertedSumUpOfDay(Archiver.getDayToArchive())); - DataSaver.saveForGlobalArchive(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); + return DataSaver.saveForGlobalArchive(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); } String getHashOfArchivedFile(File archivedFile) { diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java index 45813150d..b82abfe0a 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java @@ -6,6 +6,7 @@ public enum ArchiveName { ALL_LOGEMENTS(() -> "all_logements"), + HASH_ALL_LOGEMENTS(() -> "hash_all_logements"), DAY_SUM_UP(() -> "sum-up"), HOUR(() -> OffsetDateTime.now().toLocalTime().format(DateTimeFormatter.ofPattern("HH"))); diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java index a0db38715..7e00ab0d0 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java @@ -26,9 +26,13 @@ public static File saveForCurrentDay(ArchiveName archiveName, List log } public static File saveForGlobalArchive(ArchiveName archiveName, List logements) { - File archiveFile = FileManager.getOrCreateArchiveFileForGlobal(archiveName); String logementsJson = Convertor.convertLogementsToJson(logements); - writeDataInsideArchiveFile(logementsJson, archiveFile); + return saveForGlobalArchive(archiveName, logementsJson); + } + + public static File saveForGlobalArchive(ArchiveName archiveName, String content) { + File archiveFile = FileManager.getOrCreateArchiveFileForGlobal(archiveName); + writeDataInsideArchiveFile(content, archiveFile); return archiveFile; } diff --git a/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java b/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java index 505445f70..2d68d7c0f 100644 --- a/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java +++ b/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java @@ -1,18 +1,30 @@ package io.github.mathieusoysal.archivers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; import java.security.NoSuchAlgorithmException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; class ArchiverAllLogementsTest { private static File jsonTestFile1 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash1.txt"); private static File jsonTestFile2 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash2.txt"); private static File jsonTestFile3 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash3.txt"); + @Test + @EnabledIfEnvironmentVariable(named = "CI", matches = "true") + void testArchive_shouldCallArchiveAllLogementsAndUpdateHashOfAllLogement() throws NoSuchAlgorithmException, IOException { + ArchiverAllLogements archiverAllLogements = new ArchiverAllLogements(); + + archiverAllLogements.archive(); + + assertTrue(new File("archive/all_logements").exists()); + assertTrue(new File("archive/hash_all_logements").exists()); + } @Test void testGetHashOfArchivedFile_shouldBeEqual_IfContentIsEquals() throws NoSuchAlgorithmException, IOException { @@ -29,5 +41,4 @@ void testGetHashOfArchivedFile_shouldBeDifferent_IfContentIsDifferent() throws N String hash2 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile2); assertEquals(false, hash1.equals(hash2)); } - } \ No newline at end of file From d69dcd3afcae1187d197233e441e59ab8fe15310 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 12:51:48 +0100 Subject: [PATCH 24/71] Update src/main/java/io/github/mathieusoysal/archivers/Archiver.java Signed-off-by: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> --- src/main/java/io/github/mathieusoysal/archivers/Archiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java index dc4a2a07a..c90a94314 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -6,7 +6,7 @@ @FunctionalInterface public interface Archiver { - public static final String DEFAULT_LINK_TO_ARCHIVE = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"; + public static final String DEFAULT_LINK_TO_ARCHIVE = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available"; static String getLinkToArchive() { if (Properties.LINK_TO_ARCHIVE.isPresent()) From 38070f8dae96179e87f4113edb05e2212f0501a0 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 14:11:29 +0000 Subject: [PATCH 25/71] Fix typo in environment variable name --- .github/workflows/Integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index ad2ce3865..755cc4be4 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -23,7 +23,7 @@ jobs: - name: Run tests run: mvn test env: - SPESIFIC_DAY: "2024-01-03" + SPECIFIC_DAY: "2024-01-03" ARCHIVE_MODE: HOUR TEST_MAIL : ${{ secrets.TEST_MAIL }} TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} From 2fd24c4888f4274b92be656430075356f05fb2a3 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:12:07 +0000 Subject: [PATCH 26/71] fix SPECIFIC_DAY enum value --- src/main/java/io/github/mathieusoysal/Properties.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/Properties.java b/src/main/java/io/github/mathieusoysal/Properties.java index c6edb8cf7..86ac952a9 100644 --- a/src/main/java/io/github/mathieusoysal/Properties.java +++ b/src/main/java/io/github/mathieusoysal/Properties.java @@ -9,7 +9,7 @@ public enum Properties { MAIL("MAIL"), PASSWORD("PASSWORD"), ARCHIVE_MODE("ARCHIVE_MODE"), - SPECIFIC_DAY("DAY_TO_SUM_UP"); + SPECIFIC_DAY("SPECIFIC_DAY"); private static final Logger LOGGER = Logger.getLogger(); private final String name; From 52f3a22dfb3e75db7a8a41f646e7aef8cd1f4349 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:12:57 +0000 Subject: [PATCH 27/71] Add ArchiveModeTest class with test cases for getArchiveMod method --- .../github/mathieusoysal/ArchiveModeTest.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/test/java/io/github/mathieusoysal/ArchiveModeTest.java diff --git a/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java b/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java new file mode 100644 index 000000000..94df37687 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java @@ -0,0 +1,35 @@ +package io.github.mathieusoysal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class ArchiveModeTest { + + @Test + void testGetArchiveMod_shouldReturnHour_whenArchiveModIsHour() { + ArchiveMode archiveMode = ArchiveMode.getArchiveMod("HOUR"); + assertEquals(ArchiveMode.HOUR, archiveMode); + } + + @Test + void testGetArchiveMod_shouldReturnDay_whenArchiveModIsDay() { + ArchiveMode archiveMode = ArchiveMode.getArchiveMod("DAY_SUM_UP"); + assertEquals(ArchiveMode.DAY_SUM_UP, archiveMode); + } + + @Test + void testGetArchiveMod_shouldReturnWeek_whenArchiveModIsWeek() { + ArchiveMode archiveMode = ArchiveMode.getArchiveMod("ALL_LOGEMENTS"); + assertEquals(ArchiveMode.ALL_LOGEMENTS, archiveMode); + } + + + @Test + void testGetArchiveMod_shouldLogErrorAndExit_whenArchiveModIsInvalid() { + assertThrows(IllegalArgumentException.class, () -> { + ArchiveMode.getArchiveMod("INVALID"); + }); + } +} \ No newline at end of file From 4564509f20abb9dde17ac0033b5a08cc9e7428c4 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:14:39 +0000 Subject: [PATCH 28/71] Remove Playwright workflow --- .github/workflows/playwright.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/playwright.yml diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml deleted file mode 100644 index 731c5e7f1..000000000 --- a/.github/workflows/playwright.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Playwright Tests -on: - push: - branches: [ master, beta ] - pull_request: - branches: [ master, beta ] -jobs: - test: - timeout-minutes: 60 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - - name: Build & Install - run: mvn -B install -D skipTests --no-transfer-progress - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: Run tests - run: mvn test - env: - TEST_MAIL : ${{ secrets.TEST_MAIL }} - TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} \ No newline at end of file From 2a9375c0c01579a34a386a739d93549d3f963403 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:30:54 +0000 Subject: [PATCH 29/71] Refactor getArchiveMod to getArchiveMode --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 7 +++---- .../java/io/github/mathieusoysal/ArchiveModeTest.java | 8 ++++---- .../data/managment/collectors/DataCollectorTest.java | 2 ++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index a14eaa3d8..b56ac76db 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -23,19 +23,18 @@ public void archive() { archiver.archive(); } - public static ArchiveMode getArchiveMod(String archiveMod) { + public static ArchiveMode getArchiveMode(String archiveMod) { try { return ArchiveMode.valueOf(archiveMod); } catch (IllegalArgumentException e) { LOGGER.error(() -> archiveMod + " is not a valid archive mod"); - System.exit(1); - return HOUR; + throw e; } } public static ArchiveMode getArchiveModFromEnvironmentVariables() { if (Properties.ARCHIVE_MODE.isPresent()) - return getArchiveMod(Properties.ARCHIVE_MODE.getValue()); + return getArchiveMode(Properties.ARCHIVE_MODE.getValue()); else return HOUR; } diff --git a/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java b/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java index 94df37687..e2237ccc3 100644 --- a/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java +++ b/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java @@ -9,19 +9,19 @@ class ArchiveModeTest { @Test void testGetArchiveMod_shouldReturnHour_whenArchiveModIsHour() { - ArchiveMode archiveMode = ArchiveMode.getArchiveMod("HOUR"); + ArchiveMode archiveMode = ArchiveMode.getArchiveMode("HOUR"); assertEquals(ArchiveMode.HOUR, archiveMode); } @Test void testGetArchiveMod_shouldReturnDay_whenArchiveModIsDay() { - ArchiveMode archiveMode = ArchiveMode.getArchiveMod("DAY_SUM_UP"); + ArchiveMode archiveMode = ArchiveMode.getArchiveMode("DAY_SUM_UP"); assertEquals(ArchiveMode.DAY_SUM_UP, archiveMode); } @Test void testGetArchiveMod_shouldReturnWeek_whenArchiveModIsWeek() { - ArchiveMode archiveMode = ArchiveMode.getArchiveMod("ALL_LOGEMENTS"); + ArchiveMode archiveMode = ArchiveMode.getArchiveMode("ALL_LOGEMENTS"); assertEquals(ArchiveMode.ALL_LOGEMENTS, archiveMode); } @@ -29,7 +29,7 @@ void testGetArchiveMod_shouldReturnWeek_whenArchiveModIsWeek() { @Test void testGetArchiveMod_shouldLogErrorAndExit_whenArchiveModIsInvalid() { assertThrows(IllegalArgumentException.class, () -> { - ArchiveMode.getArchiveMod("INVALID"); + ArchiveMode.getArchiveMode("INVALID"); }); } } \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java b/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java index 2b1649284..96ed12949 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java @@ -10,6 +10,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.exc.StreamReadException; @@ -64,6 +65,7 @@ void testConvertion() throws StreamReadException, DatabindException, ApiRequestF } @Test + @EnabledIfEnvironmentVariable(named = "CI", matches = "true") void testGetLogementsWithConnection_returnsLogements() throws StreamReadException, DatabindException, ApiRequestFailedException, IOException, InterruptedException { From bd43901e9b22d06e97a02860432e9be3ef4b9455 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:02:59 +0000 Subject: [PATCH 30/71] Fix typo in method name --- src/main/java/io/github/mathieusoysal/App.java | 2 +- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index 2e42765fe..ed0b1612d 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -7,7 +7,7 @@ public class App { public static void main(String[] args) { LOGGER.info(() -> "Starting application"); - ArchiveMode archiveMod = ArchiveMode.getArchiveModFromEnvironmentVariables(); + ArchiveMode archiveMod = ArchiveMode.getArchiveModeFromEnvironmentVariables(); archiveMod.archive(); LOGGER.info(() -> "Application successfully finished"); } diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index b56ac76db..30e4a03c8 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -32,10 +32,7 @@ public static ArchiveMode getArchiveMode(String archiveMod) { } } - public static ArchiveMode getArchiveModFromEnvironmentVariables() { - if (Properties.ARCHIVE_MODE.isPresent()) - return getArchiveMode(Properties.ARCHIVE_MODE.getValue()); - else - return HOUR; + public static ArchiveMode getArchiveModeFromEnvironmentVariables() { + return getArchiveMode(Properties.ARCHIVE_MODE.getValue()); } } From 076c8eb4240e49cb1e463395c05bdb8e7389d17b Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:13:41 +0000 Subject: [PATCH 31/71] Add logging for archive mode --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index 30e4a03c8..0b54a620b 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -25,7 +25,9 @@ public void archive() { public static ArchiveMode getArchiveMode(String archiveMod) { try { - return ArchiveMode.valueOf(archiveMod); + ArchiveMode m = ArchiveMode.valueOf(archiveMod); + LOGGER.info(() -> "Archive mode is " + m.name()); + return m; } catch (IllegalArgumentException e) { LOGGER.error(() -> archiveMod + " is not a valid archive mod"); throw e; From 68aaea83574f745c4ff30ea31a29c7ff17cb7af0 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:15:33 +0000 Subject: [PATCH 32/71] Add ARCHIVE_MODE environment variable --- .github/workflows/Integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 755cc4be4..8345658e2 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -46,6 +46,7 @@ jobs: (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ (echo "==== Maven Deploy Step Failed ====" && exit 1) env: + ARCHIVE_MODE: HOUR MAIL : ${{ secrets.TEST_MAIL }} PASSWORD : ${{ secrets.TEST_PASSWORD }} creating-archive-day-sum-up: From cdbe3f4ecacfcd7c21246c5cfef2a7217565ebb7 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:30:07 +0000 Subject: [PATCH 33/71] Add assert-command-line-output action to Integration-test workflow --- .github/workflows/Integration-test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 8345658e2..5f411ed98 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -71,6 +71,10 @@ jobs: ARCHIVE_MODE: DAY_SUM_UP SPESIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + - uses: GuillaumeFalourd/assert-command-line-output@v2 + with: + command_line: ls archive/* + contains: sum-up creating-archive-all-logement: needs: unit-test runs-on: ubuntu-latest @@ -92,4 +96,8 @@ jobs: env: ARCHIVE_MODE: ALL_LOGEMENTS SPESIFIC_DAY: "2024-01-03" - LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available \ No newline at end of file + LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + - uses: GuillaumeFalourd/assert-command-line-output@v2 + with: + command_line: ls archive/* + contains: all_logements \ No newline at end of file From dadacb5354a1a75bbd7255694bde28cc879c9721 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:40:39 +0000 Subject: [PATCH 34/71] Add assert-command-line-output action to Integration-test.yml --- .github/workflows/Integration-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 5f411ed98..5ec76939f 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -49,6 +49,10 @@ jobs: ARCHIVE_MODE: HOUR MAIL : ${{ secrets.TEST_MAIL }} PASSWORD : ${{ secrets.TEST_PASSWORD }} + - uses: GuillaumeFalourd/assert-command-line-output@v2 + with: + command_line: ls archive/* + contains: archive creating-archive-day-sum-up: needs: unit-test runs-on: ubuntu-latest @@ -75,6 +79,7 @@ jobs: with: command_line: ls archive/* contains: sum-up + creating-archive-all-logement: needs: unit-test runs-on: ubuntu-latest @@ -97,6 +102,7 @@ jobs: ARCHIVE_MODE: ALL_LOGEMENTS SPESIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + - uses: GuillaumeFalourd/assert-command-line-output@v2 with: command_line: ls archive/* From 2c8b35d1cefab90e64252c934936e3233596fb4f Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:41:16 +0000 Subject: [PATCH 35/71] Fix bug ArchiveMode --- src/main/java/io/github/mathieusoysal/ArchiveMode.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java index 0b54a620b..c43e238ad 100644 --- a/src/main/java/io/github/mathieusoysal/ArchiveMode.java +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -8,9 +8,9 @@ import io.github.mathieusoysal.archivers.Archiver; public enum ArchiveMode { - DAY_SUM_UP(ArchiverDay::new), - ALL_LOGEMENTS(ArchiverAllLogements::new), - HOUR(ArchiverHour::new); + DAY_SUM_UP(new ArchiverDay()), + ALL_LOGEMENTS(new ArchiverAllLogements()), + HOUR(new ArchiverHour()); private static final Logger LOGGER = Logger.getLogger(); private final Archiver archiver; From d91a84b3d5c0c1cc9656410d9bafb6b4b5818707 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:48:47 +0000 Subject: [PATCH 36/71] Fix typo in environment variable name --- .github/workflows/Integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 5ec76939f..807c56ac8 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -73,7 +73,7 @@ jobs: (echo "==== Maven Deploy Step Failed ====" && exit 1) env: ARCHIVE_MODE: DAY_SUM_UP - SPESIFIC_DAY: "2024-01-03" + SPECIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - uses: GuillaumeFalourd/assert-command-line-output@v2 with: @@ -100,7 +100,7 @@ jobs: (echo "==== Maven Deploy Step Failed ====" && exit 1) env: ARCHIVE_MODE: ALL_LOGEMENTS - SPESIFIC_DAY: "2024-01-03" + SPECIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - uses: GuillaumeFalourd/assert-command-line-output@v2 From 6a290b2617f16e2c91e4b4ffade2567790dfb79b Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:59:05 +0000 Subject: [PATCH 37/71] Update command_line in Integration-test.yml --- .github/workflows/Integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 807c56ac8..5f0275d97 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -51,7 +51,7 @@ jobs: PASSWORD : ${{ secrets.TEST_PASSWORD }} - uses: GuillaumeFalourd/assert-command-line-output@v2 with: - command_line: ls archive/* + command_line: ls contains: archive creating-archive-day-sum-up: needs: unit-test From 89f6353d7233b954480f2c5370bbb3a84607f1e9 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:56:27 +0000 Subject: [PATCH 38/71] solves #36 --- .../github/mathieusoysal/data/managment/savers/ArchiveName.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java index b82abfe0a..941ecc9d7 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java @@ -8,7 +8,7 @@ public enum ArchiveName { ALL_LOGEMENTS(() -> "all_logements"), HASH_ALL_LOGEMENTS(() -> "hash_all_logements"), DAY_SUM_UP(() -> "sum-up"), - HOUR(() -> OffsetDateTime.now().toLocalTime().format(DateTimeFormatter.ofPattern("HH"))); + HOUR(() -> OffsetDateTime.now().format(DateTimeFormatter.ofPattern("HH"))); private final Supplier nameSupplier; From 02f7ef804307caeb4cb8da8206ba54a2f3cefe0f Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 11:27:07 +0000 Subject: [PATCH 39/71] Add JIB container build with Playwright image resolves #51 --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 747a7b5ec..b26554dab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,6 +47,7 @@ jobs: - name: Build JIB container and publish to GitHub Packages run: | mvn package com.google.cloud.tools:jib-maven-plugin:3.3.2:build \ + -Djib.from.image=mcr.microsoft.com/playwright/java:v1.41.0-jammy \ -Djib.to.image=ghcr.io/${{ steps.downcase.outputs.lowercase }}:latest \ -Djib.to.auth.username=${{ github.actor }} \ -Djib.to.auth.password=${{ secrets.GITHUB_TOKEN }} $MULTI_MODULE_ARGS $MAIN_CLASS_ARGS From e703a47806b93b160a9a724bc88f2232cf3bd38d Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 11:27:31 +0000 Subject: [PATCH 40/71] Add ARCHIVE_MODE environment variable --- .github/workflows/archiving-day.yml | 1 + .github/workflows/archiving-hour.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/archiving-day.yml b/.github/workflows/archiving-day.yml index 3e58051f3..2428bbf6c 100644 --- a/.github/workflows/archiving-day.yml +++ b/.github/workflows/archiving-day.yml @@ -24,6 +24,7 @@ jobs: (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ (echo "==== Maven Deploy Step Failed ====" && exit 1) env: + ARCHIVE_MODE: DAY_SUM_UP LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - uses: JamesIves/github-pages-deploy-action@v4.4.3 with: diff --git a/.github/workflows/archiving-hour.yml b/.github/workflows/archiving-hour.yml index 76fbe0c8a..1e739a7f8 100644 --- a/.github/workflows/archiving-hour.yml +++ b/.github/workflows/archiving-hour.yml @@ -24,6 +24,7 @@ jobs: (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ (echo "==== Maven Deploy Step Failed ====" && exit 1) env: + ARCHIVE_MODE: HOUR MAIL : ${{ secrets.TEST_MAIL }} PASSWORD : ${{ secrets.TEST_PASSWORD }} - uses: JamesIves/github-pages-deploy-action@v4.4.3 From 854a6b8855894c7019e890ea0f8e8b76eb2d07b9 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 12:47:20 +0000 Subject: [PATCH 41/71] Add workflow to update all logements --- .github/workflows/update-all-logements.yml | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/update-all-logements.yml diff --git a/.github/workflows/update-all-logements.yml b/.github/workflows/update-all-logements.yml new file mode 100644 index 000000000..5df8599d6 --- /dev/null +++ b/.github/workflows/update-all-logements.yml @@ -0,0 +1,40 @@ +name: Update all logements + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + +jobs: + creating-archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + ARCHIVE_MODE: ALL_LOGEMENTS + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + - uses: JamesIves/github-pages-deploy-action@v4.4.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + clean: false + branch: api + folder: archive + target-folder: v1/logements-crous + + + + + \ No newline at end of file From e9d38b124a23adedec8238b6db60f93891e9a206 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:23:38 +0000 Subject: [PATCH 42/71] Refactor archiving workflow to use reusable workflow resolves #54 --- .github/workflows/Integration-test.yml | 68 +++++------------------- .github/workflows/reusable-archiving.yml | 57 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/reusable-archiving.yml diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 5f0275d97..12ae32174 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -28,50 +28,24 @@ jobs: TEST_MAIL : ${{ secrets.TEST_MAIL }} TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} creating-archive-with-connection: - needs: unit-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: - ARCHIVE_MODE: HOUR - MAIL : ${{ secrets.TEST_MAIL }} - PASSWORD : ${{ secrets.TEST_PASSWORD }} - - uses: GuillaumeFalourd/assert-command-line-output@v2 - with: - command_line: ls - contains: archive + needs: unit-test + runs-on: ubuntu-latest + steps: + - uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: HOUR + MAIL : ${{ secrets.TEST_MAIL }} + PASSWORD : ${{ secrets.TEST_PASSWORD }} + - uses: GuillaumeFalourd/assert-command-line-output@v2 + with: + command_line: ls + contains: archive creating-archive-day-sum-up: needs: unit-test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: ./.github/workflows/reusable-archiving.yml with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: ARCHIVE_MODE: DAY_SUM_UP SPECIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available @@ -84,25 +58,11 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: ./.github/workflows/reusable-archiving.yml with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: ARCHIVE_MODE: ALL_LOGEMENTS SPECIFIC_DAY: "2024-01-03" LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - - uses: GuillaumeFalourd/assert-command-line-output@v2 with: command_line: ls archive/* diff --git a/.github/workflows/reusable-archiving.yml b/.github/workflows/reusable-archiving.yml new file mode 100644 index 000000000..76f7737df --- /dev/null +++ b/.github/workflows/reusable-archiving.yml @@ -0,0 +1,57 @@ +name: Archiving data with reusable workflow + +on: + workflow_call: + inputs: + ARCHIVE_MODE: + description: 'The mode of archiving' + required: true + type: string + default: 'HOUR' + LINK_TO_DATA: + description: 'The link to the data to archive' + required: false + type: string + default: 'https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available' + MAIL: + description: 'The mail to send the archive to hour, mail is used olny if ARCHIVE_MODE is HOUR' + required: false + type: string + PASSWORD: + description: 'The password of the mail to send the archive to hour, password is used olny if ARCHIVE_MODE is HOUR' + required: false + type: string + SPECIFIC_DAY: + description: 'The day to archive, day is used olny if ARCHIVE_MODE is DAY' + required: false + type: string + default: '' + +jobs: + creating-archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + + - name: set specific day when input is not empty + if: inputs.SPECIFIC_DAY != '' + run: echo "SPECIFIC_DAY=${{ inputs.SPECIFIC_DAY }}" >> $GITHUB_ENV + + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + LINK_TO_DATA: ${{ inputs.LINK_TO_DATA }} + ARCHIVE_MODE: ${{ inputs.ARCHIVE_MODE }} + MAIL: ${{ inputs.MAIL }} + PASSWORD: ${{ inputs.PASSWORD }} \ No newline at end of file From 6741e74b71f74cfc8a0cff7b5e92694b8832c009 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:46:16 +0000 Subject: [PATCH 43/71] Update GitHub Actions setup-java version and add Maven cache --- .github/workflows/Integration-test.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 12ae32174..5e6d5e36f 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -11,11 +11,12 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '17' + cache: 'maven' - name: Build & Install run: mvn -B install -D skipTests --no-transfer-progress - name: Install Playwright @@ -31,7 +32,7 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: ./.github/workflows/reusable-archiving.yml + - uses: ./reusable-archiving.yml with: ARCHIVE_MODE: HOUR MAIL : ${{ secrets.TEST_MAIL }} @@ -44,7 +45,7 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: ./.github/workflows/reusable-archiving.yml + - uses: ./reusable-archiving.yml with: ARCHIVE_MODE: DAY_SUM_UP SPECIFIC_DAY: "2024-01-03" @@ -58,7 +59,7 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: ./.github/workflows/reusable-archiving.yml + - uses: ./reusable-archiving.yml with: ARCHIVE_MODE: ALL_LOGEMENTS SPECIFIC_DAY: "2024-01-03" From c37228c12a323ce719fc4929272174c7816de0de Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:03:36 +0000 Subject: [PATCH 44/71] Update reusable-archiving.yml and Integration-test.yml resolves Refactor workflows #54 --- .github/workflows/Integration-test.yml | 9 ++++++--- .github/workflows/reusable-archiving.yml | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 5e6d5e36f..9a1d0f4c9 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -32,7 +32,8 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: ./reusable-archiving.yml + - uses: actions/checkout@v4 + - uses: ./.github/workflows/reusable-archiving.yml with: ARCHIVE_MODE: HOUR MAIL : ${{ secrets.TEST_MAIL }} @@ -45,7 +46,8 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: ./reusable-archiving.yml + - uses: actions/checkout@v4 + - uses: ./.github/workflows/reusable-archiving.yml with: ARCHIVE_MODE: DAY_SUM_UP SPECIFIC_DAY: "2024-01-03" @@ -59,7 +61,8 @@ jobs: needs: unit-test runs-on: ubuntu-latest steps: - - uses: ./reusable-archiving.yml + - uses: actions/checkout@v4 + - uses: ./.github/workflows/reusable-archiving.yml with: ARCHIVE_MODE: ALL_LOGEMENTS SPECIFIC_DAY: "2024-01-03" diff --git a/.github/workflows/reusable-archiving.yml b/.github/workflows/reusable-archiving.yml index 76f7737df..ae2abcbf3 100644 --- a/.github/workflows/reusable-archiving.yml +++ b/.github/workflows/reusable-archiving.yml @@ -31,7 +31,6 @@ jobs: creating-archive: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: distribution: 'temurin' From 210707ce355f6724f452cd17feb5bdbbb238c8cc Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:25:53 +0000 Subject: [PATCH 45/71] Refactor reusable-archiving.yml and Integration-test.yml workflows resolves Refactor workflows #54 --- .github/workflows/Integration-test.yml | 53 ++++++++---------------- .github/workflows/reusable-archiving.yml | 18 ++++---- 2 files changed, 25 insertions(+), 46 deletions(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 9a1d0f4c9..8c245a1f7 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -30,44 +30,25 @@ jobs: TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} creating-archive-with-connection: needs: unit-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/workflows/reusable-archiving.yml - with: - ARCHIVE_MODE: HOUR - MAIL : ${{ secrets.TEST_MAIL }} - PASSWORD : ${{ secrets.TEST_PASSWORD }} - - uses: GuillaumeFalourd/assert-command-line-output@v2 - with: - command_line: ls - contains: archive + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: HOUR + secrets: + MAIL : ${{ secrets.TEST_MAIL }} + PASSWORD : ${{ secrets.TEST_PASSWORD }} + creating-archive-day-sum-up: needs: unit-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/workflows/reusable-archiving.yml - with: - ARCHIVE_MODE: DAY_SUM_UP - SPECIFIC_DAY: "2024-01-03" - LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - - uses: GuillaumeFalourd/assert-command-line-output@v2 - with: - command_line: ls archive/* - contains: sum-up + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: DAY_SUM_UP + SPECIFIC_DAY: "2024-01-03" + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available creating-archive-all-logement: needs: unit-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/workflows/reusable-archiving.yml - with: - ARCHIVE_MODE: ALL_LOGEMENTS - SPECIFIC_DAY: "2024-01-03" - LINK_TO_ARCHIVE: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - - uses: GuillaumeFalourd/assert-command-line-output@v2 - with: - command_line: ls archive/* - contains: all_logements \ No newline at end of file + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: ALL_LOGEMENTS + SPECIFIC_DAY: "2024-01-03" + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available \ No newline at end of file diff --git a/.github/workflows/reusable-archiving.yml b/.github/workflows/reusable-archiving.yml index ae2abcbf3..7324187c6 100644 --- a/.github/workflows/reusable-archiving.yml +++ b/.github/workflows/reusable-archiving.yml @@ -13,24 +13,22 @@ on: required: false type: string default: 'https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available' - MAIL: - description: 'The mail to send the archive to hour, mail is used olny if ARCHIVE_MODE is HOUR' - required: false - type: string - PASSWORD: - description: 'The password of the mail to send the archive to hour, password is used olny if ARCHIVE_MODE is HOUR' - required: false - type: string SPECIFIC_DAY: description: 'The day to archive, day is used olny if ARCHIVE_MODE is DAY' required: false type: string default: '' + secrets: + MAIL: + description: 'The mail to send the archive to hour, mail is used olny if ARCHIVE_MODE is HOUR' + PASSWORD: + description: 'The password of the mail to send the archive to hour, password is used olny if ARCHIVE_MODE is HOUR' jobs: creating-archive: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: distribution: 'temurin' @@ -52,5 +50,5 @@ jobs: env: LINK_TO_DATA: ${{ inputs.LINK_TO_DATA }} ARCHIVE_MODE: ${{ inputs.ARCHIVE_MODE }} - MAIL: ${{ inputs.MAIL }} - PASSWORD: ${{ inputs.PASSWORD }} \ No newline at end of file + MAIL: ${{ secrets.MAIL }} + PASSWORD: ${{ secrets.PASSWORD }} \ No newline at end of file From 2099df4f0a6ecd8c38adec04b898a3f4c18d1a21 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:58:39 +0000 Subject: [PATCH 46/71] Add deploy action to reusable-archiving workflow resolves Refactor workflows #54 --- .github/workflows/Integration-test.yml | 1 + .github/workflows/reusable-archiving.yml | 54 +++++++++++++++++------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index 8c245a1f7..f0d3fa202 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -28,6 +28,7 @@ jobs: ARCHIVE_MODE: HOUR TEST_MAIL : ${{ secrets.TEST_MAIL }} TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} + creating-archive-with-connection: needs: unit-test uses: ./.github/workflows/reusable-archiving.yml diff --git a/.github/workflows/reusable-archiving.yml b/.github/workflows/reusable-archiving.yml index 7324187c6..ccf8f0f75 100644 --- a/.github/workflows/reusable-archiving.yml +++ b/.github/workflows/reusable-archiving.yml @@ -3,26 +3,39 @@ name: Archiving data with reusable workflow on: workflow_call: inputs: - ARCHIVE_MODE: - description: 'The mode of archiving' - required: true - type: string - default: 'HOUR' - LINK_TO_DATA: - description: 'The link to the data to archive' - required: false - type: string - default: 'https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available' - SPECIFIC_DAY: - description: 'The day to archive, day is used olny if ARCHIVE_MODE is DAY' - required: false - type: string - default: '' + ARCHIVE_MODE: + description: 'The mode of archiving' + required: true + type: string + default: 'HOUR' + LINK_TO_DATA: + description: 'The link to the data to archive' + required: false + type: string + default: 'https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available' + SPECIFIC_DAY: + description: 'The day to archive, day is used olny if ARCHIVE_MODE is DAY' + required: false + type: string + default: '' + deploy: + description: 'Deploy the archive to github pages' + required: false + type: boolean + default: false + target-folder: + description: 'The target folder to deploy the archive to github pages' + required: false + type: string + default: 'v1/logements-crous/available' + secrets: MAIL: description: 'The mail to send the archive to hour, mail is used olny if ARCHIVE_MODE is HOUR' PASSWORD: description: 'The password of the mail to send the archive to hour, password is used olny if ARCHIVE_MODE is HOUR' + token: + description: 'The token to deploy the archive to github pages' jobs: creating-archive: @@ -51,4 +64,13 @@ jobs: LINK_TO_DATA: ${{ inputs.LINK_TO_DATA }} ARCHIVE_MODE: ${{ inputs.ARCHIVE_MODE }} MAIL: ${{ secrets.MAIL }} - PASSWORD: ${{ secrets.PASSWORD }} \ No newline at end of file + PASSWORD: ${{ secrets.PASSWORD }} + - name: Deploy to Github Pages + if: inputs.deploy == true + uses: JamesIves/github-pages-deploy-action@v4.4.3 + with: + token: ${{ secrets.token }} + clean: false + branch: api + folder: archive + target-folder: ${{ inputs.target-folder }} \ No newline at end of file From 22798abad5271a5813920322d8f804e4ec9a1691 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:17:40 +0000 Subject: [PATCH 47/71] update archiving-day resolves Add possibility to run a workflow archive day for a specific day #53 --- .github/workflows/archiving-day.yml | 54 ++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/archiving-day.yml b/.github/workflows/archiving-day.yml index 2428bbf6c..e766c162b 100644 --- a/.github/workflows/archiving-day.yml +++ b/.github/workflows/archiving-day.yml @@ -4,35 +4,35 @@ on: schedule: - cron: '40 23 * * *' workflow_dispatch: + inputs: + specific-date: + description: 'Specific date to archive' + required: true + default: '2024-01-01' jobs: - creating-archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: - ARCHIVE_MODE: DAY_SUM_UP - LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - - uses: JamesIves/github-pages-deploy-action@v4.4.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - clean: false - branch: api - folder: archive - target-folder: v1/logements-crous/available + creating-archive-schedule: + permissions: + contents: write + if: github.event_name == 'schedule' + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: DAY_SUM_UP + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + creating-archive-manual: + permissions: + contents: write + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-archiving.yml + with: + SPECIFIC_DAY: ${{ github.event.inputs.specific-date }} + ARCHIVE_MODE: DAY_SUM_UP + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + secrets: + token: ${{ secrets.GITHUB_TOKEN }} From c4222fb7acc56e3de3cd7161f6ce564e50a20bfb Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:24:33 +0000 Subject: [PATCH 48/71] Refactor archiving-hour.yml and update-all-logements.yml workflows resolves Refactor workflows #54 and Add possibility to run a workflow archive day for a specific day #53 --- .github/workflows/archiving-hour.yml | 36 ++++---------- .github/workflows/update-all-logements.yml | 57 ++++++++++------------ 2 files changed, 35 insertions(+), 58 deletions(-) diff --git a/.github/workflows/archiving-hour.yml b/.github/workflows/archiving-hour.yml index 1e739a7f8..83947dd0e 100644 --- a/.github/workflows/archiving-hour.yml +++ b/.github/workflows/archiving-hour.yml @@ -7,33 +7,15 @@ on: jobs: creating-archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: - ARCHIVE_MODE: HOUR - MAIL : ${{ secrets.TEST_MAIL }} - PASSWORD : ${{ secrets.TEST_PASSWORD }} - - uses: JamesIves/github-pages-deploy-action@v4.4.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - clean: false - branch: api - folder: archive - target-folder: v1/logements-crous/available + permissions: + contents: write + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: HOUR + secrets: + MAIL : ${{ secrets.TEST_MAIL }} + PASSWORD : ${{ secrets.TEST_PASSWORD }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/update-all-logements.yml b/.github/workflows/update-all-logements.yml index 5df8599d6..fed01b0d9 100644 --- a/.github/workflows/update-all-logements.yml +++ b/.github/workflows/update-all-logements.yml @@ -4,37 +4,32 @@ on: schedule: - cron: '40 23 * * *' workflow_dispatch: + inputs: + specific-date: + description: 'Specific date to archive' + required: true + default: '2024-01-01' jobs: - creating-archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: - ARCHIVE_MODE: ALL_LOGEMENTS - LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - - uses: JamesIves/github-pages-deploy-action@v4.4.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - clean: false - branch: api - folder: archive - target-folder: v1/logements-crous + creating-archive-schedule: + permissions: + contents: write + if: github.event_name == 'schedule' + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: ALL_LOGEMENTS + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + secrets: + token: ${{ secrets.GITHUB_TOKEN }} - - - - \ No newline at end of file + creating-archive-manual: + permissions: + contents: write + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-archiving.yml + with: + SPECIFIC_DAY: ${{ github.event.inputs.specific-date }} + ARCHIVE_MODE: ALL_LOGEMENTS + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + secrets: + token: ${{ secrets.GITHUB_TOKEN }} From a008508beafe821ee3bde7c41238402eb88b47f2 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:36:49 +0000 Subject: [PATCH 49/71] Add deploy flag to workflow files resolves Refactor workflows #54 --- .github/workflows/archiving-day.yml | 2 ++ .github/workflows/archiving-hour.yml | 1 + .github/workflows/update-all-logements.yml | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/.github/workflows/archiving-day.yml b/.github/workflows/archiving-day.yml index e766c162b..a7395aac9 100644 --- a/.github/workflows/archiving-day.yml +++ b/.github/workflows/archiving-day.yml @@ -19,6 +19,7 @@ jobs: with: ARCHIVE_MODE: DAY_SUM_UP LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + deploy: true secrets: token: ${{ secrets.GITHUB_TOKEN }} @@ -31,6 +32,7 @@ jobs: SPECIFIC_DAY: ${{ github.event.inputs.specific-date }} ARCHIVE_MODE: DAY_SUM_UP LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + deploy: true secrets: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/archiving-hour.yml b/.github/workflows/archiving-hour.yml index 83947dd0e..5369f2334 100644 --- a/.github/workflows/archiving-hour.yml +++ b/.github/workflows/archiving-hour.yml @@ -12,6 +12,7 @@ jobs: uses: ./.github/workflows/reusable-archiving.yml with: ARCHIVE_MODE: HOUR + deploy: true secrets: MAIL : ${{ secrets.TEST_MAIL }} PASSWORD : ${{ secrets.TEST_PASSWORD }} diff --git a/.github/workflows/update-all-logements.yml b/.github/workflows/update-all-logements.yml index fed01b0d9..f19ea9003 100644 --- a/.github/workflows/update-all-logements.yml +++ b/.github/workflows/update-all-logements.yml @@ -19,6 +19,8 @@ jobs: with: ARCHIVE_MODE: ALL_LOGEMENTS LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + target-folder: 'v1/logements-crous' + deploy: true secrets: token: ${{ secrets.GITHUB_TOKEN }} @@ -31,5 +33,7 @@ jobs: SPECIFIC_DAY: ${{ github.event.inputs.specific-date }} ARCHIVE_MODE: ALL_LOGEMENTS LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + target-folder: 'v1/logements-crous' + deploy: true secrets: token: ${{ secrets.GITHUB_TOKEN }} From ee2b67df58a95edded60f0eac380b3ef99eb5f41 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:42:13 +0000 Subject: [PATCH 50/71] Add workflow_call to Integration-test.yml and update setup-java version in release.yml resolves Refactor workflows #54 --- .github/workflows/Integration-test.yml | 1 + .github/workflows/release.yml | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index f0d3fa202..f72530d93 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -5,6 +5,7 @@ on: branches: [ master, beta ] pull_request: branches: [ master, beta ] + workflow_call: jobs: unit-test: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b26554dab..ddc1c99f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,11 +5,17 @@ on: - created jobs: + + tests: + uses: ./.github/workflows/Integration-test.yml + secrets: inherit + build: + needs: tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '17' From f0537face86005a3c17a92f1094b3cbc76a85253 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Fri, 26 Jan 2024 17:05:05 +0000 Subject: [PATCH 51/71] Fix API request error handling resolves Handle the 404 with the day archiving #52 --- .../collectors/RequestorToGetSumUpOfDay.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java index 82c40455f..aa55a17fc 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java @@ -47,7 +47,7 @@ public Logement[][] getSumUpOfDay(String url) { .map(link -> getFromUrl(link, context)) .toArray(Logement[][]::new); LOGGER.info(() -> "Logements received"); - } + } LOGGER.info(() -> "profil closed"); return sumUp; } @@ -55,8 +55,13 @@ public Logement[][] getSumUpOfDay(String url) { private static Logement[] getFromUrl(String url, APIRequestContext context) { LOGGER.info(() -> "Getting data from url: " + url); var respons = context.get(url); - if (!respons.ok()) - throw new ApiRequestErrorRuntimeException(respons); + if (!respons.ok()) { + if (respons.status() == 404) { + LOGGER.warning(() -> "No data found to url: " + url); + return new Logement[0]; + } else + throw new ApiRequestErrorRuntimeException(respons); + } LOGGER.info(() -> "Data received"); return Convertor.convertJsonToArrayOfLogements(respons.text()); } From 20953a9c8e021f2e88aee79744f577adadb7ddc1 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 13:52:11 +0000 Subject: [PATCH 52/71] Fix exception handling and add test case for site maintenance resolves Handle when the CROUS site is on maintenance #47 --- .../collectors/RequestorWithConnection.java | 18 +++++++-- .../exceptions/CannotBeConnectedError.java | 4 +- .../LoginOptionCantBeSelectedError.java | 4 +- .../RequestFailedRuntimeException.java | 17 ++++++++ .../RequestWithConnectionFailedException.java | 20 ++++++++++ .../SiteOnMaintenanceException.java | 13 +++++++ .../RequestorWithConnectionTest.java | 39 +++++++++++++++++++ 7 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java create mode 100644 src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java create mode 100644 src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java create mode 100644 src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java index 9af9d446b..10a797506 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java @@ -18,6 +18,8 @@ import io.github.mathieusoysal.exceptions.ApiRequestFailedRuntimeException; import io.github.mathieusoysal.exceptions.CannotBeConnectedError; import io.github.mathieusoysal.exceptions.LoginOptionCantBeSelectedError; +import io.github.mathieusoysal.exceptions.RequestFailedRuntimeException; +import io.github.mathieusoysal.exceptions.SiteOnMaintenanceException; class RequestorWithConnection implements Requestor { @@ -54,7 +56,10 @@ public String requestWitGet(String url) { LOGGER.error("Request failed", e); context.tracing().stop(new Tracing.StopOptions() .setPath(Paths.get("trace.zip"))); - throw e; + throw new RequestFailedRuntimeException(e); + } catch (SiteOnMaintenanceException e) { + LOGGER.warning(() -> "Site on maintenance"); + jsonLogements = "[]"; } finally { page.close(); context.close(); @@ -65,7 +70,8 @@ public String requestWitGet(String url) { return jsonLogements; } - private void etablishConnectionWithWebsite(Playwright playwright, BrowserContext context, Page page) { + private void etablishConnectionWithWebsite(Playwright playwright, BrowserContext context, Page page) + throws LoginOptionCantBeSelectedError, CannotBeConnectedError, SiteOnMaintenanceException { context.tracing().start(new Tracing.StartOptions() .setScreenshots(true) .setSnapshots(true) @@ -93,7 +99,8 @@ private static void goToLoginPage(Page page) { page.waitForLoadState(); } - private static void selectLoginOption(Playwright playwright, Page page) { + static void selectLoginOption(Playwright playwright, Page page) + throws LoginOptionCantBeSelectedError, SiteOnMaintenanceException { playwright.selectors().setTestIdAttribute("id"); String currentUrl = page.url(); LOGGER.info(() -> "Selecting login option"); @@ -109,12 +116,15 @@ private static void selectLoginOption(Playwright playwright, Page page) { waitForUrlChange(currentUrl, page); } catch (TimeoutError e) { LOGGER.error(() -> "Login option can't be selected"); + if (SiteOnMaintenanceException.isSiteOnMaintenance(page.content())) + throw new SiteOnMaintenanceException(page.content()); throw new LoginOptionCantBeSelectedError(e.getMessage(), page.content()); } LOGGER.info(() -> "Login option selected"); } - private static void connectToTheCrous(String email, String password, Playwright playwright, Page page) { + private static void connectToTheCrous(String email, String password, Playwright playwright, Page page) + throws CannotBeConnectedError { LOGGER.info(() -> "Connecting to the crous"); playwright.selectors().setTestIdAttribute("type"); String currentUrl = page.url(); diff --git a/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java b/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java index 6cbe29a5a..35716cf06 100644 --- a/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java +++ b/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java @@ -1,7 +1,7 @@ package io.github.mathieusoysal.exceptions; -public class CannotBeConnectedError extends Error { +public class CannotBeConnectedError extends RequestWithConnectionFailedException { public CannotBeConnectedError(String message, String html) { - super("Cannot be connected: " + message + " the html code:\n" + html); + super("Cannot be connected: " + message + " the html code:\n" + html, html); } } diff --git a/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java b/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java index e5d5f3f2a..d8f4e605b 100644 --- a/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java +++ b/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java @@ -1,7 +1,7 @@ package io.github.mathieusoysal.exceptions; -public class LoginOptionCantBeSelectedError extends Error { +public class LoginOptionCantBeSelectedError extends RequestWithConnectionFailedException { public LoginOptionCantBeSelectedError(String message, String html) { - super("Login option can't be selected: " + message + "The html code:\n" + html); + super("Login option can't be selected: " + message + "The html code:\n" + html, html); } } diff --git a/src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java b/src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java new file mode 100644 index 000000000..0afcbbc58 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java @@ -0,0 +1,17 @@ +package io.github.mathieusoysal.exceptions; + +public class RequestFailedRuntimeException extends RuntimeException { + + + public RequestFailedRuntimeException(String message) { + super(message); + } + + public RequestFailedRuntimeException(String message, Throwable cause) { + super(message, cause); + } + + public RequestFailedRuntimeException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java b/src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java new file mode 100644 index 000000000..cc4a5cad5 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java @@ -0,0 +1,20 @@ +package io.github.mathieusoysal.exceptions; + +public class RequestWithConnectionFailedException extends Exception { + + private final String html; + + public RequestWithConnectionFailedException(String message, String html) { + super("Request with connection failed: " + message); + this.html = html; + } + + public String getHtml() { + return html; + } + + public static boolean isCausedByRequestWithConnectionFailedException(Throwable throwable) { + return throwable instanceof RequestWithConnectionFailedException; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java b/src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java new file mode 100644 index 000000000..0a1b4f9e2 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java @@ -0,0 +1,13 @@ +package io.github.mathieusoysal.exceptions; + +public class SiteOnMaintenanceException extends RequestWithConnectionFailedException { + + public SiteOnMaintenanceException(String html) { + super("The site is on maintenance", html); + } + + public static boolean isSiteOnMaintenance(String html) { + return html.contains("Maintenance") || html.contains("maintenance"); + } + +} diff --git a/src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java b/src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java new file mode 100644 index 000000000..ec2e8df8a --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java @@ -0,0 +1,39 @@ +package io.github.mathieusoysal.data.managment.collectors; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; + +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import com.microsoft.playwright.Selectors; +import com.microsoft.playwright.TimeoutError; + +import io.github.mathieusoysal.exceptions.SiteOnMaintenanceException; + +public class RequestorWithConnectionTest { + + @Test + void testSelectLoginOption_withSiteOnMaintenance_throwsSiteOnMaintenanceException() { + // Arrange + Playwright pw = spy(Playwright.class); + Page page = mock(Page.class); + Selectors selectors = mock(Selectors.class); + + // Act + when(page.url()).thenReturn("https://www.demande-logement-social.gouv.fr/demande-en-ligne"); + when(page.locator(any())).thenThrow(new TimeoutError("")); + when(page.content()).thenReturn("Site en maintenance"); + when(pw.selectors()).thenReturn(selectors); + doNothing().when(selectors).setTestIdAttribute(anyString()); + + // Assert + assertThrows(SiteOnMaintenanceException.class, () -> RequestorWithConnection.selectLoginOption(pw, page)); + } +} From 91554ad5c23cd7b2342b49b1a9c1658c4d150083 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:16:24 +0000 Subject: [PATCH 53/71] Add test case for address with minus inside fix FIX Address bug #46 --- .../github/mathieusoysal/logement/AddressTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressTest.java index 66e8e7124..308ea37b0 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressTest.java @@ -18,6 +18,19 @@ void testGetGoodAttributes_withSimpleAddress() { Assertions.assertEquals(address.getFullAddress(), address2.getFullAddress()); } + @Test + void testGetGoodAttributes_withMinusInsideAddress() { + // Arrange + String addressString = "10 rue Henri Poincaré -13388 Marseille"; + + // Act + Address address = new Address(addressString, null); + Address address2 = new Address(address.getStreet(), address.getCity(), address.getZipCode(), null); + + // Assert + Assertions.assertEquals(address.getFullAddress(), address2.getFullAddress()); + } + @Test void testGetFullAddressFromAttributes() { // Arrange From bcb645b6e74d7e19642c06bc7db7aa83e75732f1 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:26:14 +0000 Subject: [PATCH 54/71] Fix regex pattern for street address fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtils.java | 2 +- .../io/github/mathieusoysal/logement/AddressUtilsTest.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java index 6b4faacb8..c66b6b11f 100644 --- a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java +++ b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java @@ -7,7 +7,7 @@ class AddressUtils { private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*$", Pattern.MULTILINE); private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d*(?=[a-zA-Z]*[^\\d]*$)", Pattern.MULTILINE); - private static final Pattern PATTERN_REGEX_STREET = Pattern.compile(".*[^ ](?= +[a-zA-Z]*[\\d]+ +[^\\d]*$)", + private static final Pattern PATTERN_REGEX_STREET = Pattern.compile(".*[^ ](?= +[a-zA-Z]*-?[\\d]+ +[^\\d]*$)", Pattern.MULTILINE); private AddressUtils() { diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index b66bc6635..d4f160a5f 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -9,6 +9,7 @@ class AddressUtilsTest { @ParameterizedTest @CsvSource(value = { + "Marseille;10 rue Henri Poincaré -13388 Marseille", "city de la garde;123 Main St 84450 city de la garde", "Belle -_éà@ç=$*ù+}ç_ç& city;123 Main St 84450 Belle -_éà@ç=$*ù+}ç_ç& city", "qsdsqddqssdqdqst qsd sqdsq sqd sqd sqd sqcity;123 Main St 84450 11841 qsdsqddqssdqdqst qsd sqdsq sqd sqd sqd sqcity", @@ -19,9 +20,9 @@ void testGetCityFromString(String expectedCity, String address) { assertEquals(expectedCity, city); } - @ParameterizedTest @CsvSource(value = { + "10 rue Henri Poincaré -13388 Marseille;10 rue Henri Poincaré", "123 Main St 84450 city de la garde;123 Main St", "123 - - - 8451 8451 Main St 84450 city;123 - - - 8451 8451 Main St", "123 Main St 84450 11841 city;123 Main St 84450", From 1f3bd4bccedfebf345f1061cef3003e00cdb6ddd Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:27:37 +0000 Subject: [PATCH 55/71] Fix address parsing in AddressUtilsTest fix FIX Address bug #46 --- .../io/github/mathieusoysal/logement/AddressUtilsTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index d4f160a5f..0f0a469e6 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -22,7 +22,7 @@ void testGetCityFromString(String expectedCity, String address) { @ParameterizedTest @CsvSource(value = { - "10 rue Henri Poincaré -13388 Marseille;10 rue Henri Poincaré", + "10 rue Henri Poincaré -13388 Marseille;10 rue Henri Poincaré", "123 Main St 84450 city de la garde;123 Main St", "123 - - - 8451 8451 Main St 84450 city;123 - - - 8451 8451 Main St", "123 Main St 84450 11841 city;123 Main St 84450", @@ -35,6 +35,7 @@ void testGetStreetFromString(String address, String expectedStreet) { @ParameterizedTest @CsvSource(value = { + "10 rue Henri Poincaré -13388 Marseille;13388", "123 Main St 84 city de la garde;84", "123 - - - 8451 8451 Main St 84450 city;84450", "123 Main St 84450 11841 city;11841", From d50ddf13246842dfb3e786264bbc3bbef33e3201 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:29:20 +0000 Subject: [PATCH 56/71] Add test case for NICE cedex2 address fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtilsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index 0f0a469e6..785ea5acb 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -9,6 +9,7 @@ class AddressUtilsTest { @ParameterizedTest @CsvSource(value = { + "NICE cedex2;96, avenue de Valrose - 06106 NICE cedex2", "Marseille;10 rue Henri Poincaré -13388 Marseille", "city de la garde;123 Main St 84450 city de la garde", "Belle -_éà@ç=$*ù+}ç_ç& city;123 Main St 84450 Belle -_éà@ç=$*ù+}ç_ç& city", From f9a78e133f3619e97a1fb61f6cdecab34f66dd58 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:40:27 +0000 Subject: [PATCH 57/71] Fix regex pattern for city and zip code matching fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java index c66b6b11f..48424ea16 100644 --- a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java +++ b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java @@ -4,7 +4,7 @@ import java.util.regex.Pattern; class AddressUtils { - private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*$", Pattern.MULTILINE); + private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*[\\d ]*$", Pattern.MULTILINE); private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d*(?=[a-zA-Z]*[^\\d]*$)", Pattern.MULTILINE); private static final Pattern PATTERN_REGEX_STREET = Pattern.compile(".*[^ ](?= +[a-zA-Z]*-?[\\d]+ +[^\\d]*$)", From b4ce49b7bcc1308590118ee2126a9e2caf322848 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 14:45:24 +0000 Subject: [PATCH 58/71] =?UTF-8?q?=F0=9F=A7=AA=20Add=20test=20case=20for=20?= =?UTF-8?q?address=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtilsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index 785ea5acb..a3753af02 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -23,6 +23,7 @@ void testGetCityFromString(String expectedCity, String address) { @ParameterizedTest @CsvSource(value = { + "96, avenue de Valrose - 06106 NICE cedex2;96, avenue de Valrose", "10 rue Henri Poincaré -13388 Marseille;10 rue Henri Poincaré", "123 Main St 84450 city de la garde;123 Main St", "123 - - - 8451 8451 Main St 84450 city;123 - - - 8451 8451 Main St", From 4d4e6cc53379873e2abd44921b1d077259d94702 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:33:49 +0000 Subject: [PATCH 59/71] =?UTF-8?q?=E2=9C=85=20Fix=20regex=20pattern=20for?= =?UTF-8?q?=20street=20matching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java index 48424ea16..751ca71d4 100644 --- a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java +++ b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java @@ -7,7 +7,7 @@ class AddressUtils { private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*[\\d ]*$", Pattern.MULTILINE); private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d*(?=[a-zA-Z]*[^\\d]*$)", Pattern.MULTILINE); - private static final Pattern PATTERN_REGEX_STREET = Pattern.compile(".*[^ ](?= +[a-zA-Z]*-?[\\d]+ +[^\\d]*$)", + private static final Pattern PATTERN_REGEX_STREET = Pattern.compile("^.*[^ \\-](?= +[a-zA-Z]*[- ]*[\\d]{4,} +[^\\d]*[\\d ]*$)", Pattern.MULTILINE); private AddressUtils() { From 6587440b486c8fc43fcb24607100a84a6e3cf80e Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:42:24 +0000 Subject: [PATCH 60/71] =?UTF-8?q?=F0=9F=A7=AA=20Add=20test=20case=20for=20?= =?UTF-8?q?address=20with=20zip=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtilsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index a3753af02..3081ff328 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -37,6 +37,7 @@ void testGetStreetFromString(String address, String expectedStreet) { @ParameterizedTest @CsvSource(value = { + "96, avenue de Valrose - 06106 NICE cedex2;06106", "10 rue Henri Poincaré -13388 Marseille;13388", "123 Main St 84 city de la garde;84", "123 - - - 8451 8451 Main St 84450 city;84450", From 65eafd0aa3bf161c4bcb0178e20c6f7aced5ac91 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:42:42 +0000 Subject: [PATCH 61/71] =?UTF-8?q?=E2=9C=85=20Fix=20regex=20pattern=20for?= =?UTF-8?q?=20zip=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix FIX Address bug #46 --- .../java/io/github/mathieusoysal/logement/AddressUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java index 751ca71d4..a5196c25c 100644 --- a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java +++ b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java @@ -5,7 +5,7 @@ class AddressUtils { private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*[\\d ]*$", Pattern.MULTILINE); - private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d*(?=[a-zA-Z]*[^\\d]*$)", + private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d+(?=[a-zA-Z]*[^\\d]*[\\d ]*$)", Pattern.MULTILINE); private static final Pattern PATTERN_REGEX_STREET = Pattern.compile("^.*[^ \\-](?= +[a-zA-Z]*[- ]*[\\d]{4,} +[^\\d]*[\\d ]*$)", Pattern.MULTILINE); From 5f424af37b1e2fc54ad9467ccf1ef4a5153b4176 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:46:24 +0000 Subject: [PATCH 62/71] =?UTF-8?q?=E2=9C=85=20Add=20test=20cases=20for=20ex?= =?UTF-8?q?tracting=20postal=20code=20from=20address?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/github/mathieusoysal/logement/AddressUtilsTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index 3081ff328..dcaf95ee7 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -9,6 +9,7 @@ class AddressUtilsTest { @ParameterizedTest @CsvSource(value = { + "Toulouse Cedex 4;118 Route de Narbonne CS 17705 31077 Toulouse Cedex 4", "NICE cedex2;96, avenue de Valrose - 06106 NICE cedex2", "Marseille;10 rue Henri Poincaré -13388 Marseille", "city de la garde;123 Main St 84450 city de la garde", @@ -23,6 +24,7 @@ void testGetCityFromString(String expectedCity, String address) { @ParameterizedTest @CsvSource(value = { + "118 Route de Narbonne CS 17705 31077 Toulouse Cedex 4;118 Route de Narbonne CS 17705", "96, avenue de Valrose - 06106 NICE cedex2;96, avenue de Valrose", "10 rue Henri Poincaré -13388 Marseille;10 rue Henri Poincaré", "123 Main St 84450 city de la garde;123 Main St", @@ -37,6 +39,7 @@ void testGetStreetFromString(String address, String expectedStreet) { @ParameterizedTest @CsvSource(value = { + "118 Route de Narbonne CS 17705 31077 Toulouse Cedex 4;31077", "96, avenue de Valrose - 06106 NICE cedex2;06106", "10 rue Henri Poincaré -13388 Marseille;13388", "123 Main St 84 city de la garde;84", From c35f5d37948a3ca9e69ece5be0b8fc556caff7b7 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 28 Jan 2024 11:16:03 +0000 Subject: [PATCH 63/71] Fix address format in testGetGoodAttributes_withMinusInsideAddress method --- src/test/java/io/github/mathieusoysal/logement/AddressTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressTest.java index 308ea37b0..1730ad127 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressTest.java @@ -21,7 +21,7 @@ void testGetGoodAttributes_withSimpleAddress() { @Test void testGetGoodAttributes_withMinusInsideAddress() { // Arrange - String addressString = "10 rue Henri Poincaré -13388 Marseille"; + String addressString = "10 rue Henri Poincaré 13388 Marseille"; // Act Address address = new Address(addressString, null); From 0b25ffe6ddbe8b8083fa010490f78f81c088ba88 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 29 Jan 2024 18:29:23 +0000 Subject: [PATCH 64/71] Add rerun-failed-jobs step to archiving-hour workflow resolves #55 --- .github/workflows/archiving-hour.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/archiving-hour.yml b/.github/workflows/archiving-hour.yml index 5369f2334..ce2285406 100644 --- a/.github/workflows/archiving-hour.yml +++ b/.github/workflows/archiving-hour.yml @@ -18,6 +18,16 @@ jobs: PASSWORD : ${{ secrets.TEST_PASSWORD }} token: ${{ secrets.GITHUB_TOKEN }} + rerun-failed-jobs: + runs-on: ubuntu-latest + needs: creating-archive + if: failure() + steps: + - name: Rerun failed jobs in the current workflow + env: + GH_TOKEN: ${{ github.token }} + run: gh run rerun ${{ github.run_id }} --failed + From 2668c737dd5a36fef1b707c837352f724aab4a90 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 4 Feb 2024 20:43:49 +0000 Subject: [PATCH 65/71] Refactor file and folder management resvoles Update the path archiving hour/day, to include ``available/` #56 --- .../archivers/ArchiverAllLogements.java | 11 +++- .../mathieusoysal/archivers/ArchiverDay.java | 9 ++- .../mathieusoysal/archivers/ArchiverHour.java | 10 +++- .../managment/savers/ArchivePathBuilder.java | 57 +++++++++++++++++++ .../data/managment/savers/DataSaver.java | 22 +------ .../data/managment/savers/FileManager.java | 2 +- .../data/managment/savers/FolderManager.java | 15 +++++ 7 files changed, 100 insertions(+), 26 deletions(-) create mode 100644 src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java index aaff14ce2..5471aabb6 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -5,11 +5,14 @@ import java.nio.file.Files; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.LocalDate; +import java.time.OffsetDateTime; import com.github.forax.beautifullogger.Logger; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; +import io.github.mathieusoysal.data.managment.savers.ArchivePathBuilder; import io.github.mathieusoysal.data.managment.savers.DataSaver; import io.github.mathieusoysal.logement.LogementsClassifier; @@ -25,7 +28,9 @@ public void archive() { private void updateHashOfAllLogement(File archivedFile) { var hash = getHashOfArchivedFile(archivedFile); - DataSaver.saveForGlobalArchive(ArchiveName.HASH_ALL_LOGEMENTS, hash); + ArchivePathBuilder + .startPath() + .endPathAndSaveData(ArchiveName.HASH_ALL_LOGEMENTS, hash); } private File archiveAllLogements() { @@ -33,7 +38,9 @@ private File archiveAllLogements() { var logements = new LogementsClassifier(); logements.addLogements(dataCollector.getAllLogements()); logements.addLogements(dataCollector.getConvertedSumUpOfDay(Archiver.getDayToArchive())); - return DataSaver.saveForGlobalArchive(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); + return ArchivePathBuilder + .startPath() + .endPathAndSaveData(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); } String getHashOfArchivedFile(File archivedFile) { diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java index 13d861369..2f63ad17e 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java @@ -1,8 +1,10 @@ package io.github.mathieusoysal.archivers; +import java.time.LocalDate; + import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.DataSaver; +import io.github.mathieusoysal.data.managment.savers.ArchivePathBuilder; public class ArchiverDay implements Archiver { @@ -10,7 +12,10 @@ public class ArchiverDay implements Archiver { public void archive() { var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); var sumUpOfTheDay = dataCollector.getSumUpOfDay(Archiver.getDayToArchive()); - DataSaver.saveForCurrentDay(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); + ArchivePathBuilder.startPath() + .addPath("available") + .addPath(LocalDate.now()) + .endPathAndSaveData(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); } diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java index 928810465..c69af1547 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java @@ -1,9 +1,11 @@ package io.github.mathieusoysal.archivers; +import java.time.LocalDate; + import io.github.mathieusoysal.Properties; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.DataSaver; +import io.github.mathieusoysal.data.managment.savers.ArchivePathBuilder; public class ArchiverHour implements Archiver { @@ -11,7 +13,11 @@ public class ArchiverHour implements Archiver { public void archive() { var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(Properties.MAIL.getValue(), Properties.PASSWORD.getValue()); - DataSaver.saveForCurrentDay(ArchiveName.HOUR, logements); + ArchivePathBuilder + .startPath() + .addPath("available") + .addPath(LocalDate.now()) + .endPathAndSaveData(ArchiveName.HOUR, logements); } } diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java new file mode 100644 index 000000000..a57482952 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java @@ -0,0 +1,57 @@ +package io.github.mathieusoysal.data.managment.savers; + +import java.io.File; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +import io.github.mathieusoysal.data.managment.convertors.Convertor; +import io.github.mathieusoysal.logement.Logement; + +public class ArchivePathBuilder { + + private Queue folders; + + public static ArchivePathBuilder startPath() { + var archivePath = new ArchivePathBuilder(); + archivePath.folders = new LinkedList<>(); + return archivePath; + } + + public ArchivePathBuilder addPath(final String name) { + var archivePath = new ArchivePathBuilder(); + archivePath.folders = new LinkedList<>(folders); + archivePath.folders.add(name); + return archivePath; + } + + public ArchivePathBuilder addPath(OffsetDateTime date) { + return addPath(date.format(DateTimeFormatter.ISO_LOCAL_DATE)); + } + + public ArchivePathBuilder addPath(LocalDate date) { + return addPath(date.format(DateTimeFormatter.ISO_LOCAL_DATE)); + } + + public File endPathAndSaveData(final ArchiveName name, final String archivedData) { + File archiveFile = generateArchiveFile(name); + DataSaver.writeDataInsideArchiveFile(archivedData, archiveFile); + return archiveFile; + } + + public File endPathAndSaveData(final ArchiveName name, final List logements) { + return endPathAndSaveData(name, Convertor.convertLogementsToJson(logements)); + } + + private File generateArchiveFile(final ArchiveName name) { + var folderAmbed = FolderManager.getOrCreateArchiveFolder(); + for (String folderName : folders) + folderAmbed = FolderManager.getOrCreateArchiveFolderWithGivenFolderName(folderName, folderAmbed); + var archiveFile = FileManager.getArchiveFile(folderAmbed, name.getName()); + return archiveFile; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java index 7e00ab0d0..438215d6b 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java @@ -14,29 +14,13 @@ public class DataSaver { private static final Logger LOGGER = Logger.getLogger(); - public static File saveForCurrentDay(ArchiveName archiveName, String content) { - File archiveFile = FileManager.getOrCreateArchiveFileForCurrentDay(archiveName); - writeDataInsideArchiveFile(content, archiveFile); - return archiveFile; - } - - public static File saveForCurrentDay(ArchiveName archiveName, List logements) { - String logementsJson = Convertor.convertLogementsToJson(logements); - return saveForCurrentDay(archiveName, logementsJson); - } - public static File saveForGlobalArchive(ArchiveName archiveName, List logements) { - String logementsJson = Convertor.convertLogementsToJson(logements); - return saveForGlobalArchive(archiveName, logementsJson); - } - public static File saveForGlobalArchive(ArchiveName archiveName, String content) { - File archiveFile = FileManager.getOrCreateArchiveFileForGlobal(archiveName); - writeDataInsideArchiveFile(content, archiveFile); - return archiveFile; + static void writeDataInsideArchiveFile(List logementsJson, File archiveFile) { + writeDataInsideArchiveFile(Convertor.convertLogementsToJson(logementsJson), archiveFile); } - private static void writeDataInsideArchiveFile(String logementsJson, File archiveFile) { + static void writeDataInsideArchiveFile(String logementsJson, File archiveFile) { LOGGER.info(() -> "Writing logements to file"); try (FileWriter fileWriter = new FileWriter(archiveFile)) { fileWriter.write(logementsJson); diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java index 01157cad6..a8aef4c07 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java @@ -21,7 +21,7 @@ static File getOrCreateArchiveFileForGlobal(ArchiveName archiveName) { return getArchiveFile(archiveFolder, archiveName.getName()); } - private static File getArchiveFile(File archiveFolder, String archiveFileName) throws DateTimeException { + static File getArchiveFile(File archiveFolder, String archiveFileName) throws DateTimeException { LOGGER.info(() -> "Getting archive file"); Stream.of(archiveFolder.listFiles()) .filter(file -> file.getName().equals(archiveFileName)) diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java index 8e86e942f..7ea0ca5ab 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java @@ -32,4 +32,19 @@ static File getOrCreateArchiveFolderWithGivenDate(OffsetDateTime date) { } return archiveFile; } + + static File getOrCreateArchiveFolderWithGivenFolderName(final String name, File file) { + LOGGER.info(() -> "Getting archive folder for current date"); + File archiveFile = new File(file, name); + if (!archiveFile.exists()) { + archiveFile.mkdir(); + LOGGER.info(() -> "Archive folder for current date created"); + } + return archiveFile; + } + + static File getOrCreateArchiveFolderWithGivenFolderName(final String name) { + File archiveFolder = getOrCreateArchiveFolder(); + return getOrCreateArchiveFolderWithGivenFolderName(name, archiveFolder); + } } From c0f19243667146f403334bad8bd4bf0b0b0c6e8a Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 4 Feb 2024 20:54:08 +0000 Subject: [PATCH 66/71] Update default target folder for archive deployment resolves #56 --- .github/workflows/reusable-archiving.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-archiving.yml b/.github/workflows/reusable-archiving.yml index ccf8f0f75..a339c27c7 100644 --- a/.github/workflows/reusable-archiving.yml +++ b/.github/workflows/reusable-archiving.yml @@ -27,7 +27,7 @@ on: description: 'The target folder to deploy the archive to github pages' required: false type: string - default: 'v1/logements-crous/available' + default: 'v1/logements-crous' secrets: MAIL: From 3ce459a26cd40e2b766ce6f58eb68beab6013055 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 4 Feb 2024 20:54:39 +0000 Subject: [PATCH 67/71] Refactor DataSaverTest to use ArchivePathBuilder resolves #56 --- .../data/managment/savers/DataSaverTest.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java index f43c643d0..cc6f850fd 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java @@ -5,7 +5,6 @@ import java.io.File; import java.io.IOException; import java.time.LocalDate; -import java.time.format.DateTimeFormatter; import java.util.List; import org.junit.jupiter.api.AfterEach; @@ -28,17 +27,23 @@ void tearDown() { @Test void testCreateArchiveLogements() throws ApiRequestFailedException, IOException { List logements = DataCollectorFromCrous.getAllLogementsWithoutConnection().stream().limit(2).toList(); - var file = assertDoesNotThrow(() -> DataSaver.saveForCurrentDay(ArchiveName.HOUR, logements)); - file.delete(); + assertDoesNotThrow(() -> ArchivePathBuilder.startPath().endPathAndSaveData(ArchiveName.HOUR, logements)); } @Test void testCreateArchiveLogementsForDay() throws ApiRequestFailedException, IOException { - System.out.println(LocalDate.of(2024, 1, 11).format(DateTimeFormatter.ISO_LOCAL_DATE)); + LocalDate chosenDate = LocalDate.of(2024, 1, 10); var dataCollector = new DataCollectorFromArchive( "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"); - String data = assertDoesNotThrow(() -> dataCollector.getSumUpOfDay(LocalDate.of(2024, 1, 10))); - assertDoesNotThrow(() -> DataSaver.saveForCurrentDay(ArchiveName.DAY_SUM_UP, data)); + String data = assertDoesNotThrow(() -> + dataCollector.getSumUpOfDay(chosenDate) + ); + assertDoesNotThrow( + () -> ArchivePathBuilder + .startPath() + .addPath("available") + .addPath(chosenDate) + .endPathAndSaveData(ArchiveName.DAY_SUM_UP, data)); } } From 8e10b4491cb84d9b503ec09cdc657fc66333c53b Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Sun, 4 Feb 2024 21:42:07 +0000 Subject: [PATCH 68/71] Remove Maven cache from integration test workflow --- .github/workflows/Integration-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml index f72530d93..23ea3d4c4 100644 --- a/.github/workflows/Integration-test.yml +++ b/.github/workflows/Integration-test.yml @@ -17,7 +17,6 @@ jobs: with: distribution: 'temurin' java-version: '17' - cache: 'maven' - name: Build & Install run: mvn -B install -D skipTests --no-transfer-progress - name: Install Playwright From 484249c30bc633eed4d4c57edffab68dca6e8d5f Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 5 Feb 2024 16:11:01 +0000 Subject: [PATCH 69/71] Refactor archive path builder to use ArchiveSaver --- .../archivers/ArchiverAllLogements.java | 9 +++------ .../mathieusoysal/archivers/ArchiverDay.java | 4 ++-- .../mathieusoysal/archivers/ArchiverHour.java | 4 ++-- .../{ArchivePathBuilder.java => ArchiveSaver.java} | 14 +++++++------- .../data/managment/savers/DataSaver.java | 4 +--- .../data/managment/savers/DataSaverTest.java | 4 ++-- 6 files changed, 17 insertions(+), 22 deletions(-) rename src/main/java/io/github/mathieusoysal/data/managment/savers/{ArchivePathBuilder.java => ArchiveSaver.java} (81%) diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java index 5471aabb6..137510974 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -5,15 +5,12 @@ import java.nio.file.Files; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.time.LocalDate; -import java.time.OffsetDateTime; import com.github.forax.beautifullogger.Logger; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.ArchivePathBuilder; -import io.github.mathieusoysal.data.managment.savers.DataSaver; +import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; import io.github.mathieusoysal.logement.LogementsClassifier; public class ArchiverAllLogements implements Archiver { @@ -28,7 +25,7 @@ public void archive() { private void updateHashOfAllLogement(File archivedFile) { var hash = getHashOfArchivedFile(archivedFile); - ArchivePathBuilder + ArchiveSaver .startPath() .endPathAndSaveData(ArchiveName.HASH_ALL_LOGEMENTS, hash); } @@ -38,7 +35,7 @@ private File archiveAllLogements() { var logements = new LogementsClassifier(); logements.addLogements(dataCollector.getAllLogements()); logements.addLogements(dataCollector.getConvertedSumUpOfDay(Archiver.getDayToArchive())); - return ArchivePathBuilder + return ArchiveSaver .startPath() .endPathAndSaveData(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); } diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java index 2f63ad17e..445e5faf0 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java @@ -4,7 +4,7 @@ import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.ArchivePathBuilder; +import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; public class ArchiverDay implements Archiver { @@ -12,7 +12,7 @@ public class ArchiverDay implements Archiver { public void archive() { var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); var sumUpOfTheDay = dataCollector.getSumUpOfDay(Archiver.getDayToArchive()); - ArchivePathBuilder.startPath() + ArchiveSaver.startPath() .addPath("available") .addPath(LocalDate.now()) .endPathAndSaveData(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java index c69af1547..02a747ef1 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java @@ -5,7 +5,7 @@ import io.github.mathieusoysal.Properties; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.ArchivePathBuilder; +import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; public class ArchiverHour implements Archiver { @@ -13,7 +13,7 @@ public class ArchiverHour implements Archiver { public void archive() { var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(Properties.MAIL.getValue(), Properties.PASSWORD.getValue()); - ArchivePathBuilder + ArchiveSaver .startPath() .addPath("available") .addPath(LocalDate.now()) diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveSaver.java similarity index 81% rename from src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java rename to src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveSaver.java index a57482952..5efc4cb02 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchivePathBuilder.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveSaver.java @@ -11,28 +11,28 @@ import io.github.mathieusoysal.data.managment.convertors.Convertor; import io.github.mathieusoysal.logement.Logement; -public class ArchivePathBuilder { +public class ArchiveSaver { private Queue folders; - public static ArchivePathBuilder startPath() { - var archivePath = new ArchivePathBuilder(); + public static ArchiveSaver startPath() { + var archivePath = new ArchiveSaver(); archivePath.folders = new LinkedList<>(); return archivePath; } - public ArchivePathBuilder addPath(final String name) { - var archivePath = new ArchivePathBuilder(); + public ArchiveSaver addPath(final String name) { + var archivePath = new ArchiveSaver(); archivePath.folders = new LinkedList<>(folders); archivePath.folders.add(name); return archivePath; } - public ArchivePathBuilder addPath(OffsetDateTime date) { + public ArchiveSaver addPath(OffsetDateTime date) { return addPath(date.format(DateTimeFormatter.ISO_LOCAL_DATE)); } - public ArchivePathBuilder addPath(LocalDate date) { + public ArchiveSaver addPath(LocalDate date) { return addPath(date.format(DateTimeFormatter.ISO_LOCAL_DATE)); } diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java index 438215d6b..749dfa47a 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java @@ -11,11 +11,9 @@ import io.github.mathieusoysal.exceptions.ImpossibleWriteRuntimeException; import io.github.mathieusoysal.logement.Logement; -public class DataSaver { +class DataSaver { private static final Logger LOGGER = Logger.getLogger(); - - static void writeDataInsideArchiveFile(List logementsJson, File archiveFile) { writeDataInsideArchiveFile(Convertor.convertLogementsToJson(logementsJson), archiveFile); } diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java index cc6f850fd..49b06a809 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java @@ -27,7 +27,7 @@ void tearDown() { @Test void testCreateArchiveLogements() throws ApiRequestFailedException, IOException { List logements = DataCollectorFromCrous.getAllLogementsWithoutConnection().stream().limit(2).toList(); - assertDoesNotThrow(() -> ArchivePathBuilder.startPath().endPathAndSaveData(ArchiveName.HOUR, logements)); + assertDoesNotThrow(() -> ArchiveSaver.startPath().endPathAndSaveData(ArchiveName.HOUR, logements)); } @Test @@ -39,7 +39,7 @@ void testCreateArchiveLogementsForDay() throws ApiRequestFailedException, IOExce dataCollector.getSumUpOfDay(chosenDate) ); assertDoesNotThrow( - () -> ArchivePathBuilder + () -> ArchiveSaver .startPath() .addPath("available") .addPath(chosenDate) From 391f28756e54ad13253fdae47a5f6ba28c11c863 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 5 Feb 2024 16:15:23 +0000 Subject: [PATCH 70/71] Add ArchiveSaver as a static field --- .../java/io/github/mathieusoysal/archivers/Archiver.java | 2 ++ .../mathieusoysal/archivers/ArchiverAllLogements.java | 7 ++----- .../io/github/mathieusoysal/archivers/ArchiverDay.java | 3 +-- .../io/github/mathieusoysal/archivers/ArchiverHour.java | 4 +--- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java index c90a94314..5808a8c2a 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -3,10 +3,12 @@ import java.time.LocalDate; import io.github.mathieusoysal.Properties; +import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; @FunctionalInterface public interface Archiver { public static final String DEFAULT_LINK_TO_ARCHIVE = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available"; + static final ArchiveSaver ARCHIVE_SAVER = ArchiveSaver.startPath(); static String getLinkToArchive() { if (Properties.LINK_TO_ARCHIVE.isPresent()) diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java index 137510974..569393836 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -10,7 +10,6 @@ import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; import io.github.mathieusoysal.logement.LogementsClassifier; public class ArchiverAllLogements implements Archiver { @@ -25,8 +24,7 @@ public void archive() { private void updateHashOfAllLogement(File archivedFile) { var hash = getHashOfArchivedFile(archivedFile); - ArchiveSaver - .startPath() + ARCHIVE_SAVER .endPathAndSaveData(ArchiveName.HASH_ALL_LOGEMENTS, hash); } @@ -35,8 +33,7 @@ private File archiveAllLogements() { var logements = new LogementsClassifier(); logements.addLogements(dataCollector.getAllLogements()); logements.addLogements(dataCollector.getConvertedSumUpOfDay(Archiver.getDayToArchive())); - return ArchiveSaver - .startPath() + return ARCHIVE_SAVER .endPathAndSaveData(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); } diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java index 445e5faf0..595add77d 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java @@ -4,7 +4,6 @@ import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; public class ArchiverDay implements Archiver { @@ -12,7 +11,7 @@ public class ArchiverDay implements Archiver { public void archive() { var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); var sumUpOfTheDay = dataCollector.getSumUpOfDay(Archiver.getDayToArchive()); - ArchiveSaver.startPath() + ARCHIVE_SAVER .addPath("available") .addPath(LocalDate.now()) .endPathAndSaveData(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java index 02a747ef1..7a7ecea3f 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java @@ -5,7 +5,6 @@ import io.github.mathieusoysal.Properties; import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; public class ArchiverHour implements Archiver { @@ -13,8 +12,7 @@ public class ArchiverHour implements Archiver { public void archive() { var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(Properties.MAIL.getValue(), Properties.PASSWORD.getValue()); - ArchiveSaver - .startPath() + ARCHIVE_SAVER .addPath("available") .addPath(LocalDate.now()) .endPathAndSaveData(ArchiveName.HOUR, logements); From c842d8d6352486b4d438bda323180735eb289401 Mon Sep 17 00:00:00 2001 From: Mathieu Soysal <43273304+MathieuSoysal@users.noreply.github.com> Date: Mon, 5 Feb 2024 16:16:08 +0000 Subject: [PATCH 71/71] Fix formatting in ArchiverDay class --- .../java/io/github/mathieusoysal/archivers/Archiver.java | 2 +- .../io/github/mathieusoysal/archivers/ArchiverDay.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java index 5808a8c2a..6403eb590 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -18,7 +18,7 @@ static String getLinkToArchive() { } static LocalDate getDayToArchive() { - if(Properties.SPECIFIC_DAY.isPresent()) + if (Properties.SPECIFIC_DAY.isPresent()) return LocalDate.parse(Properties.SPECIFIC_DAY.getValue()); else return LocalDate.now(); diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java index 595add77d..683d8b291 100644 --- a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java @@ -12,10 +12,9 @@ public void archive() { var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); var sumUpOfTheDay = dataCollector.getSumUpOfDay(Archiver.getDayToArchive()); ARCHIVE_SAVER - .addPath("available") - .addPath(LocalDate.now()) - .endPathAndSaveData(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); + .addPath("available") + .addPath(LocalDate.now()) + .endPathAndSaveData(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); } - }