diff --git a/.github/workflows/Integration-test.yml b/.github/workflows/Integration-test.yml new file mode 100644 index 000000000..23ea3d4c4 --- /dev/null +++ b/.github/workflows/Integration-test.yml @@ -0,0 +1,55 @@ +name: Integration Tests + +on: + push: + branches: [ master, beta ] + pull_request: + branches: [ master, beta ] + workflow_call: + +jobs: + unit-test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + - name: Build & Install + run: mvn -B install -D skipTests --no-transfer-progress + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + - name: Run tests + run: mvn test + env: + SPECIFIC_DAY: "2024-01-03" + ARCHIVE_MODE: HOUR + TEST_MAIL : ${{ secrets.TEST_MAIL }} + TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} + + creating-archive-with-connection: + needs: unit-test + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: HOUR + secrets: + MAIL : ${{ secrets.TEST_MAIL }} + PASSWORD : ${{ secrets.TEST_PASSWORD }} + + creating-archive-day-sum-up: + needs: unit-test + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: DAY_SUM_UP + SPECIFIC_DAY: "2024-01-03" + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + + creating-archive-all-logement: + needs: unit-test + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: ALL_LOGEMENTS + SPECIFIC_DAY: "2024-01-03" + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available \ No newline at end of file diff --git a/.github/workflows/archiving-day.yml b/.github/workflows/archiving-day.yml index fd48ff137..a7395aac9 100644 --- a/.github/workflows/archiving-day.yml +++ b/.github/workflows/archiving-day.yml @@ -1,37 +1,40 @@ -name: Archiving data +name: Archiving data for day summary on: schedule: - cron: '40 23 * * *' workflow_dispatch: + inputs: + specific-date: + description: 'Specific date to archive' + required: true + default: '2024-01-01' jobs: - creating-archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: - LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available - - uses: JamesIves/github-pages-deploy-action@v4.4.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - clean: false - branch: api - folder: archive - target-folder: v1/logements-crous/available + creating-archive-schedule: + permissions: + contents: write + if: github.event_name == 'schedule' + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: DAY_SUM_UP + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + deploy: true + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + creating-archive-manual: + permissions: + contents: write + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-archiving.yml + with: + SPECIFIC_DAY: ${{ github.event.inputs.specific-date }} + ARCHIVE_MODE: DAY_SUM_UP + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + deploy: true + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/archiving-hour.yml b/.github/workflows/archiving-hour.yml new file mode 100644 index 000000000..ce2285406 --- /dev/null +++ b/.github/workflows/archiving-hour.yml @@ -0,0 +1,34 @@ +name: Archiving data for hour + +on: + schedule: + - cron: '10 * * * *' + workflow_dispatch: + +jobs: + creating-archive: + permissions: + contents: write + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: HOUR + deploy: true + secrets: + MAIL : ${{ secrets.TEST_MAIL }} + PASSWORD : ${{ secrets.TEST_PASSWORD }} + token: ${{ secrets.GITHUB_TOKEN }} + + rerun-failed-jobs: + runs-on: ubuntu-latest + needs: creating-archive + if: failure() + steps: + - name: Rerun failed jobs in the current workflow + env: + GH_TOKEN: ${{ github.token }} + run: gh run rerun ${{ github.run_id }} --failed + + + + + \ No newline at end of file diff --git a/.github/workflows/archiving.yml b/.github/workflows/archiving.yml deleted file mode 100644 index 2ff6593a1..000000000 --- a/.github/workflows/archiving.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Archiving data - -on: - schedule: - - cron: '10 * * * *' - workflow_dispatch: - -jobs: - creating-archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: archive - run: | - (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ - (echo "==== Maven Deploy Step Failed ====" && exit 1) - env: - MAIL : ${{ secrets.TEST_MAIL }} - PASSWORD : ${{ secrets.TEST_PASSWORD }} - - uses: JamesIves/github-pages-deploy-action@v4.4.3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - clean: false - branch: api - folder: archive - target-folder: v1/logements-crous/available - - - - - \ No newline at end of file diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml deleted file mode 100644 index 731c5e7f1..000000000 --- a/.github/workflows/playwright.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Playwright Tests -on: - push: - branches: [ master, beta ] - pull_request: - branches: [ master, beta ] -jobs: - test: - timeout-minutes: 60 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - - name: Build & Install - run: mvn -B install -D skipTests --no-transfer-progress - - name: Install Playwright - run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" - - name: Run tests - run: mvn test - env: - TEST_MAIL : ${{ secrets.TEST_MAIL }} - TEST_PASSWORD : ${{ secrets.TEST_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 747a7b5ec..ddc1c99f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,11 +5,17 @@ on: - created jobs: + + tests: + uses: ./.github/workflows/Integration-test.yml + secrets: inherit + build: + needs: tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '17' @@ -47,6 +53,7 @@ jobs: - name: Build JIB container and publish to GitHub Packages run: | mvn package com.google.cloud.tools:jib-maven-plugin:3.3.2:build \ + -Djib.from.image=mcr.microsoft.com/playwright/java:v1.41.0-jammy \ -Djib.to.image=ghcr.io/${{ steps.downcase.outputs.lowercase }}:latest \ -Djib.to.auth.username=${{ github.actor }} \ -Djib.to.auth.password=${{ secrets.GITHUB_TOKEN }} $MULTI_MODULE_ARGS $MAIN_CLASS_ARGS diff --git a/.github/workflows/reusable-archiving.yml b/.github/workflows/reusable-archiving.yml new file mode 100644 index 000000000..a339c27c7 --- /dev/null +++ b/.github/workflows/reusable-archiving.yml @@ -0,0 +1,76 @@ +name: Archiving data with reusable workflow + +on: + workflow_call: + inputs: + ARCHIVE_MODE: + description: 'The mode of archiving' + required: true + type: string + default: 'HOUR' + LINK_TO_DATA: + description: 'The link to the data to archive' + required: false + type: string + default: 'https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available' + SPECIFIC_DAY: + description: 'The day to archive, day is used olny if ARCHIVE_MODE is DAY' + required: false + type: string + default: '' + deploy: + description: 'Deploy the archive to github pages' + required: false + type: boolean + default: false + target-folder: + description: 'The target folder to deploy the archive to github pages' + required: false + type: string + default: 'v1/logements-crous' + + secrets: + MAIL: + description: 'The mail to send the archive to hour, mail is used olny if ARCHIVE_MODE is HOUR' + PASSWORD: + description: 'The password of the mail to send the archive to hour, password is used olny if ARCHIVE_MODE is HOUR' + token: + description: 'The token to deploy the archive to github pages' + +jobs: + creating-archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + - name: Install Playwright + run: mvn exec:java -e -D exec.mainClass=com.microsoft.playwright.CLI -D exec.args="install --with-deps" + + - name: set specific day when input is not empty + if: inputs.SPECIFIC_DAY != '' + run: echo "SPECIFIC_DAY=${{ inputs.SPECIFIC_DAY }}" >> $GITHUB_ENV + + - name: archive + run: | + (echo "===== Maven Deploy Attempt: 1 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 2 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "===== Maven Deploy Attempt: 3 ====" && mvn install -DskipTests=true -Dmaven.javadoc.skip=true -PArchiving exec:java) || \ + (echo "==== Maven Deploy Step Failed ====" && exit 1) + env: + LINK_TO_DATA: ${{ inputs.LINK_TO_DATA }} + ARCHIVE_MODE: ${{ inputs.ARCHIVE_MODE }} + MAIL: ${{ secrets.MAIL }} + PASSWORD: ${{ secrets.PASSWORD }} + - name: Deploy to Github Pages + if: inputs.deploy == true + uses: JamesIves/github-pages-deploy-action@v4.4.3 + with: + token: ${{ secrets.token }} + clean: false + branch: api + folder: archive + target-folder: ${{ inputs.target-folder }} \ No newline at end of file diff --git a/.github/workflows/update-all-logements.yml b/.github/workflows/update-all-logements.yml new file mode 100644 index 000000000..f19ea9003 --- /dev/null +++ b/.github/workflows/update-all-logements.yml @@ -0,0 +1,39 @@ +name: Update all logements + +on: + schedule: + - cron: '40 23 * * *' + workflow_dispatch: + inputs: + specific-date: + description: 'Specific date to archive' + required: true + default: '2024-01-01' + +jobs: + creating-archive-schedule: + permissions: + contents: write + if: github.event_name == 'schedule' + uses: ./.github/workflows/reusable-archiving.yml + with: + ARCHIVE_MODE: ALL_LOGEMENTS + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + target-folder: 'v1/logements-crous' + deploy: true + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + creating-archive-manual: + permissions: + contents: write + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-archiving.yml + with: + SPECIFIC_DAY: ${{ github.event.inputs.specific-date }} + ARCHIVE_MODE: ALL_LOGEMENTS + LINK_TO_DATA: https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available + target-folder: 'v1/logements-crous' + deploy: true + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/main/java/io/github/mathieusoysal/App.java b/src/main/java/io/github/mathieusoysal/App.java index 663e7fa03..ed0b1612d 100644 --- a/src/main/java/io/github/mathieusoysal/App.java +++ b/src/main/java/io/github/mathieusoysal/App.java @@ -1,65 +1,15 @@ package io.github.mathieusoysal; -import java.io.IOException; -import java.time.LocalDate; - import com.github.forax.beautifullogger.Logger; -import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; -import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; -import io.github.mathieusoysal.data.managment.savers.ArchiveName; -import io.github.mathieusoysal.data.managment.savers.DataSaver; -import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; - public class App { private static final Logger LOGGER = Logger.getLogger(); - private static final String MAIL_PROPERTIES_NAME = "MAIL"; - private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD"; - private static final String LINK_TO_DATA_PROPERTIE_NAME = "LINK_TO_DATA"; - public static void main(String[] args) - throws IOException { + public static void main(String[] args) { LOGGER.info(() -> "Starting application"); - if (sumupdayModIsActivated()) - createArchiveSumUpForThisDay(); - else - createArchiveForThisHour(); - LOGGER.info(() -> "Application finished"); - } - - private static void createArchiveSumUpForThisDay() { - String linkToData = getPropertie(LINK_TO_DATA_PROPERTIE_NAME); - var dataCollector = new DataCollectorFromArchive(linkToData); - var sumUpOfTheDay = dataCollector.getSumUpOfDay(LocalDate.now()); - DataSaver.save(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); - } - - private static boolean sumupdayModIsActivated() { - return System.getenv(LINK_TO_DATA_PROPERTIE_NAME) != null; - } - - private static void createArchiveForThisHour() - throws IOException { - var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(getEmail(), getPassword()); - DataSaver.save(ArchiveName.HOUR, logements); - } - - private static String getPropertie(final String propertieName) { - LOGGER.info(() -> "Getting " + propertieName + " from environment variables"); - String propertie = System.getenv(propertieName); - if (propertie == null) { - LOGGER.error(() -> propertieName + " not found in environment variables"); - throw new PropertiesNotFoundRuntimeException(propertieName); - } - return propertie; - } - - private static String getEmail() { - return getPropertie(MAIL_PROPERTIES_NAME); - } - - private static String getPassword() { - return getPropertie(PASSWORD_PROPERTIES_NAME); + ArchiveMode archiveMod = ArchiveMode.getArchiveModeFromEnvironmentVariables(); + archiveMod.archive(); + LOGGER.info(() -> "Application successfully finished"); } } diff --git a/src/main/java/io/github/mathieusoysal/ArchiveMode.java b/src/main/java/io/github/mathieusoysal/ArchiveMode.java new file mode 100644 index 000000000..c43e238ad --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/ArchiveMode.java @@ -0,0 +1,40 @@ +package io.github.mathieusoysal; + +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.archivers.ArchiverAllLogements; +import io.github.mathieusoysal.archivers.ArchiverDay; +import io.github.mathieusoysal.archivers.ArchiverHour; +import io.github.mathieusoysal.archivers.Archiver; + +public enum ArchiveMode { + DAY_SUM_UP(new ArchiverDay()), + ALL_LOGEMENTS(new ArchiverAllLogements()), + HOUR(new ArchiverHour()); + + private static final Logger LOGGER = Logger.getLogger(); + private final Archiver archiver; + + private ArchiveMode(Archiver archiver) { + this.archiver = archiver; + } + + public void archive() { + archiver.archive(); + } + + public static ArchiveMode getArchiveMode(String archiveMod) { + try { + ArchiveMode m = ArchiveMode.valueOf(archiveMod); + LOGGER.info(() -> "Archive mode is " + m.name()); + return m; + } catch (IllegalArgumentException e) { + LOGGER.error(() -> archiveMod + " is not a valid archive mod"); + throw e; + } + } + + public static ArchiveMode getArchiveModeFromEnvironmentVariables() { + return getArchiveMode(Properties.ARCHIVE_MODE.getValue()); + } +} diff --git a/src/main/java/io/github/mathieusoysal/Properties.java b/src/main/java/io/github/mathieusoysal/Properties.java new file mode 100644 index 000000000..86ac952a9 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/Properties.java @@ -0,0 +1,38 @@ +package io.github.mathieusoysal; + +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException; + +public enum Properties { + LINK_TO_ARCHIVE("LINK_TO_ARCHIVE"), + MAIL("MAIL"), + PASSWORD("PASSWORD"), + ARCHIVE_MODE("ARCHIVE_MODE"), + SPECIFIC_DAY("SPECIFIC_DAY"); + + private static final Logger LOGGER = Logger.getLogger(); + private final String name; + + Properties(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public boolean isPresent() { + return System.getenv(name) != null; + } + + public String getValue() { + LOGGER.info(() -> "Getting " + name + " from environment variables"); + String propertie = System.getenv(name); + if (propertie == null) { + LOGGER.error(() -> name + " not found in environment variables"); + throw new PropertiesNotFoundRuntimeException(name); + } + return propertie; + } +} diff --git a/src/main/java/io/github/mathieusoysal/Utils.java b/src/main/java/io/github/mathieusoysal/Utils.java new file mode 100644 index 000000000..4fb73b1cb --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/Utils.java @@ -0,0 +1,10 @@ +package io.github.mathieusoysal; + +import java.util.HashSet; +import java.util.List; + +public class Utils { + public static boolean listEqualsIgnoreOrder(List list1, List list2) { + return new HashSet<>(list1).equals(new HashSet<>(list2)); + } +} diff --git a/src/main/java/io/github/mathieusoysal/archivers/Archiver.java b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java new file mode 100644 index 000000000..6403eb590 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/archivers/Archiver.java @@ -0,0 +1,28 @@ +package io.github.mathieusoysal.archivers; + +import java.time.LocalDate; + +import io.github.mathieusoysal.Properties; +import io.github.mathieusoysal.data.managment.savers.ArchiveSaver; + +@FunctionalInterface +public interface Archiver { + public static final String DEFAULT_LINK_TO_ARCHIVE = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available"; + static final ArchiveSaver ARCHIVE_SAVER = ArchiveSaver.startPath(); + + static String getLinkToArchive() { + if (Properties.LINK_TO_ARCHIVE.isPresent()) + return Properties.LINK_TO_ARCHIVE.getValue(); + else + return DEFAULT_LINK_TO_ARCHIVE; + } + + static LocalDate getDayToArchive() { + if (Properties.SPECIFIC_DAY.isPresent()) + return LocalDate.parse(Properties.SPECIFIC_DAY.getValue()); + else + return LocalDate.now(); + } + + public void archive(); +} diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java new file mode 100644 index 000000000..569393836 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverAllLogements.java @@ -0,0 +1,68 @@ +package io.github.mathieusoysal.archivers; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import com.github.forax.beautifullogger.Logger; + +import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; +import io.github.mathieusoysal.data.managment.savers.ArchiveName; +import io.github.mathieusoysal.logement.LogementsClassifier; + +public class ArchiverAllLogements implements Archiver { + + private static final Logger LOGGER = Logger.getLogger(); + + @Override + public void archive() { + var archivedFile = archiveAllLogements(); + updateHashOfAllLogement(archivedFile); + } + + private void updateHashOfAllLogement(File archivedFile) { + var hash = getHashOfArchivedFile(archivedFile); + ARCHIVE_SAVER + .endPathAndSaveData(ArchiveName.HASH_ALL_LOGEMENTS, hash); + } + + private File archiveAllLogements() { + var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); + var logements = new LogementsClassifier(); + logements.addLogements(dataCollector.getAllLogements()); + logements.addLogements(dataCollector.getConvertedSumUpOfDay(Archiver.getDayToArchive())); + return ARCHIVE_SAVER + .endPathAndSaveData(ArchiveName.ALL_LOGEMENTS, logements.getLogements()); + } + + String getHashOfArchivedFile(File archivedFile) { + String result = ""; + try { + byte[] digest = getHashedBytesFrom(archivedFile); + result = convertBytesToString(digest); + } catch (NoSuchAlgorithmException e) { + LOGGER.error("Error while getting hash of archived file the MD5 algorithm doesn't found in system", e); + System.exit(1); + } catch (IOException e) { + LOGGER.error("Error while reading archived file", e); + System.exit(1); + } + return result; + } + + private String convertBytesToString(byte[] digest) { + StringBuilder sb = new StringBuilder(); + for (byte b : digest) + sb.append(String.format("%02x", b)); + return sb.toString(); + } + + private byte[] getHashedBytesFrom(File archivedFile) throws NoSuchAlgorithmException, IOException { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(archivedFile.toPath())); + return md.digest(); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java new file mode 100644 index 000000000..683d8b291 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverDay.java @@ -0,0 +1,20 @@ +package io.github.mathieusoysal.archivers; + +import java.time.LocalDate; + +import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive; +import io.github.mathieusoysal.data.managment.savers.ArchiveName; + +public class ArchiverDay implements Archiver { + + @Override + public void archive() { + var dataCollector = new DataCollectorFromArchive(Archiver.getLinkToArchive()); + var sumUpOfTheDay = dataCollector.getSumUpOfDay(Archiver.getDayToArchive()); + ARCHIVE_SAVER + .addPath("available") + .addPath(LocalDate.now()) + .endPathAndSaveData(ArchiveName.DAY_SUM_UP, sumUpOfTheDay); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java new file mode 100644 index 000000000..7a7ecea3f --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/archivers/ArchiverHour.java @@ -0,0 +1,21 @@ +package io.github.mathieusoysal.archivers; + +import java.time.LocalDate; + +import io.github.mathieusoysal.Properties; +import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous; +import io.github.mathieusoysal.data.managment.savers.ArchiveName; + +public class ArchiverHour implements Archiver { + + @Override + public void archive() { + var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(Properties.MAIL.getValue(), + Properties.PASSWORD.getValue()); + ARCHIVE_SAVER + .addPath("available") + .addPath(LocalDate.now()) + .endPathAndSaveData(ArchiveName.HOUR, logements); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java index a9e5124ec..abdd189c6 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java @@ -1,6 +1,10 @@ package io.github.mathieusoysal.data.managment.collectors; import java.time.LocalDate; +import java.util.List; + +import io.github.mathieusoysal.data.managment.convertors.Convertor; +import io.github.mathieusoysal.logement.Logement; public class DataCollectorFromArchive { private final String archiveUrl; @@ -9,8 +13,17 @@ public DataCollectorFromArchive(String archiveUrl) { this.archiveUrl = archiveUrl; } + public List getAllLogements() { + String json = new RequestorToGetAllLogements().requestWitGet(archiveUrl); + return Convertor.convertJsonToListOfLogements(json); + } + public String getSumUpOfDay(LocalDate day) { return new RequestorToGetSumUpOfDay(day).requestWitGet(archiveUrl); } + + public Logement[][] getConvertedSumUpOfDay(LocalDate day) { + return new RequestorToGetSumUpOfDay(day).getSumUpOfDay(archiveUrl); + } } diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java index ddb7ac3bb..1e8cdeb73 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java @@ -1,6 +1,5 @@ package io.github.mathieusoysal.data.managment.collectors; -import java.io.IOException; import java.util.List; import com.github.forax.beautifullogger.Logger; @@ -12,23 +11,20 @@ public class DataCollectorFromCrous { private static final Logger LOGGER = Logger.getLogger(); private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32"; - public static List getAvailableLogementsWithoutConnection() - throws IOException { + public static List getAvailableLogementsWithoutConnection() { Requestor requestor = new RequestorWithoutConnection(); String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS); return Convertor.getLogementsFromBruteJsonString(jsonLogements); } - public static List getAllLogementsWithoutConnection() - throws IOException { + public static List getAllLogementsWithoutConnection() { LOGGER.info(() -> "Getting all logements"); Requestor requestor = new RequestorWithoutConnection(); String jsonLogements = requestor.requestWitGet("https://trouverunlogement.lescrous.fr/api/fr/search/29"); return Convertor.getLogementsFromBruteJsonString(jsonLogements); } - public static List getAvailableLogementsWithConnection(String email, String password) - throws IOException { + public static List getAvailableLogementsWithConnection(String email, String password) { Requestor requestor = new RequestorWithConnection(email, password); String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS); return Convertor.getLogementsFromBruteJsonString(jsonLogements); diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java new file mode 100644 index 000000000..ac561402f --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetAllLogements.java @@ -0,0 +1,31 @@ +package io.github.mathieusoysal.data.managment.collectors; + +import com.github.forax.beautifullogger.Logger; +import com.microsoft.playwright.Playwright; + +import io.github.mathieusoysal.exceptions.ApiRequestFailedRuntimeException; + +class RequestorToGetAllLogements implements Requestor { + private static final String ALL_LOGEMENTS = "/all_logements"; + private static final Logger LOGGER = Logger.getLogger(); + + @Override + public String requestWitGet(String url) { + LOGGER.info(() -> "Getting all logements from archive"); + String linkToDataForTheDay = url + ALL_LOGEMENTS; + String result; + LOGGER.info(() -> "Creating profil to request logements"); + try (Playwright playwright = Playwright.create()) { + LOGGER.info(() -> "profil created"); + var context = playwright.request().newContext(); + var respons = context.get(linkToDataForTheDay); + if (!respons.ok()) + throw new ApiRequestFailedRuntimeException(respons); + result = respons.text(); + LOGGER.info(() -> "Logements received"); + } + LOGGER.info(() -> "profil closed"); + return result; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java index 669e81257..aa55a17fc 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java @@ -30,6 +30,11 @@ public RequestorToGetSumUpOfDay(String date) { @Override public String requestWitGet(String url) { + var sumUp = getSumUpOfDay(url); + return Convertor.convertLogementMatrixToJson(sumUp); + } + + public Logement[][] getSumUpOfDay(String url) { LOGGER.info(() -> "Creating sum up of the day: " + date); String linkToDataForTheDay = url + "/" + date; Logement[][] sumUp; @@ -42,16 +47,21 @@ public String requestWitGet(String url) { .map(link -> getFromUrl(link, context)) .toArray(Logement[][]::new); LOGGER.info(() -> "Logements received"); - } + } LOGGER.info(() -> "profil closed"); - return Convertor.convertLogementMatrixToJson(sumUp); + return sumUp; } private static Logement[] getFromUrl(String url, APIRequestContext context) { LOGGER.info(() -> "Getting data from url: " + url); var respons = context.get(url); - if (!respons.ok()) - throw new ApiRequestErrorRuntimeException(respons); + if (!respons.ok()) { + if (respons.status() == 404) { + LOGGER.warning(() -> "No data found to url: " + url); + return new Logement[0]; + } else + throw new ApiRequestErrorRuntimeException(respons); + } LOGGER.info(() -> "Data received"); return Convertor.convertJsonToArrayOfLogements(respons.text()); } diff --git a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java index 9af9d446b..10a797506 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java @@ -18,6 +18,8 @@ import io.github.mathieusoysal.exceptions.ApiRequestFailedRuntimeException; import io.github.mathieusoysal.exceptions.CannotBeConnectedError; import io.github.mathieusoysal.exceptions.LoginOptionCantBeSelectedError; +import io.github.mathieusoysal.exceptions.RequestFailedRuntimeException; +import io.github.mathieusoysal.exceptions.SiteOnMaintenanceException; class RequestorWithConnection implements Requestor { @@ -54,7 +56,10 @@ public String requestWitGet(String url) { LOGGER.error("Request failed", e); context.tracing().stop(new Tracing.StopOptions() .setPath(Paths.get("trace.zip"))); - throw e; + throw new RequestFailedRuntimeException(e); + } catch (SiteOnMaintenanceException e) { + LOGGER.warning(() -> "Site on maintenance"); + jsonLogements = "[]"; } finally { page.close(); context.close(); @@ -65,7 +70,8 @@ public String requestWitGet(String url) { return jsonLogements; } - private void etablishConnectionWithWebsite(Playwright playwright, BrowserContext context, Page page) { + private void etablishConnectionWithWebsite(Playwright playwright, BrowserContext context, Page page) + throws LoginOptionCantBeSelectedError, CannotBeConnectedError, SiteOnMaintenanceException { context.tracing().start(new Tracing.StartOptions() .setScreenshots(true) .setSnapshots(true) @@ -93,7 +99,8 @@ private static void goToLoginPage(Page page) { page.waitForLoadState(); } - private static void selectLoginOption(Playwright playwright, Page page) { + static void selectLoginOption(Playwright playwright, Page page) + throws LoginOptionCantBeSelectedError, SiteOnMaintenanceException { playwright.selectors().setTestIdAttribute("id"); String currentUrl = page.url(); LOGGER.info(() -> "Selecting login option"); @@ -109,12 +116,15 @@ private static void selectLoginOption(Playwright playwright, Page page) { waitForUrlChange(currentUrl, page); } catch (TimeoutError e) { LOGGER.error(() -> "Login option can't be selected"); + if (SiteOnMaintenanceException.isSiteOnMaintenance(page.content())) + throw new SiteOnMaintenanceException(page.content()); throw new LoginOptionCantBeSelectedError(e.getMessage(), page.content()); } LOGGER.info(() -> "Login option selected"); } - private static void connectToTheCrous(String email, String password, Playwright playwright, Page page) { + private static void connectToTheCrous(String email, String password, Playwright playwright, Page page) + throws CannotBeConnectedError { LOGGER.info(() -> "Connecting to the crous"); playwright.selectors().setTestIdAttribute("type"); String currentUrl = page.url(); diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java index 5f5e2e379..941ecc9d7 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveName.java @@ -5,8 +5,10 @@ import java.util.function.Supplier; public enum ArchiveName { + ALL_LOGEMENTS(() -> "all_logements"), + HASH_ALL_LOGEMENTS(() -> "hash_all_logements"), DAY_SUM_UP(() -> "sum-up"), - HOUR(() -> OffsetDateTime.now().toLocalTime().format(DateTimeFormatter.ofPattern("HH"))); + HOUR(() -> OffsetDateTime.now().format(DateTimeFormatter.ofPattern("HH"))); private final Supplier nameSupplier; diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveSaver.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveSaver.java new file mode 100644 index 000000000..5efc4cb02 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/ArchiveSaver.java @@ -0,0 +1,57 @@ +package io.github.mathieusoysal.data.managment.savers; + +import java.io.File; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +import io.github.mathieusoysal.data.managment.convertors.Convertor; +import io.github.mathieusoysal.logement.Logement; + +public class ArchiveSaver { + + private Queue folders; + + public static ArchiveSaver startPath() { + var archivePath = new ArchiveSaver(); + archivePath.folders = new LinkedList<>(); + return archivePath; + } + + public ArchiveSaver addPath(final String name) { + var archivePath = new ArchiveSaver(); + archivePath.folders = new LinkedList<>(folders); + archivePath.folders.add(name); + return archivePath; + } + + public ArchiveSaver addPath(OffsetDateTime date) { + return addPath(date.format(DateTimeFormatter.ISO_LOCAL_DATE)); + } + + public ArchiveSaver addPath(LocalDate date) { + return addPath(date.format(DateTimeFormatter.ISO_LOCAL_DATE)); + } + + public File endPathAndSaveData(final ArchiveName name, final String archivedData) { + File archiveFile = generateArchiveFile(name); + DataSaver.writeDataInsideArchiveFile(archivedData, archiveFile); + return archiveFile; + } + + public File endPathAndSaveData(final ArchiveName name, final List logements) { + return endPathAndSaveData(name, Convertor.convertLogementsToJson(logements)); + } + + private File generateArchiveFile(final ArchiveName name) { + var folderAmbed = FolderManager.getOrCreateArchiveFolder(); + for (String folderName : folders) + folderAmbed = FolderManager.getOrCreateArchiveFolderWithGivenFolderName(folderName, folderAmbed); + var archiveFile = FileManager.getArchiveFile(folderAmbed, name.getName()); + return archiveFile; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java index 27e416bc4..749dfa47a 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/DataSaver.java @@ -11,21 +11,14 @@ import io.github.mathieusoysal.exceptions.ImpossibleWriteRuntimeException; import io.github.mathieusoysal.logement.Logement; -public class DataSaver { +class DataSaver { private static final Logger LOGGER = Logger.getLogger(); - public static File save(ArchiveName archiveName, String content) { - File archiveFile = FileManager.getOrCreateArchiveFile(archiveName); - writeDataInsideArchiveFile(content, archiveFile); - return archiveFile; + static void writeDataInsideArchiveFile(List logementsJson, File archiveFile) { + writeDataInsideArchiveFile(Convertor.convertLogementsToJson(logementsJson), archiveFile); } - public static File save(ArchiveName archiveName, List logements) { - String logementsJson = Convertor.convertLogementsToJson(logements); - return save(archiveName, logementsJson); - } - - private static void writeDataInsideArchiveFile(String logementsJson, File archiveFile) { + static void writeDataInsideArchiveFile(String logementsJson, File archiveFile) { LOGGER.info(() -> "Writing logements to file"); try (FileWriter fileWriter = new FileWriter(archiveFile)) { fileWriter.write(logementsJson); diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java index d61a4c61a..a8aef4c07 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FileManager.java @@ -11,13 +11,17 @@ class FileManager { private static final Logger LOGGER = Logger.getLogger(); - static File getOrCreateArchiveFile(ArchiveName archiveName) { + static File getOrCreateArchiveFileForCurrentDay(ArchiveName archiveName) { var archiveFolder = FolderManager.getOrCreateArchiveFolderWithCurrentDate(); - return getArchiveFile(archiveFolder, - archiveName.getName()); + return getArchiveFile(archiveFolder, archiveName.getName()); } - private static File getArchiveFile(File archiveFolder, String archiveFileName) throws DateTimeException { + static File getOrCreateArchiveFileForGlobal(ArchiveName archiveName) { + var archiveFolder = FolderManager.getOrCreateArchiveFolder(); + return getArchiveFile(archiveFolder, archiveName.getName()); + } + + static File getArchiveFile(File archiveFolder, String archiveFileName) throws DateTimeException { LOGGER.info(() -> "Getting archive file"); Stream.of(archiveFolder.listFiles()) .filter(file -> file.getName().equals(archiveFileName)) diff --git a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java index b100f9d0c..7ea0ca5ab 100644 --- a/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java +++ b/src/main/java/io/github/mathieusoysal/data/managment/savers/FolderManager.java @@ -7,7 +7,7 @@ class FolderManager { private static final java.util.logging.Logger LOGGER = java.util.logging.Logger .getLogger(FolderManager.class.getName()); - static File createArchiveFolder() { + static File getOrCreateArchiveFolder() { LOGGER.info(() -> "getting archive folder"); File archiveFolder = new File("archive"); if (!archiveFolder.exists()) { @@ -18,9 +18,13 @@ static File createArchiveFolder() { } static File getOrCreateArchiveFolderWithCurrentDate() { + return getOrCreateArchiveFolderWithGivenDate(OffsetDateTime.now()); + } + + static File getOrCreateArchiveFolderWithGivenDate(OffsetDateTime date) { LOGGER.info(() -> "Getting archive folder for current date"); - File archiveFolder = createArchiveFolder(); - String archiveFolderName = OffsetDateTime.now().toLocalDate().toString(); + File archiveFolder = getOrCreateArchiveFolder(); + String archiveFolderName = date.format(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE); File archiveFile = new File(archiveFolder, archiveFolderName); if (!archiveFile.exists()) { archiveFile.mkdir(); @@ -28,4 +32,19 @@ static File getOrCreateArchiveFolderWithCurrentDate() { } return archiveFile; } + + static File getOrCreateArchiveFolderWithGivenFolderName(final String name, File file) { + LOGGER.info(() -> "Getting archive folder for current date"); + File archiveFile = new File(file, name); + if (!archiveFile.exists()) { + archiveFile.mkdir(); + LOGGER.info(() -> "Archive folder for current date created"); + } + return archiveFile; + } + + static File getOrCreateArchiveFolderWithGivenFolderName(final String name) { + File archiveFolder = getOrCreateArchiveFolder(); + return getOrCreateArchiveFolderWithGivenFolderName(name, archiveFolder); + } } diff --git a/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java b/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java index 6cbe29a5a..35716cf06 100644 --- a/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java +++ b/src/main/java/io/github/mathieusoysal/exceptions/CannotBeConnectedError.java @@ -1,7 +1,7 @@ package io.github.mathieusoysal.exceptions; -public class CannotBeConnectedError extends Error { +public class CannotBeConnectedError extends RequestWithConnectionFailedException { public CannotBeConnectedError(String message, String html) { - super("Cannot be connected: " + message + " the html code:\n" + html); + super("Cannot be connected: " + message + " the html code:\n" + html, html); } } diff --git a/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java b/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java index e5d5f3f2a..d8f4e605b 100644 --- a/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java +++ b/src/main/java/io/github/mathieusoysal/exceptions/LoginOptionCantBeSelectedError.java @@ -1,7 +1,7 @@ package io.github.mathieusoysal.exceptions; -public class LoginOptionCantBeSelectedError extends Error { +public class LoginOptionCantBeSelectedError extends RequestWithConnectionFailedException { public LoginOptionCantBeSelectedError(String message, String html) { - super("Login option can't be selected: " + message + "The html code:\n" + html); + super("Login option can't be selected: " + message + "The html code:\n" + html, html); } } diff --git a/src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java b/src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java new file mode 100644 index 000000000..0afcbbc58 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/RequestFailedRuntimeException.java @@ -0,0 +1,17 @@ +package io.github.mathieusoysal.exceptions; + +public class RequestFailedRuntimeException extends RuntimeException { + + + public RequestFailedRuntimeException(String message) { + super(message); + } + + public RequestFailedRuntimeException(String message, Throwable cause) { + super(message, cause); + } + + public RequestFailedRuntimeException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java b/src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java new file mode 100644 index 000000000..cc4a5cad5 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/RequestWithConnectionFailedException.java @@ -0,0 +1,20 @@ +package io.github.mathieusoysal.exceptions; + +public class RequestWithConnectionFailedException extends Exception { + + private final String html; + + public RequestWithConnectionFailedException(String message, String html) { + super("Request with connection failed: " + message); + this.html = html; + } + + public String getHtml() { + return html; + } + + public static boolean isCausedByRequestWithConnectionFailedException(Throwable throwable) { + return throwable instanceof RequestWithConnectionFailedException; + } + +} diff --git a/src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java b/src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java new file mode 100644 index 000000000..0a1b4f9e2 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/exceptions/SiteOnMaintenanceException.java @@ -0,0 +1,13 @@ +package io.github.mathieusoysal.exceptions; + +public class SiteOnMaintenanceException extends RequestWithConnectionFailedException { + + public SiteOnMaintenanceException(String html) { + super("The site is on maintenance", html); + } + + public static boolean isSiteOnMaintenance(String html) { + return html.contains("Maintenance") || html.contains("maintenance"); + } + +} diff --git a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java index 6b4faacb8..a5196c25c 100644 --- a/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java +++ b/src/main/java/io/github/mathieusoysal/logement/AddressUtils.java @@ -4,10 +4,10 @@ import java.util.regex.Pattern; class AddressUtils { - private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*$", Pattern.MULTILINE); - private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d*(?=[a-zA-Z]*[^\\d]*$)", + private static final Pattern PATTERN_REGEX_CITY = Pattern.compile("[^ \\d]+[^\\d]*[\\d ]*$", Pattern.MULTILINE); + private static final Pattern PATTER_REGEX_ZIP_CODE = Pattern.compile("\\d+(?=[a-zA-Z]*[^\\d]*[\\d ]*$)", Pattern.MULTILINE); - private static final Pattern PATTERN_REGEX_STREET = Pattern.compile(".*[^ ](?= +[a-zA-Z]*[\\d]+ +[^\\d]*$)", + private static final Pattern PATTERN_REGEX_STREET = Pattern.compile("^.*[^ \\-](?= +[a-zA-Z]*[- ]*[\\d]{4,} +[^\\d]*[\\d ]*$)", Pattern.MULTILINE); private AddressUtils() { diff --git a/src/main/java/io/github/mathieusoysal/logement/Logement.java b/src/main/java/io/github/mathieusoysal/logement/Logement.java index 8566558fd..69bfb1fb5 100644 --- a/src/main/java/io/github/mathieusoysal/logement/Logement.java +++ b/src/main/java/io/github/mathieusoysal/logement/Logement.java @@ -1,6 +1,9 @@ package io.github.mathieusoysal.logement; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Logement @@ -26,13 +29,12 @@ public class Logement { private List equipements; private double areaMin; private double areaMax; - private List occupationMods; + private Set occupationMods; private List transports; - // Create constructor - public Logement() - {} + public Logement() { + } /** * Constructs a new Logement object with the specified ID. @@ -56,22 +58,42 @@ public int hashCode() { return result; } - /** - * Indicates whether some other object is "equal to" this one. - * - * @param obj the reference object with which to compare - * @return true if this object is the same as the obj argument; false otherwise - */ @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) + if (obj == null) + return false; + if (!(obj instanceof Logement)) return false; Logement other = (Logement) obj; return id == other.id; } + /** + * Adds the specified Logement object to this Logement object. + * + * @param logement the Logement object to be added + */ + public void addLogement(Logement logement) { + if (logement == null) + return; + label = logement.getLabel(); + address = logement.getAddress(); + bedCount = logement.getBedCount(); + bedKind = logement.getBedKind(); + bedroomCount = logement.getBedroomCount(); + roomCount = logement.getRoomCount(); + inUnavailabilityPeriod = logement.isInUnavailabilityPeriod(); + description = logement.getDescription(); + available = logement.isAvailable(); + highDemand = logement.isHighDemand(); + lowStock = logement.isLowStock(); + equipements = logement.getEquipements(); + areaMin = logement.getAreaMin(); + areaMax = logement.getAreaMax(); + occupationMods.addAll(logement.occupationMods); + transports = logement.getTransports(); + } + /** * Returns the ID of this Logement object. * @@ -214,7 +236,7 @@ public double getAreaMax() { * @return the list of occupation modifications for this Logement object */ public List getOccupationMods() { - return occupationMods; + return new ArrayList<>(occupationMods); } /** @@ -297,7 +319,7 @@ public void setAreaMax(double areaMax) { } public void setOccupationMods(List occupationMods) { - this.occupationMods = occupationMods; + this.occupationMods = new HashSet<>(occupationMods); } public void setTransports(List transports) { diff --git a/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java b/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java new file mode 100644 index 000000000..50cbeb489 --- /dev/null +++ b/src/main/java/io/github/mathieusoysal/logement/LogementsClassifier.java @@ -0,0 +1,47 @@ +package io.github.mathieusoysal.logement; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class LogementsClassifier { + + private Map logements; + + public LogementsClassifier(Set logements) { + this.logements = logements + .stream() + .collect(Collectors.toMap(Logement::getId, logement -> logement)); + } + + public LogementsClassifier() { + this.logements = new HashMap<>(); + } + + public void addLogement(Logement logement) { + if (!logements.containsKey(logement.getId())) + logements.put(logement.getId(), logement); + else { + Logement existingLogement = logements.get(logement.getId()); + existingLogement.addLogement(logement); + } + } + + public void addLogements(Collection logements) { + logements.forEach(this::addLogement); + } + + public void addLogements(Logement[][] logements) { + for (Logement[] logements2 : logements) + for (Logement logement : logements2) + addLogement(logement); + } + + public List getLogements() { + return new ArrayList<>(logements.values()); + } +} \ No newline at end of file diff --git a/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java b/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java index 77ec30c70..345db7e49 100644 --- a/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java +++ b/src/main/java/io/github/mathieusoysal/logement/OccupationMode.java @@ -62,4 +62,26 @@ public int getRentMin() { public int getRentMax() { return rentMax; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((occupationKind == null) ? 0 : occupationKind.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + OccupationMode other = (OccupationMode) obj; + return occupationKind == other.occupationKind; + } + + } diff --git a/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java b/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java index a9ed3d126..7cea44fbe 100644 --- a/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java +++ b/src/main/java/io/github/mathieusoysal/logement/pojo/Convertor.java @@ -21,18 +21,30 @@ public class Convertor { private Convertor() { } - static List getItemsFromJsonFile(File file) throws IOException { - LOGGER.info(() -> "Reading json file for convertion to java object"); - ObjectMapper objectMapper = new ObjectMapper(); - Input results = objectMapper.readValue(file, Input.class); + static List getItemsFromJsonFile(File file) { + Input results; + try { + LOGGER.info(() -> "Reading json file for convertion to java object"); + ObjectMapper objectMapper = new ObjectMapper(); + results = objectMapper.readValue(file, Input.class); + } catch (IOException e) { + LOGGER.error(() -> "Error while reading json file"); + throw new ConvertorException("Error while reading json file", e); + } LOGGER.info(() -> "Json file converted to java object"); return results.getResults().getItems(); } - static List getItemsFromJsonString(String json) throws IOException { - LOGGER.info(() -> "Reading json string for convertion to java object"); - ObjectMapper objectMapper = new ObjectMapper(); - Input results = objectMapper.readValue(json, Input.class); + static List getItemsFromJsonString(String json) { + Input results; + try { + LOGGER.info(() -> "Reading json string for convertion to java object"); + ObjectMapper objectMapper = new ObjectMapper(); + results = objectMapper.readValue(json, Input.class); + } catch (IOException e) { + LOGGER.error(() -> "Error while reading json file"); + throw new ConvertorException("Error while reading json file", e); + } LOGGER.info(() -> "Json string converted to java object"); return results.getResults().getItems(); } @@ -44,14 +56,12 @@ static List convertItemsToLogements(List items) { return result; } - public static List getLogementsFromBruteJsonFile(File file) - throws IOException { + public static List getLogementsFromBruteJsonFile(File file) { List items = getItemsFromJsonFile(file); return convertItemsToLogements(items); } - public static List getLogementsFromBruteJsonString(String json) - throws IOException { + public static List getLogementsFromBruteJsonString(String json) { List items = getItemsFromJsonString(json); return convertItemsToLogements(items); } @@ -107,4 +117,10 @@ private static List getTransports(It .toList(); } + static class ConvertorException extends RuntimeException { + public ConvertorException(String message, Throwable cause) { + super(message, cause); + } + } + } diff --git a/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java b/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java new file mode 100644 index 000000000..e2237ccc3 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/ArchiveModeTest.java @@ -0,0 +1,35 @@ +package io.github.mathieusoysal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class ArchiveModeTest { + + @Test + void testGetArchiveMod_shouldReturnHour_whenArchiveModIsHour() { + ArchiveMode archiveMode = ArchiveMode.getArchiveMode("HOUR"); + assertEquals(ArchiveMode.HOUR, archiveMode); + } + + @Test + void testGetArchiveMod_shouldReturnDay_whenArchiveModIsDay() { + ArchiveMode archiveMode = ArchiveMode.getArchiveMode("DAY_SUM_UP"); + assertEquals(ArchiveMode.DAY_SUM_UP, archiveMode); + } + + @Test + void testGetArchiveMod_shouldReturnWeek_whenArchiveModIsWeek() { + ArchiveMode archiveMode = ArchiveMode.getArchiveMode("ALL_LOGEMENTS"); + assertEquals(ArchiveMode.ALL_LOGEMENTS, archiveMode); + } + + + @Test + void testGetArchiveMod_shouldLogErrorAndExit_whenArchiveModIsInvalid() { + assertThrows(IllegalArgumentException.class, () -> { + ArchiveMode.getArchiveMode("INVALID"); + }); + } +} \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java b/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java new file mode 100644 index 000000000..2d68d7c0f --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/archivers/ArchiverAllLogementsTest.java @@ -0,0 +1,44 @@ +package io.github.mathieusoysal.archivers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +class ArchiverAllLogementsTest { + private static File jsonTestFile1 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash1.txt"); + private static File jsonTestFile2 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash2.txt"); + private static File jsonTestFile3 = new File("src/test/java/io/github/mathieusoysal/resources/test-hash3.txt"); + + @Test + @EnabledIfEnvironmentVariable(named = "CI", matches = "true") + void testArchive_shouldCallArchiveAllLogementsAndUpdateHashOfAllLogement() throws NoSuchAlgorithmException, IOException { + ArchiverAllLogements archiverAllLogements = new ArchiverAllLogements(); + + archiverAllLogements.archive(); + + assertTrue(new File("archive/all_logements").exists()); + assertTrue(new File("archive/hash_all_logements").exists()); + } + + @Test + void testGetHashOfArchivedFile_shouldBeEqual_IfContentIsEquals() throws NoSuchAlgorithmException, IOException { + ArchiverAllLogements archiverAllLogements = new ArchiverAllLogements(); + String hash1 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile3); + String hash2 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile2); + assertEquals(hash1, hash2); + } + + @Test + void testGetHashOfArchivedFile_shouldBeDifferent_IfContentIsDifferent() throws NoSuchAlgorithmException, IOException { + ArchiverAllLogements archiverAllLogements = new ArchiverAllLogements(); + String hash1 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile1); + String hash2 = archiverAllLogements.getHashOfArchivedFile(jsonTestFile2); + assertEquals(false, hash1.equals(hash2)); + } +} \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java b/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java index ae2a7292d..96ed12949 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorTest.java @@ -10,6 +10,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.exc.StreamReadException; @@ -64,6 +65,7 @@ void testConvertion() throws StreamReadException, DatabindException, ApiRequestF } @Test + @EnabledIfEnvironmentVariable(named = "CI", matches = "true") void testGetLogementsWithConnection_returnsLogements() throws StreamReadException, DatabindException, ApiRequestFailedException, IOException, InterruptedException { @@ -93,4 +95,11 @@ void testCreateSumUpOfTheDay() throws JsonProcessingException { dataCollectorFromArchive.getSumUpOfDay(date);}); } + @Test + void testGetAllLogements() + { + String linkToData = "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available"; + assertDoesNotThrow(() -> new DataCollectorFromArchive(linkToData).getAllLogements()); + } + } \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java b/src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java new file mode 100644 index 000000000..ec2e8df8a --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnectionTest.java @@ -0,0 +1,39 @@ +package io.github.mathieusoysal.data.managment.collectors; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; + +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import com.microsoft.playwright.Selectors; +import com.microsoft.playwright.TimeoutError; + +import io.github.mathieusoysal.exceptions.SiteOnMaintenanceException; + +public class RequestorWithConnectionTest { + + @Test + void testSelectLoginOption_withSiteOnMaintenance_throwsSiteOnMaintenanceException() { + // Arrange + Playwright pw = spy(Playwright.class); + Page page = mock(Page.class); + Selectors selectors = mock(Selectors.class); + + // Act + when(page.url()).thenReturn("https://www.demande-logement-social.gouv.fr/demande-en-ligne"); + when(page.locator(any())).thenThrow(new TimeoutError("")); + when(page.content()).thenReturn("Site en maintenance"); + when(pw.selectors()).thenReturn(selectors); + doNothing().when(selectors).setTestIdAttribute(anyString()); + + // Assert + assertThrows(SiteOnMaintenanceException.class, () -> RequestorWithConnection.selectLoginOption(pw, page)); + } +} diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java index 1a0c69855..49b06a809 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/DataSaverTest.java @@ -24,18 +24,26 @@ void tearDown() { archiveFolder.delete(); } - @Test void testCreateArchiveLogements() throws ApiRequestFailedException, IOException { List logements = DataCollectorFromCrous.getAllLogementsWithoutConnection().stream().limit(2).toList(); - var file = assertDoesNotThrow(() -> DataSaver.save(ArchiveName.HOUR, logements)); - file.delete(); + assertDoesNotThrow(() -> ArchiveSaver.startPath().endPathAndSaveData(ArchiveName.HOUR, logements)); } @Test void testCreateArchiveLogementsForDay() throws ApiRequestFailedException, IOException { - var dataCollector = new DataCollectorFromArchive("https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"); - assertDoesNotThrow(() -> dataCollector.getSumUpOfDay(LocalDate.of(2024, 1, 3))); + LocalDate chosenDate = LocalDate.of(2024, 1, 10); + var dataCollector = new DataCollectorFromArchive( + "https://mathieusoysal.github.io/CROUS-assistant-Collector/v1/logements-crous/available/"); + String data = assertDoesNotThrow(() -> + dataCollector.getSumUpOfDay(chosenDate) + ); + assertDoesNotThrow( + () -> ArchiveSaver + .startPath() + .addPath("available") + .addPath(chosenDate) + .endPathAndSaveData(ArchiveName.DAY_SUM_UP, data)); } } diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java new file mode 100644 index 000000000..8298cb642 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/FileManagerTest.java @@ -0,0 +1,18 @@ +package io.github.mathieusoysal.data.managment.savers; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; + +import org.junit.jupiter.api.Test; + + +class FileManagerTest { + + @Test + void testGetOrCreateArchiveFile() { + // Assert that the file has the correct name + assertEquals(OffsetDateTime.now().format(DateTimeFormatter.ofPattern("HH")), ArchiveName.HOUR.getName()); + } +} \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java b/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java index 025e13346..63768d72a 100644 --- a/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java +++ b/src/test/java/io/github/mathieusoysal/data/managment/savers/FolderManagerTest.java @@ -4,18 +4,28 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; +import java.time.OffsetDateTime; import org.junit.jupiter.api.Test; -public class FolderManagerTest { +class FolderManagerTest { @Test void testCreateArchiveFolder() { - File archiveFolder = FolderManager.createArchiveFolder(); + File archiveFolder = FolderManager.getOrCreateArchiveFolder(); assertTrue(archiveFolder.exists()); assertTrue(archiveFolder.isDirectory()); assertEquals("archive", archiveFolder.getName()); archiveFolder.delete(); } -} + + @Test + void testGetOrCreateArchiveFolderWithGivenDate() { + OffsetDateTime date = OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, OffsetDateTime.now().getOffset()); + + File actualArchiveFile = FolderManager.getOrCreateArchiveFolderWithGivenDate(date); + + assertEquals("2024-01-01", actualArchiveFile.getName()); + } +} \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressTest.java index 66e8e7124..1730ad127 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressTest.java @@ -18,6 +18,19 @@ void testGetGoodAttributes_withSimpleAddress() { Assertions.assertEquals(address.getFullAddress(), address2.getFullAddress()); } + @Test + void testGetGoodAttributes_withMinusInsideAddress() { + // Arrange + String addressString = "10 rue Henri Poincaré 13388 Marseille"; + + // Act + Address address = new Address(addressString, null); + Address address2 = new Address(address.getStreet(), address.getCity(), address.getZipCode(), null); + + // Assert + Assertions.assertEquals(address.getFullAddress(), address2.getFullAddress()); + } + @Test void testGetFullAddressFromAttributes() { // Arrange diff --git a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java index b66bc6635..dcaf95ee7 100644 --- a/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java +++ b/src/test/java/io/github/mathieusoysal/logement/AddressUtilsTest.java @@ -9,6 +9,9 @@ class AddressUtilsTest { @ParameterizedTest @CsvSource(value = { + "Toulouse Cedex 4;118 Route de Narbonne CS 17705 31077 Toulouse Cedex 4", + "NICE cedex2;96, avenue de Valrose - 06106 NICE cedex2", + "Marseille;10 rue Henri Poincaré -13388 Marseille", "city de la garde;123 Main St 84450 city de la garde", "Belle -_éà@ç=$*ù+}ç_ç& city;123 Main St 84450 Belle -_éà@ç=$*ù+}ç_ç& city", "qsdsqddqssdqdqst qsd sqdsq sqd sqd sqd sqcity;123 Main St 84450 11841 qsdsqddqssdqdqst qsd sqdsq sqd sqd sqd sqcity", @@ -19,9 +22,11 @@ void testGetCityFromString(String expectedCity, String address) { assertEquals(expectedCity, city); } - @ParameterizedTest @CsvSource(value = { + "118 Route de Narbonne CS 17705 31077 Toulouse Cedex 4;118 Route de Narbonne CS 17705", + "96, avenue de Valrose - 06106 NICE cedex2;96, avenue de Valrose", + "10 rue Henri Poincaré -13388 Marseille;10 rue Henri Poincaré", "123 Main St 84450 city de la garde;123 Main St", "123 - - - 8451 8451 Main St 84450 city;123 - - - 8451 8451 Main St", "123 Main St 84450 11841 city;123 Main St 84450", @@ -34,6 +39,9 @@ void testGetStreetFromString(String address, String expectedStreet) { @ParameterizedTest @CsvSource(value = { + "118 Route de Narbonne CS 17705 31077 Toulouse Cedex 4;31077", + "96, avenue de Valrose - 06106 NICE cedex2;06106", + "10 rue Henri Poincaré -13388 Marseille;13388", "123 Main St 84 city de la garde;84", "123 - - - 8451 8451 Main St 84450 city;84450", "123 Main St 84450 11841 city;11841", diff --git a/src/test/java/io/github/mathieusoysal/resources/test-hash1.txt b/src/test/java/io/github/mathieusoysal/resources/test-hash1.txt new file mode 100644 index 000000000..16cbe371e --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/resources/test-hash1.txt @@ -0,0 +1 @@ +TESTTest \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/resources/test-hash2.txt b/src/test/java/io/github/mathieusoysal/resources/test-hash2.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/resources/test-hash2.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file diff --git a/src/test/java/io/github/mathieusoysal/resources/test-hash3.txt b/src/test/java/io/github/mathieusoysal/resources/test-hash3.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/src/test/java/io/github/mathieusoysal/resources/test-hash3.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file