26 create archive for sum up the day

pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>io.github.mathieusoysal</groupId>
-    <artifactId>crous-artificat-collector</artifactId>
+    <artifactId>crous-assistant-collector</artifactId>
     <version>1.0</version>
     <packaging>jar</packaging>
 

src/main/java/io/github/mathieusoysal/App.java

@@ -1,50 +1,64 @@
 package io.github.mathieusoysal;
 
 import java.io.IOException;
+import java.time.LocalDate;
 
-import com.fasterxml.jackson.core.exc.StreamReadException;
-import com.fasterxml.jackson.databind.DatabindException;
 import com.github.forax.beautifullogger.Logger;
 
-import io.github.mathieusoysal.exceptions.ApiRequestFailedException;
+import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive;
+import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous;
+import io.github.mathieusoysal.data.managment.savers.ArchiveName;
+import io.github.mathieusoysal.data.managment.savers.DataSaver;
 import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException;
-import io.github.mathieusoysal.logement.data.DataCollector;
-import io.github.mathieusoysal.logement.data.DataSaver;
 
 public class App {
     private static final Logger LOGGER = Logger.getLogger();
     private static final String MAIL_PROPERTIES_NAME = "MAIL";
     private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD";
+    private static final String LINK_TO_DATA_PROPERTIE_NAME = "LINK_TO_DATA";
 
     public static void main(String[] args)
-            throws StreamReadException, DatabindException, ApiRequestFailedException, IOException,
-            InterruptedException {
+            throws IOException {
         LOGGER.info(() -> "Starting application");
-        var logements = DataCollector.getAvailableLogementsWithConnection(getEmail(), getPassword());
-        DataSaver.createArchiveLogements(logements);
+        if (sumupdayModIsActivated())
+            createArchiveSumUpForThisDay();
+        else
+            createArchiveForThisHour();
         LOGGER.info(() -> "Application finished");
     }
 
-    private static String getEmail() {
-        LOGGER.info(() -> "Getting email from environment variables");
-        String email = System.getenv(MAIL_PROPERTIES_NAME);
-        if (email == null)
-        {
-            LOGGER.error(() -> "Email not found in environment variables");
-            throw new PropertiesNotFoundRuntimeException(MAIL_PROPERTIES_NAME);
+    private static void createArchiveSumUpForThisDay() {
+        var dataCollector = new DataCollectorFromArchive(LINK_TO_DATA_PROPERTIE_NAME);
+        var sumUpOfTheDay = dataCollector.getSumUpOfDay(LocalDate.now());
+        DataSaver.save(ArchiveName.DAY_SUM_UP, sumUpOfTheDay);
+    }
+
+    private static boolean sumupdayModIsActivated() {
+        return System.getenv(LINK_TO_DATA_PROPERTIE_NAME) != null;
+    }
+
+    private static void createArchiveForThisHour()
+            throws IOException {
+        var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(getEmail(), getPassword());
+        DataSaver.save(ArchiveName.HOUR, logements);
+    }
+
+    private static String getPropertie(final String propertieName) {
+        LOGGER.info(() -> "Getting " + propertieName + " from environment variables");
+        String propertie = System.getenv(propertieName);
+        if (propertie == null) {
+            LOGGER.error(() -> propertieName + " not found in environment variables");
+            throw new PropertiesNotFoundRuntimeException(propertieName);
         }
-        return email;
+        return propertie;
+    }
+
+    private static String getEmail() {
+        return getPropertie(MAIL_PROPERTIES_NAME);
     }
 
     private static String getPassword() {
-        LOGGER.info(() -> "Getting password from environment variables");
-        String password = System.getenv(PASSWORD_PROPERTIES_NAME);
-        if (password == null)
-        {
-            LOGGER.error(() -> "Password not found in environment variables");
-            throw new PropertiesNotFoundRuntimeException(PASSWORD_PROPERTIES_NAME);
-        }
-        return password;
+        return getPropertie(PASSWORD_PROPERTIES_NAME);
     }
 
 }

src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java

@@ -0,0 +1,16 @@
+package io.github.mathieusoysal.data.managment.collectors;
+
+import java.time.LocalDate;
+
+public class DataCollectorFromArchive {
+    private final String archiveUrl;
+
+    public DataCollectorFromArchive(String archiveUrl) {
+        this.archiveUrl = archiveUrl;
+    }
+
+    public String getSumUpOfDay(LocalDate day) {
+        return new RequestorToGetSumUpOfDay(day).requestWitGet(archiveUrl);
+    }
+
+}

src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java

@@ -0,0 +1,39 @@
+package io.github.mathieusoysal.data.managment.collectors;
+
+import java.io.IOException;
+import java.util.List;
+
+import com.github.forax.beautifullogger.Logger;
+
+import io.github.mathieusoysal.logement.Logement;
+import io.github.mathieusoysal.logement.pojo.Convertor;
+
+public class DataCollectorFromCrous {
+    private static final Logger LOGGER = Logger.getLogger();
+    private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32";
+
+    public static List<Logement> getAvailableLogementsWithoutConnection()
+            throws IOException {
+        Requestor requestor = new RequestorWithoutConnection();
+        String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS);
+        return Convertor.getLogementsFromBruteJsonString(jsonLogements);
+    }
+
+    public static List<Logement> getAllLogementsWithoutConnection()
+            throws IOException {
+        LOGGER.info(() -> "Getting all logements");
+        Requestor requestor = new RequestorWithoutConnection();
+        String jsonLogements = requestor.requestWitGet("https://trouverunlogement.lescrous.fr/api/fr/search/29");
+        return Convertor.getLogementsFromBruteJsonString(jsonLogements);
+    }
+
+    public static List<Logement> getAvailableLogementsWithConnection(String email, String password)
+            throws IOException {
+        Requestor requestor = new RequestorWithConnection(email, password);
+        String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS);
+        return Convertor.getLogementsFromBruteJsonString(jsonLogements);
+    }
+
+    private DataCollectorFromCrous() {
+    }
+}

src/main/java/io/github/mathieusoysal/data/managment/collectors/Requestor.java

@@ -0,0 +1,14 @@
+package io.github.mathieusoysal.data.managment.collectors;
+
+import com.microsoft.playwright.options.RequestOptions;
+
+interface Requestor {
+    static final String BODY_POST_TO_GET_LOGEMENTS = "{\r\n    \"idTool\": 32,\r\n    \"need_aggregation\": false,\r\n    \"page\": 1,\r\n    \"pageSize\": 2500,\r\n    \"sector\": null,\r\n    \"occupationModes\": [],\r\n    \"location\": [\r\n        {\r\n            \"lon\": -9.9079,\r\n            \"lat\": 51.7087\r\n        },\r\n        {\r\n            \"lon\": 14.3224,\r\n            \"lat\": 40.5721\r\n        }\r\n    ],\r\n    \"residence\": null,\r\n    \"precision\": 9,\r\n    \"equipment\": [],\r\n    \"price\": {\r\n        \"min\": 0,\r\n        \"max\": 10000000\r\n    }\r\n}";
+
+    static final RequestOptions REQUEST_TO_GET_LOGEMENTS = RequestOptions.create()
+            .setMethod("POST")
+            .setHeader("Content-Type", "application/json")
+            .setData(BODY_POST_TO_GET_LOGEMENTS);
+
+    String requestWitGet(String url);
+}

src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java

@@ -0,0 +1,59 @@
+package io.github.mathieusoysal.data.managment.collectors;
+
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.stream.IntStream;
+
+import com.github.forax.beautifullogger.Logger;
+import com.microsoft.playwright.APIRequestContext;
+import com.microsoft.playwright.Playwright;
+
+import io.github.mathieusoysal.data.managment.convertors.Convertor;
+import io.github.mathieusoysal.exceptions.ApiRequestErrorRuntimeException;
+import io.github.mathieusoysal.logement.Logement;
+
+class RequestorToGetSumUpOfDay implements Requestor {
+    private static final Logger LOGGER = Logger.getLogger();
+    private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("00");
+
+    private final String date;
+
+    public RequestorToGetSumUpOfDay(LocalDate date) {
+        this.date = date.format(DateTimeFormatter.ISO_LOCAL_DATE);
+    }
+
+    public RequestorToGetSumUpOfDay(String date) {
+        this.date = date;
+    }
+
+    @Override
+    public String requestWitGet(String url) {
+        LOGGER.info(() -> "Creating sum up of the day: " + date);
+        String linkToDataForTheDay = url + "/" + date;
+        Logement[][] sumUp;
+        LOGGER.info(() -> "Creating profil to request logements");
+        try (Playwright playwright = Playwright.create()) {
+            LOGGER.info(() -> "profil created");
+            var context = playwright.request().newContext();
+            sumUp = IntStream.range(0, 24)
+                    .<String>mapToObj(hour -> linkToDataForTheDay + "/" + NUMBER_FORMAT.format(hour))
+                    .<Logement[]>map(link -> getFromUrl(link, context))
+                    .toArray(Logement[][]::new);
+            LOGGER.info(() -> "Logements received");
+        } 
+        LOGGER.info(() -> "profil closed");
+        return Convertor.convertLogementMatrixToJson(sumUp);
+    }
+
+    private static Logement[] getFromUrl(String url, APIRequestContext context) {
+        LOGGER.info(() -> "Getting data from url: " + url);
+        var respons = context.get(url);
+        if (!respons.ok())
+            throw new ApiRequestErrorRuntimeException(respons);
+        LOGGER.info(() -> "Data received");
+        return Convertor.convertJsonToArrayOfLogements(respons.text());
+    }
+
+}

src/main/java/io/github/mathieusoysal/logement/data/DataCollector.java → src/main/java/io/github/mathieusoysal/data/managment/collectors/RequestorWithConnection.java

@@ -1,11 +1,7 @@
-package io.github.mathieusoysal.logement.data;
+package io.github.mathieusoysal.data.managment.collectors;
 
-import java.io.IOException;
 import java.nio.file.Paths;
-import java.util.List;
 
-import com.fasterxml.jackson.core.exc.StreamReadException;
-import com.fasterxml.jackson.databind.DatabindException;
 import com.github.forax.beautifullogger.Logger;
 import com.microsoft.playwright.Browser;
 import com.microsoft.playwright.Browser.NewContextOptions;
@@ -18,94 +14,42 @@
 import com.microsoft.playwright.Tracing;
 import com.microsoft.playwright.options.AriaRole;
 import com.microsoft.playwright.options.LoadState;
-import com.microsoft.playwright.options.RequestOptions;
 
-import io.github.mathieusoysal.exceptions.ApiRequestFailedException;
+import io.github.mathieusoysal.exceptions.ApiRequestFailedRuntimeException;
 import io.github.mathieusoysal.exceptions.CannotBeConnectedError;
 import io.github.mathieusoysal.exceptions.LoginOptionCantBeSelectedError;
-import io.github.mathieusoysal.logement.pojo.Convertor;
-import io.github.mathieusoysal.logement.pojo.Logement;
 
-public class DataCollector {
-    private static final Logger LOGGER = Logger.getLogger();
-    private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32";
-    private static final String BODY_POST_TO_GET_LOGEMENTS = "{\r\n    \"idTool\": 32,\r\n    \"need_aggregation\": false,\r\n    \"page\": 1,\r\n    \"pageSize\": 2500,\r\n    \"sector\": null,\r\n    \"occupationModes\": [],\r\n    \"location\": [\r\n        {\r\n            \"lon\": -9.9079,\r\n            \"lat\": 51.7087\r\n        },\r\n        {\r\n            \"lon\": 14.3224,\r\n            \"lat\": 40.5721\r\n        }\r\n    ],\r\n    \"residence\": null,\r\n    \"precision\": 9,\r\n    \"equipment\": [],\r\n    \"price\": {\r\n        \"min\": 0,\r\n        \"max\": 10000000\r\n    }\r\n}";
-    private static final RequestOptions REQUEST_TO_GET_LOGEMENTS = RequestOptions.create()
-            .setMethod("POST")
-            .setHeader("Content-Type", "application/json")
-            .setData(BODY_POST_TO_GET_LOGEMENTS);
-
-    public static List<Logement> getAvailableLogementsWithoutConnection()
-            throws ApiRequestFailedException, StreamReadException, DatabindException, IOException {
-        List<Logement> logements;
-        LOGGER.info(() -> "Creating profil to request logements");
-        try (Playwright playwright = Playwright.create()) {
-            LOGGER.info(() -> "profil created");
-            LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS);
-            var respons = playwright.request().newContext()
-                    .head(LINK_TO_GET_ALL_LOGEMENTS, REQUEST_TO_GET_LOGEMENTS);
-            if (!respons.ok())
-                throw new ApiRequestFailedException(respons);
-            LOGGER.info(() -> "Logements received");
-            logements = Convertor.getLogementsFromBruteJsonString(respons.text());
-        }
-        LOGGER.info(() -> "profil closed");
-        return logements;
-    }
+class RequestorWithConnection implements Requestor {
 
-    public static List<Logement> getAllLogementsWithoutConnection()
-            throws ApiRequestFailedException, StreamReadException, DatabindException, IOException {
-        LOGGER.info(() -> "Getting all logements");
-        List<Logement> logements;
-        LOGGER.info(() -> "Creating profil to request logements");
-        try (Playwright playwright = Playwright.create()) {
-            var respons = playwright.request().newContext()
-                    .head("https://trouverunlogement.lescrous.fr/api/fr/search/29", REQUEST_TO_GET_LOGEMENTS);
-            if (!respons.ok()) {
-                LOGGER.error(() -> "Request failed");
-                throw new ApiRequestFailedException(respons);
-            }
-            LOGGER.info(() -> "Request succeed");
-            logements = Convertor.getLogementsFromBruteJsonString(respons.text());
-        }
-        LOGGER.info(() -> "profil closed");
-        LOGGER.info(() -> "All logements received");
-        return logements;
+    private final String email;
+    private final String password;
+
+    public RequestorWithConnection(final String email, final String password) {
+        this.email = email;
+        this.password = password;
     }
 
-    public static List<Logement> getAvailableLogementsWithConnection(String email, String password)
-            throws ApiRequestFailedException, StreamReadException, DatabindException, IOException,
-            InterruptedException {
+    @Override
+    public String requestWitGet(String url) {
         LOGGER.info(() -> "Getting available logements");
         LOGGER.info(() -> "Creating profil to request logements");
         Playwright playwright = Playwright.create();
         Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions());
         BrowserContext context = browser.newContext(new NewContextOptions().setScreenSize(1920, 1080));
         Page page = context.newPage();
-        List<Logement> logements;
+        String jsonLogements;
         try {
-
-            context.tracing().start(new Tracing.StartOptions()
-                    .setScreenshots(true)
-                    .setSnapshots(true)
-                    .setSources(true));
-            goToLoginPage(page);
-            selectLoginOption(playwright, page);
-            connectToTheCrous(email, password, playwright, page);
-            LOGGER.info(() -> "Going to logements page");
-            page.getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setName("Lancer une recherche"))
-                    .click();
-            page.waitForLoadState();
-            LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS);
+            etablishConnectionWithWebsite(playwright, context, page);
+            LOGGER.info(() -> "Requesting logements from " + url);
             var respons = page.request()
-                    .head(LINK_TO_GET_ALL_LOGEMENTS,
+                    .head(url,
                             REQUEST_TO_GET_LOGEMENTS);
             if (!respons.ok()) {
                 LOGGER.error(() -> "Request failed");
-                throw new ApiRequestFailedException(respons);
+                throw new ApiRequestFailedRuntimeException(respons);
             }
             LOGGER.info(() -> "Logements received");
-            logements = Convertor.getLogementsFromBruteJsonString(respons.text());
+            jsonLogements = respons.text();
         } catch (TimeoutError | LoginOptionCantBeSelectedError | CannotBeConnectedError e) {
             LOGGER.error("Request failed", e);
             context.tracing().stop(new Tracing.StopOptions()
@@ -118,9 +62,30 @@ public static List<Logement> getAvailableLogementsWithConnection(String email, S
             playwright.close();
             LOGGER.info(() -> "profil closed");
         }
-        return logements;
+        return jsonLogements;
     }
 
+    private void etablishConnectionWithWebsite(Playwright playwright, BrowserContext context, Page page) {
+        context.tracing().start(new Tracing.StartOptions()
+                .setScreenshots(true)
+                .setSnapshots(true)
+                .setSources(true));
+        goToLoginPage(page);
+        try {
+            selectLoginOption(playwright, page);
+        } catch (LoginOptionCantBeSelectedError e) {
+            LOGGER.warning(() -> "Can't connect to the crous");
+            selectLoginOption(playwright, page);
+        }
+        connectToTheCrous(email, password, playwright, page);
+        LOGGER.info(() -> "Going to logements page");
+        page.getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setName("Lancer une recherche"))
+                .click();
+        page.waitForLoadState();
+    }
+
+    private static final Logger LOGGER = Logger.getLogger();
+
     private static void goToLoginPage(Page page) {
         LOGGER.info(() -> "Going to login page");
         page.navigate("https://trouverunlogement.lescrous.fr/tools/32/search");
@@ -186,6 +151,4 @@ private static void waitForUrlChange(String currentUrl, Page page) {
         page.waitForCondition(() -> !page.url().equals(currentUrl));
     }
 
-    private DataCollector() {
-    }
 }