Skip to content

Commit

Permalink
26 create archive for sum up the day (#38)
Browse files Browse the repository at this point in the history 
* WIP: DataSumUp

* Fix code formatting and add default constructors

* Add archive for day sum up

* Add data processing for current day

* General refactorisation

* General refactor

* General refactor
  • Loading branch information
@MathieuSoysal
MathieuSoysal authored Jan 9, 2024
1 parent 9e9086f commit 1b9ec88
Show file tree
Hide file tree
Showing 33 changed files with 738 additions and 369 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>

<groupId>io.github.mathieusoysal</groupId>
<artifactId>crous-artificat-collector</artifactId>
<artifactId>crous-assistant-collector</artifactId>
<version>1.0</version>
<packaging>jar</packaging>

Expand Down
64 changes: 39 additions & 25 deletions src/main/java/io/github/mathieusoysal/App.java
View file
Original file line number Diff line number Diff line change
@@ -1,50 +1,64 @@
package io.github.mathieusoysal;

import java.io.IOException;
import java.time.LocalDate;

import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.databind.DatabindException;
import com.github.forax.beautifullogger.Logger;

import io.github.mathieusoysal.exceptions.ApiRequestFailedException;
import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromArchive;
import io.github.mathieusoysal.data.managment.collectors.DataCollectorFromCrous;
import io.github.mathieusoysal.data.managment.savers.ArchiveName;
import io.github.mathieusoysal.data.managment.savers.DataSaver;
import io.github.mathieusoysal.exceptions.PropertiesNotFoundRuntimeException;
import io.github.mathieusoysal.logement.data.DataCollector;
import io.github.mathieusoysal.logement.data.DataSaver;

public class App {
private static final Logger LOGGER = Logger.getLogger();
private static final String MAIL_PROPERTIES_NAME = "MAIL";
private static final String PASSWORD_PROPERTIES_NAME = "PASSWORD";
private static final String LINK_TO_DATA_PROPERTIE_NAME = "LINK_TO_DATA";

public static void main(String[] args)
throws StreamReadException, DatabindException, ApiRequestFailedException, IOException,
InterruptedException {
throws IOException {
LOGGER.info(() -> "Starting application");
var logements = DataCollector.getAvailableLogementsWithConnection(getEmail(), getPassword());
DataSaver.createArchiveLogements(logements);
if (sumupdayModIsActivated())
createArchiveSumUpForThisDay();
else
createArchiveForThisHour();
LOGGER.info(() -> "Application finished");
}

private static String getEmail() {
LOGGER.info(() -> "Getting email from environment variables");
String email = System.getenv(MAIL_PROPERTIES_NAME);
if (email == null)
{
LOGGER.error(() -> "Email not found in environment variables");
throw new PropertiesNotFoundRuntimeException(MAIL_PROPERTIES_NAME);
private static void createArchiveSumUpForThisDay() {
var dataCollector = new DataCollectorFromArchive(LINK_TO_DATA_PROPERTIE_NAME);
var sumUpOfTheDay = dataCollector.getSumUpOfDay(LocalDate.now());
DataSaver.save(ArchiveName.DAY_SUM_UP, sumUpOfTheDay);
}

private static boolean sumupdayModIsActivated() {
return System.getenv(LINK_TO_DATA_PROPERTIE_NAME) != null;
}

private static void createArchiveForThisHour()
throws IOException {
var logements = DataCollectorFromCrous.getAvailableLogementsWithConnection(getEmail(), getPassword());
DataSaver.save(ArchiveName.HOUR, logements);
}

private static String getPropertie(final String propertieName) {
LOGGER.info(() -> "Getting " + propertieName + " from environment variables");
String propertie = System.getenv(propertieName);
if (propertie == null) {
LOGGER.error(() -> propertieName + " not found in environment variables");
throw new PropertiesNotFoundRuntimeException(propertieName);
}
return email;
return propertie;
}

private static String getEmail() {
return getPropertie(MAIL_PROPERTIES_NAME);
}

private static String getPassword() {
LOGGER.info(() -> "Getting password from environment variables");
String password = System.getenv(PASSWORD_PROPERTIES_NAME);
if (password == null)
{
LOGGER.error(() -> "Password not found in environment variables");
throw new PropertiesNotFoundRuntimeException(PASSWORD_PROPERTIES_NAME);
}
return password;
return getPropertie(PASSWORD_PROPERTIES_NAME);
}

}
16 changes: 16 additions & 0 deletions ...main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromArchive.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package io.github.mathieusoysal.data.managment.collectors;

import java.time.LocalDate;

public class DataCollectorFromArchive {
private final String archiveUrl;

public DataCollectorFromArchive(String archiveUrl) {
this.archiveUrl = archiveUrl;
}

public String getSumUpOfDay(LocalDate day) {
return new RequestorToGetSumUpOfDay(day).requestWitGet(archiveUrl);
}

}
39 changes: 39 additions & 0 deletions src/main/java/io/github/mathieusoysal/data/managment/collectors/DataCollectorFromCrous.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package io.github.mathieusoysal.data.managment.collectors;

import java.io.IOException;
import java.util.List;

import com.github.forax.beautifullogger.Logger;

import io.github.mathieusoysal.logement.Logement;
import io.github.mathieusoysal.logement.pojo.Convertor;

public class DataCollectorFromCrous {
private static final Logger LOGGER = Logger.getLogger();
private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32";

public static List<Logement> getAvailableLogementsWithoutConnection()
throws IOException {
Requestor requestor = new RequestorWithoutConnection();
String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS);
return Convertor.getLogementsFromBruteJsonString(jsonLogements);
}

public static List<Logement> getAllLogementsWithoutConnection()
throws IOException {
LOGGER.info(() -> "Getting all logements");
Requestor requestor = new RequestorWithoutConnection();
String jsonLogements = requestor.requestWitGet("https://trouverunlogement.lescrous.fr/api/fr/search/29");
return Convertor.getLogementsFromBruteJsonString(jsonLogements);
}

public static List<Logement> getAvailableLogementsWithConnection(String email, String password)
throws IOException {
Requestor requestor = new RequestorWithConnection(email, password);
String jsonLogements = requestor.requestWitGet(LINK_TO_GET_ALL_LOGEMENTS);
return Convertor.getLogementsFromBruteJsonString(jsonLogements);
}

private DataCollectorFromCrous() {
}
}
14 changes: 14 additions & 0 deletions src/main/java/io/github/mathieusoysal/data/managment/collectors/Requestor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package io.github.mathieusoysal.data.managment.collectors;

import com.microsoft.playwright.options.RequestOptions;

interface Requestor {
static final String BODY_POST_TO_GET_LOGEMENTS = "{\r\n \"idTool\": 32,\r\n \"need_aggregation\": false,\r\n \"page\": 1,\r\n \"pageSize\": 2500,\r\n \"sector\": null,\r\n \"occupationModes\": [],\r\n \"location\": [\r\n {\r\n \"lon\": -9.9079,\r\n \"lat\": 51.7087\r\n },\r\n {\r\n \"lon\": 14.3224,\r\n \"lat\": 40.5721\r\n }\r\n ],\r\n \"residence\": null,\r\n \"precision\": 9,\r\n \"equipment\": [],\r\n \"price\": {\r\n \"min\": 0,\r\n \"max\": 10000000\r\n }\r\n}";

static final RequestOptions REQUEST_TO_GET_LOGEMENTS = RequestOptions.create()
.setMethod("POST")
.setHeader("Content-Type", "application/json")
.setData(BODY_POST_TO_GET_LOGEMENTS);

String requestWitGet(String url);
}
59 changes: 59 additions & 0 deletions ...main/java/io/github/mathieusoysal/data/managment/collectors/RequestorToGetSumUpOfDay.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package io.github.mathieusoysal.data.managment.collectors;

import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.stream.IntStream;

import com.github.forax.beautifullogger.Logger;
import com.microsoft.playwright.APIRequestContext;
import com.microsoft.playwright.Playwright;

import io.github.mathieusoysal.data.managment.convertors.Convertor;
import io.github.mathieusoysal.exceptions.ApiRequestErrorRuntimeException;
import io.github.mathieusoysal.logement.Logement;

class RequestorToGetSumUpOfDay implements Requestor {
private static final Logger LOGGER = Logger.getLogger();
private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("00");

private final String date;

public RequestorToGetSumUpOfDay(LocalDate date) {
this.date = date.format(DateTimeFormatter.ISO_LOCAL_DATE);
}

public RequestorToGetSumUpOfDay(String date) {
this.date = date;
}

@Override
public String requestWitGet(String url) {
LOGGER.info(() -> "Creating sum up of the day: " + date);
String linkToDataForTheDay = url + "/" + date;
Logement[][] sumUp;
LOGGER.info(() -> "Creating profil to request logements");
try (Playwright playwright = Playwright.create()) {
LOGGER.info(() -> "profil created");
var context = playwright.request().newContext();
sumUp = IntStream.range(0, 24)
.<String>mapToObj(hour -> linkToDataForTheDay + "/" + NUMBER_FORMAT.format(hour))
.<Logement[]>map(link -> getFromUrl(link, context))
.toArray(Logement[][]::new);
LOGGER.info(() -> "Logements received");
}
LOGGER.info(() -> "profil closed");
return Convertor.convertLogementMatrixToJson(sumUp);
}

private static Logement[] getFromUrl(String url, APIRequestContext context) {
LOGGER.info(() -> "Getting data from url: " + url);
var respons = context.get(url);
if (!respons.ok())
throw new ApiRequestErrorRuntimeException(respons);
LOGGER.info(() -> "Data received");
return Convertor.convertJsonToArrayOfLogements(respons.text());
}

}
115 changes: 39 additions & 76 deletions ...eusoysal/logement/data/DataCollector.java → ...t/collectors/RequestorWithConnection.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
package io.github.mathieusoysal.logement.data;
package io.github.mathieusoysal.data.managment.collectors;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;

import com.fasterxml.jackson.core.exc.StreamReadException;
import com.fasterxml.jackson.databind.DatabindException;
import com.github.forax.beautifullogger.Logger;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.Browser.NewContextOptions;
Expand All @@ -18,94 +14,42 @@
import com.microsoft.playwright.Tracing;
import com.microsoft.playwright.options.AriaRole;
import com.microsoft.playwright.options.LoadState;
import com.microsoft.playwright.options.RequestOptions;

import io.github.mathieusoysal.exceptions.ApiRequestFailedException;
import io.github.mathieusoysal.exceptions.ApiRequestFailedRuntimeException;
import io.github.mathieusoysal.exceptions.CannotBeConnectedError;
import io.github.mathieusoysal.exceptions.LoginOptionCantBeSelectedError;
import io.github.mathieusoysal.logement.pojo.Convertor;
import io.github.mathieusoysal.logement.pojo.Logement;

public class DataCollector {
private static final Logger LOGGER = Logger.getLogger();
private static final String LINK_TO_GET_ALL_LOGEMENTS = "https://trouverunlogement.lescrous.fr/api/fr/search/32";
private static final String BODY_POST_TO_GET_LOGEMENTS = "{\r\n \"idTool\": 32,\r\n \"need_aggregation\": false,\r\n \"page\": 1,\r\n \"pageSize\": 2500,\r\n \"sector\": null,\r\n \"occupationModes\": [],\r\n \"location\": [\r\n {\r\n \"lon\": -9.9079,\r\n \"lat\": 51.7087\r\n },\r\n {\r\n \"lon\": 14.3224,\r\n \"lat\": 40.5721\r\n }\r\n ],\r\n \"residence\": null,\r\n \"precision\": 9,\r\n \"equipment\": [],\r\n \"price\": {\r\n \"min\": 0,\r\n \"max\": 10000000\r\n }\r\n}";
private static final RequestOptions REQUEST_TO_GET_LOGEMENTS = RequestOptions.create()
.setMethod("POST")
.setHeader("Content-Type", "application/json")
.setData(BODY_POST_TO_GET_LOGEMENTS);

public static List<Logement> getAvailableLogementsWithoutConnection()
throws ApiRequestFailedException, StreamReadException, DatabindException, IOException {
List<Logement> logements;
LOGGER.info(() -> "Creating profil to request logements");
try (Playwright playwright = Playwright.create()) {
LOGGER.info(() -> "profil created");
LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS);
var respons = playwright.request().newContext()
.head(LINK_TO_GET_ALL_LOGEMENTS, REQUEST_TO_GET_LOGEMENTS);
if (!respons.ok())
throw new ApiRequestFailedException(respons);
LOGGER.info(() -> "Logements received");
logements = Convertor.getLogementsFromBruteJsonString(respons.text());
}
LOGGER.info(() -> "profil closed");
return logements;
}
class RequestorWithConnection implements Requestor {

public static List<Logement> getAllLogementsWithoutConnection()
throws ApiRequestFailedException, StreamReadException, DatabindException, IOException {
LOGGER.info(() -> "Getting all logements");
List<Logement> logements;
LOGGER.info(() -> "Creating profil to request logements");
try (Playwright playwright = Playwright.create()) {
var respons = playwright.request().newContext()
.head("https://trouverunlogement.lescrous.fr/api/fr/search/29", REQUEST_TO_GET_LOGEMENTS);
if (!respons.ok()) {
LOGGER.error(() -> "Request failed");
throw new ApiRequestFailedException(respons);
}
LOGGER.info(() -> "Request succeed");
logements = Convertor.getLogementsFromBruteJsonString(respons.text());
}
LOGGER.info(() -> "profil closed");
LOGGER.info(() -> "All logements received");
return logements;
private final String email;
private final String password;

public RequestorWithConnection(final String email, final String password) {
this.email = email;
this.password = password;
}

public static List<Logement> getAvailableLogementsWithConnection(String email, String password)
throws ApiRequestFailedException, StreamReadException, DatabindException, IOException,
InterruptedException {
@Override
public String requestWitGet(String url) {
LOGGER.info(() -> "Getting available logements");
LOGGER.info(() -> "Creating profil to request logements");
Playwright playwright = Playwright.create();
Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions());
BrowserContext context = browser.newContext(new NewContextOptions().setScreenSize(1920, 1080));
Page page = context.newPage();
List<Logement> logements;
String jsonLogements;
try {

context.tracing().start(new Tracing.StartOptions()
.setScreenshots(true)
.setSnapshots(true)
.setSources(true));
goToLoginPage(page);
selectLoginOption(playwright, page);
connectToTheCrous(email, password, playwright, page);
LOGGER.info(() -> "Going to logements page");
page.getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setName("Lancer une recherche"))
.click();
page.waitForLoadState();
LOGGER.info(() -> "Requesting logements from " + LINK_TO_GET_ALL_LOGEMENTS);
etablishConnectionWithWebsite(playwright, context, page);
LOGGER.info(() -> "Requesting logements from " + url);
var respons = page.request()
.head(LINK_TO_GET_ALL_LOGEMENTS,
.head(url,
REQUEST_TO_GET_LOGEMENTS);
if (!respons.ok()) {
LOGGER.error(() -> "Request failed");
throw new ApiRequestFailedException(respons);
throw new ApiRequestFailedRuntimeException(respons);
}
LOGGER.info(() -> "Logements received");
logements = Convertor.getLogementsFromBruteJsonString(respons.text());
jsonLogements = respons.text();
} catch (TimeoutError | LoginOptionCantBeSelectedError | CannotBeConnectedError e) {
LOGGER.error("Request failed", e);
context.tracing().stop(new Tracing.StopOptions()
Expand All @@ -118,9 +62,30 @@ public static List<Logement> getAvailableLogementsWithConnection(String email, S
playwright.close();
LOGGER.info(() -> "profil closed");
}
return logements;
return jsonLogements;
}

private void etablishConnectionWithWebsite(Playwright playwright, BrowserContext context, Page page) {
context.tracing().start(new Tracing.StartOptions()
.setScreenshots(true)
.setSnapshots(true)
.setSources(true));
goToLoginPage(page);
try {
selectLoginOption(playwright, page);
} catch (LoginOptionCantBeSelectedError e) {
LOGGER.warning(() -> "Can't connect to the crous");
selectLoginOption(playwright, page);
}
connectToTheCrous(email, password, playwright, page);
LOGGER.info(() -> "Going to logements page");
page.getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setName("Lancer une recherche"))
.click();
page.waitForLoadState();
}

private static final Logger LOGGER = Logger.getLogger();

private static void goToLoginPage(Page page) {
LOGGER.info(() -> "Going to login page");
page.navigate("https://trouverunlogement.lescrous.fr/tools/32/search");
Expand Down Expand Up @@ -186,6 +151,4 @@ private static void waitForUrlChange(String currentUrl, Page page) {
page.waitForCondition(() -> !page.url().equals(currentUrl));
}

private DataCollector() {
}
}
Loading

0 comments on commit 1b9ec88

Please sign in to comment.