feat: formations associées

mission-apprentissage · Sep 10, 2024 · 8b0d548 · 8b0d548
1 parent a6a5bd8
commit 8b0d548
Show file tree

Hide file tree

Showing 44 changed files with 2,929 additions and 2,673 deletions.
diff --git a/server/migrations/20240909203559_poursuite_etudes.ts b/server/migrations/20240909203559_poursuite_etudes.ts
@@ -0,0 +1,33 @@
+import { Kysely, sql } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.executeQuery(
+    sql`
+CREATE TABLE "formationPoursuite" (
+	"formationId" uuid NOT NULL,
+	"libelle" VARCHAR NOT NULL,
+	"onisepId" VARCHAR,
+	"type" VARCHAR,
+	
+	"createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  	"updatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+	CONSTRAINT "formationPoursuite_pkey" PRIMARY KEY ("formationId", "libelle"),
+	CONSTRAINT "formationPoursuite_formation_fk" FOREIGN KEY ("formationId") REFERENCES "formation"(id)
+);
+
+CREATE TRIGGER set_timestamp
+BEFORE UPDATE ON "formationPoursuite"
+FOR EACH ROW
+EXECUTE PROCEDURE trigger_set_timestamp();
+`.compile(db)
+  );
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.executeQuery(
+    sql`
+	DROP TABLE IF EXISTS "formationPoursuite";
+	`.compile(db)
+  );
+}
diff --git a/server/package.json b/server/package.json
@@ -41,6 +41,7 @@
     "env-var": "7.1.1",
     "express": "4.18.1",
     "fast-luhn": "2.0.0",
+    "fast-xml-parser": "4.5.0",
     "geotoolbox": "2.0.3",
     "iconv-lite": "0.6.3",
     "joi": "17.13.3",
@@ -82,15 +83,15 @@
     "@types/http-cache-semantics": "4.0.4",
     "@types/node": "22.5.0",
     "@types/pg": "8.11.8",
-    "axiosist": "0.10.0",
+    "axiosist": "1.2.0",
     "chai": "4.3.7",
     "chai-as-promised": "7.1.1",
     "chai-diff": "1.0.2",
     "chai-dom": "1.12.0",
     "chai-files": "1.4.0",
     "chai-html": "2.1.0",
     "chalk-legacy": "npm:chalk@4.1.2",
-    "eslint": "7.1.0",
+    "eslint": "7.32.0",
     "eslint-config-prettier": "6.11.0",
     "eslint-import-resolver-custom-alias": "1.3.2",
     "eslint-plugin-import": "2.28.1",

diff --git a/server/src/common/db/schema.ts b/server/src/common/db/schema.ts
@@ -100,6 +100,15 @@ export interface FormationEtablissement {
   updatedAt: Generated<Timestamp>;
 }
 
+export interface FormationPoursuite {
+  createdAt: Generated<Timestamp>;
+  formationId: string;
+  libelle: string;
+  onisepId: string | null;
+  type: string | null;
+  updatedAt: Generated<Timestamp>;
+}
+
 export interface GeographyColumns {
   coord_dimension: number | null;
   f_geography_column: string | null;
@@ -187,6 +196,7 @@ export interface DB {
   formation: Formation;
   formationDomaine: FormationDomaine;
   formationEtablissement: FormationEtablissement;
+  formationPoursuite: FormationPoursuite;
   geography_columns: GeographyColumns;
   geometry_columns: GeometryColumns;
   indicateurEntree: IndicateurEntree;

diff --git a/server/src/common/repositories/base.ts b/server/src/common/repositories/base.ts
@@ -79,7 +79,7 @@ export class SqlRepository<DB, F extends keyof DB> extends Repository {
     });
   }
 
-  _base() {
+  _base(options: any) {
     throw new Error("Base repository not implemented for this repository");
   }
 }
diff --git a/server/src/common/repositories/formation.ts b/server/src/common/repositories/formation.ts
@@ -27,29 +27,52 @@ export class FormationRepository extends SqlRepository<DB, "formation"> {
     );
   }
 
-  _base() {
+  _base({ withPoursuite } = { withPoursuite: false }) {
     return <T extends SelectQueryBuilder<DB, "formation", {}>>(eb: T) => {
-      return eb.leftJoinLateral(
-        (eb) =>
-          eb
-            .selectFrom("formationDomaine")
-            .innerJoin("domaine", "domaine.id", "formationDomaine.domaineId")
-            .select((eb) => {
-              return [
-                kyselyChainFn(
-                  eb,
-                  [
-                    { fn: "to_jsonb", args: [] },
-                    { fn: "json_agg", args: [] },
-                  ],
-                  sql`domaine.*`
-                ).as("domaine"),
-              ];
-            })
-            .whereRef("formation.id", "=", "formationDomaine.formationId")
-            .as("domaine"),
-        (join) => join.on(sql`true`)
-      );
+      return eb
+        .leftJoinLateral(
+          (eb) =>
+            eb
+              .selectFrom("formationDomaine")
+              .innerJoin("domaine", "domaine.id", "formationDomaine.domaineId")
+              .select((eb) => {
+                return [
+                  kyselyChainFn(
+                    eb,
+                    [
+                      { fn: "to_jsonb", args: [] },
+                      { fn: "json_agg", args: [] },
+                    ],
+                    sql`domaine.*`
+                  ).as("domaine"),
+                ];
+              })
+              .whereRef("formation.id", "=", "formationDomaine.formationId")
+              .as("domaine"),
+          (join) => join.on(sql`true`)
+        )
+        .$if(withPoursuite, (eb) =>
+          eb.leftJoinLateral(
+            (eb) =>
+              eb
+                .selectFrom("formationPoursuite")
+                .select((eb) => {
+                  return [
+                    kyselyChainFn(
+                      eb,
+                      [
+                        { fn: "to_jsonb", args: [] },
+                        { fn: "json_agg", args: [] },
+                      ],
+                      sql`"formationPoursuite".*`
+                    ).as("formationPoursuite"),
+                  ];
+                })
+                .whereRef("formation.id", "=", "formationPoursuite.formationId")
+                .as("formationPoursuite"),
+            (join) => join.on(sql`true`)
+          )
+        );
     };
   }
 
@@ -73,7 +96,10 @@ export class FormationRepository extends SqlRepository<DB, "formation"> {
   }
 
   getKeyRelationAlias<T extends keyof DB>(eb: ExpressionBuilder<DB, T>) {
-    return [`domaine as ${this.tableName}.domaine` as AnyAliasedColumn<DB, T>];
+    return [
+      `domaine as ${this.tableName}.domaine` as AnyAliasedColumn<DB, T>,
+      `formationPoursuite as ${this.tableName}.formationPoursuite` as AnyAliasedColumn<DB, T>,
+    ];
   }
 }
 

diff --git a/server/src/common/repositories/formationEtablissement.ts b/server/src/common/repositories/formationEtablissement.ts
@@ -42,7 +42,13 @@ export class FormationEtablissementRepository extends SqlRepository<DB, "formati
       .innerJoinLateral(
         (eb) =>
           eb
-            .selectFrom(eb.selectFrom("formation").$call(FormationRepository._base()).selectAll().as("formation"))
+            .selectFrom(
+              eb
+                .selectFrom("formation")
+                .$call(FormationRepository._base({ withPoursuite: true }))
+                .selectAll()
+                .as("formation")
+            )
             .selectAll()
             .whereRef("formationEtablissement.formationId", "=", "formation.id")
             .as("formation"),

diff --git a/server/src/config.js → server/src/config.ts b/server/src/config.js → server/src/config.ts
@@ -11,6 +11,12 @@ const config = {
     destinations: env.get("ACCOMPAGNATEUR_LOG_DESTINATIONS").default("stdout").asArray(),
   },
   slackWebhookUrl: env.get("ACCOMPAGNATEUR_SLACK_WEBHOOK_URL").asString(),
+  features: {
+    graphhopper: {
+      // Use static date for now
+      useStaticDate: true,
+    },
+  },
   pgsql: {
     uri: env
       .get("ACCOMPAGNATEUR_POSTGRES_URI")

diff --git a/server/src/jobs/formations/importIdeoFichesFormations.ts b/server/src/jobs/formations/importIdeoFichesFormations.ts
@@ -4,6 +4,7 @@ import { streamIdeoFichesFormations } from "#src/services/onisep/fichesFormation
 import RawDataRepository, { RawData, RawDataType } from "#src/common/repositories/rawData";
 import { kdb } from "#src/common/db/db";
 import { omitNil } from "#src/common/utils/objectUtils";
+import { urlOnisepToId } from "#src/services/onisep/utils";
 
 const logger = getLoggerWithContext("import");
 
@@ -41,6 +42,49 @@ async function cfdsParentAndChildren(cfd) {
   return cfds;
 }
 
+async function addPoursuiteEtudes(cfds, formation) {
+  const poursuitesRaw = formation?.poursuites_etudes?.poursuite_etudes?.formation_poursuite_Etudes || [];
+  const poursuites = Array.isArray(poursuitesRaw) ? poursuitesRaw : [poursuitesRaw];
+  const formationsWithContinuum = await kdb.selectFrom("formation").where("cfd", "in", cfds).select("id").execute();
+
+  if (formationsWithContinuum.length == 0) {
+    return;
+  }
+
+  await kdb
+    .deleteFrom("formationPoursuite")
+    .where(
+      "formationId",
+      "in",
+      formationsWithContinuum.map(({ id }) => id)
+    )
+    .returning("formationId")
+    .execute();
+
+  for (const poursuite of poursuites) {
+    const checkFormation = await RawDataRepository.first(RawDataType.ONISEP_ideoFormationsInitiales, {
+      data: { libelle_formation_principal: poursuite },
+    });
+
+    const dataFormation = checkFormation.data as RawData[RawDataType.ONISEP_ideoFormationsInitiales];
+
+    if (!checkFormation) {
+      logger.error(`La formation de poursuite d'étude "${poursuite}" n'existe pas`);
+    } else {
+      for (const formationWithContinuum of formationsWithContinuum) {
+        await kdb
+          .insertInto("formationPoursuite")
+          .values({
+            formationId: formationWithContinuum.id,
+            libelle: dataFormation.data.libelle_formation_principal,
+            onisepId: urlOnisepToId(dataFormation.data.url_et_id_onisep),
+            type: dataFormation.data.libelle_type_formation,
+          })
+          .executeTakeFirst();
+      }
+    }
+  }
+}
 export async function importIdeoFichesFormations() {
   logger.info(`Importation des données des fiches de formations Idéo`);
   const stats = { total: 0, created: 0, updated: 0, failed: 0 };
@@ -63,6 +107,8 @@ export async function importIdeoFichesFormations() {
     writeData(
       async ({ cfds, formation }) => {
         try {
+          await addPoursuiteEtudes(cfds, formation);
+
           const result = await kdb
             .updateTable("formation")
             .set(
@@ -79,7 +125,7 @@ export async function importIdeoFichesFormations() {
 
           if (result && result.length > 0) {
             logger.info(`Formation(s) ${cfds.join(",")} mise(s) à jour`);
-            stats.updated++;
+            stats.updated += cfds.length;
           }
         } catch (e) {
           logger.error(e, `Impossible d'ajouter les informations de formation(s) ${cfds.join(",")}`);

diff --git a/server/src/jobs/formations/streamOnisepFormations.ts b/server/src/jobs/formations/streamOnisepFormations.ts
@@ -1,6 +1,7 @@
 import { transformData, compose, filterData, transformIntoStream } from "oleoduc";
 import { getLoggerWithContext } from "#src/common/logger.js";
 import RawDataRepository, { RawData, RawDataType } from "#src/common/repositories/rawData";
+import { urlOnisepToId } from "#src/services/onisep/utils";
 import { formatDuree } from "./importFormationEtablissement.js";
 import { Readable } from "stream";
 
@@ -94,7 +95,7 @@ export async function streamOnisepFormations({ stats }) {
       stats.total++;
 
       const urlOnisep = data.data.for_url_et_id_onisep;
-      const idOnisep = urlOnisep.match(/FOR\.[0-9]+/)[0];
+      const idOnisep = urlOnisepToId(urlOnisep);
 
       const cfd = await getCfd(idOnisep, urlOnisep);
       if (!cfd) {

diff --git a/server/src/queries/getFormations.ts b/server/src/queries/getFormations.ts
@@ -9,6 +9,7 @@ import EtablissementRepository from "#src/common/repositories/etablissement";
 import { DB, Etablissement, Formation, FormationEtablissement } from "#src/common/db/schema.js";
 import { jsonBuildObject } from "kysely/helpers/postgres";
 import FormationRepository from "#src/common/repositories/formation";
+import config from "#src/config";
 
 const logger = getLoggerWithContext("query");
 
@@ -27,6 +28,9 @@ export function buildFilterTag(eb, tag) {
 }
 
 export function getRouteDate() {
+  if (config.features.graphhopper.useStaticDate) {
+    return moment("2024-09-09").set({ hour: 8, minute: 30, second: 0, millisecond: 0 }).toDate();
+  }
   return moment().startOf("isoWeek").add(1, "week").set({ hour: 8, minute: 30, second: 0, millisecond: 0 }).toDate();
 }
 

diff --git a/server/src/services/acce.js b/server/src/services/acce.js
@@ -2,7 +2,7 @@ import { compose } from "oleoduc";
 import iconv from "iconv-lite";
 import { parseCsv } from "#src/common/utils/csvUtils.js";
 import { createReadStream } from "fs";
-import config from "#src/config.js";
+import config from "#src/config.ts";
 
 export function etablissements(filePath = null) {
   const stream = compose(

diff --git a/server/src/services/catalogueApprentissage/CatalogueApprentissageApi.js b/server/src/services/catalogueApprentissage/CatalogueApprentissageApi.js
@@ -1,5 +1,5 @@
 import { RateLimitedApi } from "#src/common/api/RateLimitedApi.js";
-import config from "#src/config.js";
+import config from "#src/config.ts";
 import { fetchJsonWithRetry, fetch } from "#src/common/utils/httpUtils.js";
 import { getLoggerWithContext } from "#src/common/logger.ts";
 

diff --git a/server/src/services/dataGouv/DataGouvApi.js b/server/src/services/dataGouv/DataGouvApi.js
@@ -1,5 +1,5 @@
 import { RateLimitedApi } from "#src/common/api/RateLimitedApi.js";
-import config from "#src/config.js";
+import config from "#src/config.ts";
 import { fetchStreamWithRetry } from "#src/common/utils/httpUtils.js";
 
 class DataGouvApi extends RateLimitedApi {

diff --git a/server/src/services/dataGouv/certifinfo.js b/server/src/services/dataGouv/certifinfo.js
@@ -1,7 +1,7 @@
 import { compose } from "oleoduc";
 import AutoDetectDecoderStream from "autodetect-decoder-stream";
 import { parseCsv } from "#src/common/utils/csvUtils.js";
-import config from "#src/config.js";
+import config from "#src/config.ts";
 import { DataGouvApi } from "./DataGouvApi.js";
 
 export async function streamCertifInfo(options = {}) {

diff --git a/server/src/services/exposition/ExpositionApi.js b/server/src/services/exposition/ExpositionApi.js
@@ -1,5 +1,5 @@
 import { RateLimitedApi } from "#src/common/api/RateLimitedApi.js";
-import config from "#src/config.js";
+import config from "#src/config.ts";
 import { fetchJsonWithRetry } from "#src/common/utils/httpUtils.js";
 import { getLoggerWithContext } from "#src/common/logger.ts";