feat: nouveaux filtres

server/migrations/20241009132828_stats.ts

@@ -0,0 +1,46 @@
+import { Kysely, sql } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.executeQuery(
+    sql`
+CREATE TABLE "indicateurPoursuiteRegional" (
+	"id" uuid  NOT NULL DEFAULT uuid_generate_v4(),
+	"cfd" varchar NOT NULL,
+	"voie" varchar NOT NULL,
+	"codeDispositif" varchar,
+	"region" varchar(2) NOT NULL,
+	"millesime" varchar,
+	"part_en_emploi_6_mois" real,
+	"taux_en_emploi_6_mois" real,
+	"taux_en_formation" real,
+	"taux_autres_6_mois" real,
+
+	"createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  	"updatedAt" TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+	CONSTRAINT "indicateurPoursuiteRegional_pkey" PRIMARY KEY ("id"),
+	CONSTRAINT "indicateurPoursuiteRegional_cfd_voie_codeDispositif_region_millesime_key" UNIQUE NULLS NOT DISTINCT ("cfd", "voie", "codeDispositif", "region", "millesime")
+);
+
+CREATE TRIGGER set_timestamp
+BEFORE UPDATE ON "indicateurPoursuiteRegional"
+FOR EACH ROW
+EXECUTE PROCEDURE trigger_set_timestamp();
+
+
+ALTER TABLE "indicateurPoursuite" 
+		ADD COLUMN "part_en_emploi_6_mois" varchar;
+`.compile(db)
+  );
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.executeQuery(
+    sql`
+	DROP TABLE IF EXISTS "indicateurPoursuiteRegional";
+
+	ALTER TABLE "indicateurPoursuite"
+		DROP COLUMN IF EXISTS "part_en_emploi_6_mois";
+	`.compile(db)
+  );
+}

server/package.json

@@ -87,6 +87,7 @@
     "@types/http-cache-semantics": "4.0.4",
     "@types/node": "22.5.0",
     "@types/pg": "8.11.8",
+    "@types/transform-coordinates": "1.0.2",
     "@typescript-eslint/eslint-plugin": "8.6.0",
     "@typescript-eslint/parser": "8.6.0",
     "axiosist": "1.2.0",

server/src/common/db/schema.ts

@@ -39,6 +39,7 @@ export interface Etablissement {
   JPODetails: string | null;
   libelle: string | null;
   onisepId: string | null;
+  region: Generated<string>;
   statut: string | null;
   statutDetail: string | null;
   uai: string;
@@ -78,6 +79,7 @@ export interface Formation {
   id: Generated<string>;
   libelle: string | null;
   mef11: string | null;
+  niveauDiplome: Generated<string>;
   onisepIdentifiant: string | null;
   updatedAt: Generated<Timestamp>;
   voie: string;
@@ -168,6 +170,20 @@ export interface IndicateurPoursuite {
   updatedAt: Generated<Timestamp>;
 }
 
+export interface IndicateurPoursuiteRegional {
+  cfd: string;
+  codeDispositif: string | null;
+  createdAt: Generated<Timestamp>;
+  id: Generated<string>;
+  millesime: string | null;
+  region: string;
+  taux_autres_6_mois: number | null;
+  taux_en_emploi_6_mois: number | null;
+  taux_en_formation: number | null;
+  updatedAt: Generated<Timestamp>;
+  voie: string;
+}
+
 export interface Log {
   createdAt: Generated<Timestamp>;
   data: Json | null;
@@ -245,6 +261,7 @@ export interface DB {
   geometry_columns: GeometryColumns;
   indicateurEntree: IndicateurEntree;
   indicateurPoursuite: IndicateurPoursuite;
+  indicateurPoursuiteRegional: IndicateurPoursuiteRegional;
   log: Log;
   metric: Metric;
   rawData: RawData;

server/src/common/repositories/base.ts

@@ -6,7 +6,7 @@ import { upsert } from "../db/db";
 import { InsertExpression } from "kysely/dist/cjs/parser/insert-values-parser";
 import { Readable } from "stream";
 import { compose, transformData } from "oleoduc";
-import { Entries } from "../utils/tsUtils";
+import { Entries } from "shared";
 
 export class Repository {}
 

server/src/common/repositories/etablissement.ts

@@ -25,6 +25,7 @@ export class EtablissementRepository extends SqlRepository<DB, "etablissement">
         updatedAt: null,
         url: null,
         hasFormation: null,
+        region: null,
       },
       kdb
     );

server/src/common/repositories/formation.ts

@@ -11,6 +11,7 @@ export class FormationRepository extends SqlRepository<DB, "formation"> {
         cfd: null,
         codeDiplome: null,
         codeDispositif: null,
+        niveauDiplome: null,
         codeRncp: null,
         createdAt: null,
         description: null,

server/src/common/repositories/formationEtablissement.ts

@@ -7,6 +7,8 @@ import { Readable } from "stream";
 import { compose, transformData } from "oleoduc";
 import { sql } from "kysely";
 import { merge } from "lodash-es";
+import IndicateurPoursuiteRegionalRepository from "./indicateurPoursuiteRegional.js";
+import { getDiplomeType } from "shared";
 
 type QueryFormationEtablissement = Partial<{ [key in keyof DB["formationEtablissement"]]: string }>;
 type QueryEtablissement = Partial<{ [key in keyof DB["etablissement"]]: string }>;
@@ -121,8 +123,17 @@ export class FormationEtablissementRepository extends SqlRepository<DB, "formati
         .limit(1)
         .executeTakeFirst();
 
+      const diplomeType = getDiplomeType(formationEtablissement.formation.niveauDiplome);
+      const indicateurPoursuiteRegional = diplomeType
+        ? await IndicateurPoursuiteRegionalRepository.quartileFor(
+            diplomeType,
+            formationEtablissement.etablissement.region,
+            formationEtablissement.formation.voie
+          )
+        : null;
+
       formationEtablissement = merge(formationEtablissement, {
-        formationEtablissement: { indicateurEntree, indicateurPoursuite },
+        formationEtablissement: { indicateurEntree, indicateurPoursuite, indicateurPoursuiteRegional },
       });
     }
 

server/src/common/repositories/indicateurPoursuiteRegional.ts

@@ -0,0 +1,66 @@
+import { SqlRepository } from "./base.js";
+import { kdb as defaultKdb } from "../db/db.js";
+import { DB } from "../db/schema.js";
+import { DiplomeType, FormationVoie } from "shared";
+import { sql } from "kysely";
+
+export class IndicateurPoursuiteRegionalRepository extends SqlRepository<DB, "indicateurPoursuiteRegional"> {
+  constructor(kdb = defaultKdb) {
+    super(
+      "indicateurPoursuiteRegional",
+      {
+        id: null,
+        cfd: null,
+        voie: null,
+        codeDispositif: null,
+        region: null,
+        millesime: null,
+        taux_en_emploi_6_mois: null,
+        taux_en_formation: null,
+        taux_autres_6_mois: null,
+        createdAt: null,
+        updatedAt: null,
+      },
+      kdb
+    );
+  }
+
+  quartileFor(type: keyof typeof DiplomeType, region?: string, voie?: FormationVoie) {
+    let query = this.kdb
+      .selectFrom("indicateurPoursuiteRegional")
+      .distinctOn(["region", "voie"])
+      .select(["millesime", "region", "voie"])
+      .select(sql<number>`PERCENTILE_CONT(0) WITHIN GROUP(ORDER BY taux_en_formation)`.as("taux_en_formation_q0"))
+      .select(sql<number>`PERCENTILE_CONT(0.25) WITHIN GROUP(ORDER BY taux_en_formation)`.as("taux_en_formation_q1"))
+      .select(sql<number>`PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY taux_en_formation)`.as("taux_en_formation_q2"))
+      .select(sql<number>`PERCENTILE_CONT(0.75) WITHIN GROUP(ORDER BY taux_en_formation)`.as("taux_en_formation_q3"))
+      .select(sql<number>`PERCENTILE_CONT(1) WITHIN GROUP(ORDER BY taux_en_formation)`.as("taux_en_formation_q4"))
+      .select(
+        sql<number>`PERCENTILE_CONT(0) WITHIN GROUP(ORDER BY taux_en_emploi_6_mois)`.as("taux_en_emploi_6_mois_q0")
+      )
+      .select(
+        sql<number>`PERCENTILE_CONT(0.25) WITHIN GROUP(ORDER BY taux_en_emploi_6_mois)`.as("taux_en_emploi_6_mois_q1")
+      )
+      .select(
+        sql<number>`PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY taux_en_emploi_6_mois)`.as("taux_en_emploi_6_mois_q2")
+      )
+      .select(
+        sql<number>`PERCENTILE_CONT(0.75) WITHIN GROUP(ORDER BY taux_en_emploi_6_mois)`.as("taux_en_emploi_6_mois_q3")
+      )
+      .select(
+        sql<number>`PERCENTILE_CONT(1) WITHIN GROUP(ORDER BY taux_en_emploi_6_mois)`.as("taux_en_emploi_6_mois_q4")
+      )
+
+      .where("taux_en_formation", "is not", null)
+      .where(this.kdb.fn("SUBSTR", ["cfd", sql.val(1), sql.val(3)]), "in", DiplomeType[type]);
+    query = region ? query.where("region", "=", region) : query;
+    query = voie ? query.where("voie", "=", voie) : query;
+    return query
+      .groupBy(["region", "voie", "millesime"])
+      .orderBy(["region", "voie", "millesime desc"])
+      .limit(1)
+      .executeTakeFirst();
+  }
+}
+
+export default new IndicateurPoursuiteRegionalRepository();

server/src/common/repositories/romeMetier.ts

@@ -13,6 +13,7 @@ export class RomeMetierRepository extends SqlRepository<DB, "romeMetier"> {
         onisepLibelle: null,
         onisepLink: null,
         franceTravailLink: null,
+        franceTravailLibelle: null,
         transitionEcologique: null,
         transitionEcologiqueDetaillee: null,
         transitionNumerique: null,

server/src/common/utils/formatUtils.ts

@@ -47,7 +47,7 @@ export function formatLibelle(libelle: string): string {
   );
 }
 
-export function formatUrl(url: string | null): string {
+export function formatUrl(url: string | null): string | null {
   if (!url) {
     return null;
   }

server/src/common/utils/objectUtils.js

@@ -1,4 +1,4 @@
-import { pickBy, pick, isNil, sortBy, isArray } from "lodash-es";
+import { pickBy, isNil, isArray } from "lodash-es";
 
 export function omitNil(obj) {
   if (isArray(obj)) {
@@ -7,24 +7,3 @@ export function omitNil(obj) {
 
   return pickBy(obj, (v) => !isNil(v));
 }
-
-export function pickWithNil(obj, keys = []) {
-  return obj ? pick(obj, keys) : null;
-}
-
-export function mergeArray(existingArray, newArray, discriminator, options = {}) {
-  const updated = newArray.map((element) => {
-    const previous = existingArray.find((e) => e[discriminator] === element[discriminator]) || {};
-    return {
-      ...previous,
-      ...element,
-      ...(options.merge ? options.merge(previous, element) : {}),
-    };
-  });
-
-  const untouched = existingArray.filter((p) => {
-    return !updated.map((u) => u[discriminator]).includes(p[discriminator]);
-  });
-
-  return sortBy([...updated, ...untouched], discriminator);
-}

server/src/config.ts

@@ -99,11 +99,11 @@ const config = {
   },
   rco: {
     bucket: {
-      endpoint: env.get("RCO_BUCKET_ENDPOINT").asString(),
-      region: env.get("RCO_BUCKET_REGION").asString(),
-      accessKey: env.get("RCO_BUCKET_ACCESS_KEY").asString(),
-      secretKey: env.get("RCO_BUCKET_SECRET_KEY").asString(),
-      name: env.get("RCO_BUCKET_NAME").asString(),
+      endpoint: env.get("RCO_BUCKET_ENDPOINT").required().asString(),
+      region: env.get("RCO_BUCKET_REGION").required().asString(),
+      accessKey: env.get("RCO_BUCKET_ACCESS_KEY").required().asString(),
+      secretKey: env.get("RCO_BUCKET_SECRET_KEY").required().asString(),
+      name: env.get("RCO_BUCKET_NAME").required().asString(),
     },
     file: {
       certifInfo: "inserJeune-certifinfo.csv",

server/src/http/routes/formationsRoutes.ts

@@ -6,13 +6,13 @@ import * as validators from "#src/http/utils/validators.js";
 import { validate } from "#src/http/utils/validators.js";
 import { addJsonHeaders } from "#src/http/utils/responseUtils.js";
 import { getRouteDate, getFormationsSQL } from "#src/queries/getFormations.js";
-import { FORMATION_TAG } from "#src/common/constants/formationEtablissement.js";
 import FormationEtablissement from "#src/common/repositories/formationEtablissement";
 import { GraphHopperApi } from "#src/services/graphHopper/graphHopper.js";
 import { stripNull } from "../utils/formatters";
 import { getFormationsSimilaire } from "#src/queries/getFormationSimilaire.js";
 import EtablissementIsochroneRepository from "#src/common/repositories/etablissementIsochrone.js";
 import { merge } from "lodash-es";
+import { FormationTag } from "shared";
 
 export default () => {
   const router = express.Router();
@@ -69,7 +69,7 @@ export default () => {
             timeLimit: Joi.number().valid().default(null),
             tag: Joi.string()
               .empty("")
-              .valid(...Object.values(FORMATION_TAG))
+              .valid(...Object.values(FormationTag))
               .default(null),
             ...validators.uais(),
             ...validators.cfds(),

server/src/jobs/etablissements/importEtablissements.ts

@@ -8,6 +8,7 @@ import { get } from "lodash-es";
 import { kdb, kyselyChainFn, upsert } from "#src/common/db/db";
 import { sql } from "kysely";
 import { formatUrl } from "#src/common/utils/formatUtils.js";
+import { RegionsService } from "shared";
 
 const logger = getLoggerWithContext("import");
 
@@ -43,26 +44,33 @@ export async function importEtablissements() {
     await RawDataRepository.search(RawDataType.ACCE),
     filterData((data) => {
       // Filtre les établissement qui nous intéressent (lycée, CFA ...)
-      return data.data.nature_uai.match(/^[2345678]/);
+      return data.data.nature_uai.match(/^[2345678]/) && data.data.pays_libe.toLowerCase() === "france";
     }),
     transformData(({ data }) => {
-      return {
-        data,
-        // TODO: validate data
-        formated: {
-          uai: data.numero_uai,
-          libelle: data.appellation_officielle,
-          url: formatUrl(data.site_web),
-          ...formatStatutEtablissement(data.secteur_public_prive_libe),
-
-          addressStreet: data.adresse_uai,
-          addressPostCode: data.code_postal_uai,
-          addressCity: data.commune_libe,
-          addressCedex: /cedex/i.test(data.localite_acheminement_uai),
-          academie: data.academie,
-        },
-      };
+      try {
+        return {
+          data,
+          // TODO: validate data
+          formated: {
+            uai: data.numero_uai,
+            libelle: data.appellation_officielle,
+            url: formatUrl(data.site_web),
+            ...formatStatutEtablissement(data.secteur_public_prive_libe),
+            addressStreet: data.adresse_uai,
+            addressPostCode: data.commune,
+            addressCity: data.commune_libe,
+            addressCedex: /cedex/i.test(data.localite_acheminement_uai),
+            academie: data.academie,
+            region: data.commune ? RegionsService.findRegionByCodePostal(data.commune)?.code : null,
+          },
+        };
+      } catch (err) {
+        logger.error(err);
+        stats.failed++;
+        return null;
+      }
     }),
+    filterData((data) => data),
     // Ajout des données de l'onisep
     transformData(
       async ({ data, formated }) => {