feat: opco import file

mission-apprentissage · Sep 5, 2023 · 2e8fdd1 · 2e8fdd1
1 parent 7f0d54c
commit 2e8fdd1
Show file tree

Hide file tree

Showing 8 changed files with 204 additions and 66 deletions.
diff --git a/server/src/db/migrations/20230905150329-organisation_person_sources.ts b/server/src/db/migrations/20230905150329-organisation_person_sources.ts
@@ -0,0 +1,69 @@
+import { Db, MongoClient } from "mongodb";
+
+export const up = async (db: Db, _client: MongoClient) => {
+  await db.collection("organisations").updateMany(
+    {
+      "_meta.source": { $exists: true, $ne: "" },
+    },
+    [
+      {
+        $set: {
+          "_meta.sources": ["$_meta.source"],
+        },
+      },
+      {
+        $unset: "_meta.source",
+      },
+    ]
+  );
+
+  await db.collection("persons").updateMany(
+    {
+      "_meta.source": { $exists: true, $ne: "" },
+    },
+    [
+      {
+        $set: {
+          "_meta.sources": ["$_meta.source"],
+        },
+      },
+      {
+        $unset: "_meta.source",
+      },
+    ]
+  );
+};
+
+export const down = async (db: Db, _client: MongoClient) => {
+  db.collection("organisations").updateMany(
+    {
+      "_meta.sources": { $exists: true },
+    },
+    [
+      {
+        $set: {
+          "_meta.source": { $arrayElemAt: ["$_meta.sources", 0] },
+        },
+      },
+      {
+        $unset: "_meta.sources",
+      },
+    ]
+  );
+
+  db.collection("persons").updateMany(
+    {
+      "_meta.sources": { $exists: true },
+    },
+    [
+      {
+        $set: {
+          "_meta.source": { $arrayElemAt: ["$_meta.sources", 0] },
+        },
+      },
+      {
+        $unset: "_meta.sources",
+      },
+    ]
+  );
+};
diff --git a/server/src/modules/actions/deca.actions.ts b/server/src/modules/actions/deca.actions.ts
@@ -1,14 +1,13 @@
 import companyEmailValidator from "company-email-validator";
 import { IPostRoutes, IResponse } from "shared";
+import { DOCUMENT_TYPES } from "shared/constants/documents";
 import { SIRET_REGEX } from "shared/constants/regex";
 import { getSirenFromSiret } from "shared/helpers/common";
-import { IOrganisation } from "shared/models/organisation.model";
 
 import { getDbCollection } from "../../common/utils/mongodbUtils";
 import {
-  findOrCreateOrganisation,
   findOrganisation,
-  updateOrganisation,
+  updateOrganisationData,
 } from "./organisations.actions";
 import { findPerson } from "./persons.actions";
 
@@ -49,38 +48,12 @@ export const importDecaContent = async (emails: string[], siret: string) => {
   const siren = getSirenFromSiret(siret);
   const domains = [...new Set(uniqueEmails.map((e) => e.split("@")[1]))];
 
-  const organisation = await findOrCreateOrganisation(
-    { siren },
-    {
-      siren,
-      etablissements: [{ siret }],
-      email_domains: domains,
-    }
-  );
-
-  const updateOrganisationData: Partial<IOrganisation> = {};
-  const etablissement = organisation.etablissements?.find(
-    (e) => e.siret === siret
-  );
-
-  if (!etablissement) {
-    const etablissements = organisation.etablissements ?? [];
-    etablissements.push({ siret });
-    updateOrganisationData.etablissements = etablissements;
-  }
-
-  const newDomains = domains.filter(
-    (domain) =>
-      !organisation.email_domains?.includes(domain) &&
-      companyEmailValidator.isCompanyDomain(domain)
-  );
-
-  if (newDomains.length) {
-    updateOrganisationData.email_domains = organisation.email_domains ?? [];
-    updateOrganisationData.email_domains?.push(...newDomains);
-  }
-
-  await updateOrganisation(organisation, updateOrganisationData);
+  const organisation = await updateOrganisationData({
+    siren,
+    sirets: [siret],
+    email_domains: domains,
+    source: DOCUMENT_TYPES.DECA,
+  });
 
   await Promise.all(
     uniqueEmails.map((email) =>
@@ -90,7 +63,7 @@ export const importDecaContent = async (emails: string[], siret: string) => {
         },
         {
           $addToSet: {
-            organisations: organisation._id.toString(),
+            ...(organisation && { organisations: organisation._id.toString() }),
             sirets: siret,
           },
           $setOnInsert: {

diff --git a/server/src/modules/actions/documents.actions.ts b/server/src/modules/actions/documents.actions.ts
@@ -44,6 +44,7 @@ import { MAILING_LIST_DOCUMENT_PREFIX } from "./mailingLists.actions";
 import {
   importOcapiatContent,
   IOcapiatParsedContentLine,
+  parseOcapiatContentLine,
 } from "./ocapiat.actions";
 
 const testMode = config.env === "test";
@@ -527,6 +528,12 @@ export const handleDocumentFileContent = async ({
         formatter: parseContentLine,
       });
       break;
+    case DOCUMENT_TYPES.OCAPIAT:
+      await extractDocumentContent({
+        document,
+        formatter: parseOcapiatContentLine,
+      });
+      break;
     case DOCUMENT_TYPES.CONSTRUCTYS:
       await extractDocumentContent({
         document,

diff --git a/server/src/modules/actions/ocapiat.actions.ts b/server/src/modules/actions/ocapiat.actions.ts
@@ -2,16 +2,16 @@ import companyEmailValidator from "company-email-validator";
 import { DOCUMENT_TYPES } from "shared/constants/documents";
 import { getSirenFromSiret } from "shared/helpers/common";
 
-import { findOrCreateOrganisation } from "./organisations.actions";
-import { createPerson } from "./persons.actions";
+import { getDbCollection } from "../../common/utils/mongodbUtils";
+import { updateOrganisationData } from "./organisations.actions";
 
 export interface IOcapiatParsedContentLine {
-  "RÈgion INSEE"?: string;
+  "Région INSEE"?: string;
   "GUH Etablissement"?: string;
   "Code OPSI"?: string;
   Siret?: string;
   "Raison sociale"?: string;
-  DÈpartement?: string;
+  Département?: string;
   "Etablissement Sous Contrat"?: string;
   "Nom du groupe"?: string;
   "Effectif Etab moyen annuel"?: string;
@@ -23,14 +23,29 @@ export interface IOcapiatParsedContentLine {
   Conseiller?: string;
   Civilite?: string;
   "Nom du contact"?: string;
-  "PrÈnom du contact"?: string;
+  "Prénom du contact"?: string;
   "Titre du contact"?: string;
   "Fonction du contact"?: string;
-  "TÈl contact"?: string;
+  "Tél contact"?: string;
   "Mobile contact"?: string;
-  "Email du contact": string;
+  "Email du contact"?: string;
 }
 
+export const parseOcapiatContentLine = (
+  line: IOcapiatParsedContentLine
+): IOcapiatParsedContentLine | undefined => {
+  // remove attributes where value is "-" considered empty
+  const content = Object.entries(line).reduce<IOcapiatParsedContentLine>(
+    (acc, [key, value]) => ({
+      ...acc,
+      ...(value === "-" ? {} : { [key]: value }),
+    }),
+    {}
+  );
+
+  return content;
+};
+
 export const importOcapiatContent = async (
   content: IOcapiatParsedContentLine
 ) => {
@@ -39,31 +54,47 @@ export const importOcapiatContent = async (
   const email = content?.["Email du contact"];
   let domains: string[] = [];
 
-  if (companyEmailValidator.isCompanyEmail(email)) {
+  if (email && companyEmailValidator.isCompanyEmail(email)) {
     domains = [email.split("@")[1]];
   }
 
-  // create organisation
-  const organisation = await findOrCreateOrganisation(
-    { siren },
+  const organisation = await updateOrganisationData({
+    siren,
+    sirets: [siret],
+    email_domains: domains,
+    source: DOCUMENT_TYPES.OCAPIAT,
+  });
+
+  if (!email) return;
+
+  const date = new Date();
+
+  getDbCollection("persons").updateOne(
+    {
+      email,
+    },
     {
-      siren,
-      etablissements: [{ siret }],
-      email_domains: domains,
-      _meta: {
-        source: DOCUMENT_TYPES.OCAPIAT,
+      $set: {
+        updated_at: date,
+      },
+      $addToSet: {
+        ...(organisation && { organisations: organisation._id.toString() }),
+        sirets: siret,
+        "_meta.sources": DOCUMENT_TYPES.OCAPIAT,
+      },
+      $setOnInsert: {
+        email,
+        ...(content?.["Nom du contact"] && {
+          nom: content?.["Nom du contact"],
+        }),
+        ...(content?.["Prénom du contact"] && {
+          prenom: content?.["Prénom du contact"],
+        }),
+        created_at: date,
       },
+    },
+    {
+      upsert: true,
     }
   );
-
-  // create person
-  await createPerson({
-    email: email,
-    organisations: [organisation?._id.toString()],
-    _meta: {
-      source: DOCUMENT_TYPES.OCAPIAT,
-    },
-  });
-
-  // todo handle already existing resources
 };
diff --git a/server/src/modules/actions/organisations.actions.ts b/server/src/modules/actions/organisations.actions.ts
@@ -1,4 +1,5 @@
 import { internal } from "@hapi/boom";
+import companyEmailValidator from "company-email-validator";
 import { Filter, FindOptions } from "mongodb";
 import { IPostRoutes, IResponse } from "shared";
 import { getSirenFromSiret } from "shared/helpers/common";
@@ -152,3 +153,60 @@ export const updateOrganisation = async (
     { returnDocument: "after" }
   );
 };
+
+interface IUpdateOrganisationData {
+  siren: string;
+  sirets: string[];
+  email_domains: string[];
+  source: string;
+}
+
+export const updateOrganisationData = async ({
+  siren,
+  sirets,
+  email_domains,
+  source,
+}: IUpdateOrganisationData) => {
+  const organisation = await findOrCreateOrganisation(
+    { siren },
+    {
+      siren,
+      etablissements: sirets.map((siret) => ({ siret })),
+      email_domains: email_domains,
+    }
+  );
+
+  const sources = organisation._meta?.sources ?? [];
+  const newSources = [...new Set([...sources, source])];
+
+  const updateOrganisationData: Partial<IOrganisation> = {
+    etablissements: organisation.etablissements ?? [],
+    email_domains: organisation.email_domains ?? [],
+    _meta: { sources: newSources },
+  };
+
+  for (const siret of sirets) {
+    const etablissement = updateOrganisationData.etablissements?.find(
+      (e) => e.siret === siret
+    );
+
+    if (!etablissement) {
+      updateOrganisationData.etablissements?.push({ siret });
+    }
+  }
+
+  const newDomains = email_domains.filter(
+    (domain) =>
+      !organisation.email_domains?.includes(domain) &&
+      companyEmailValidator.isCompanyDomain(domain)
+  );
+
+  if (newDomains.length) {
+    updateOrganisationData.email_domains = organisation.email_domains ?? [];
+    updateOrganisationData.email_domains?.push(...newDomains);
+  }
+
+  await updateOrganisation(organisation, updateOrganisationData);
+
+  return findOrganisation({ _id: organisation._id });
+};
diff --git a/server/src/modules/jobs/seed/clear.ts b/server/src/modules/jobs/seed/clear.ts
@@ -2,7 +2,7 @@ import { getDbCollection } from "../../../common/utils/mongodbUtils";
 import config from "../../../config";
 
 export const clear = async () => {
-  if (!["recette", "sandbox"].includes(config.env)) {
+  if (!["recette", "sandbox", "local"].includes(config.env)) {
     return;
   }
 

diff --git a/shared/models/organisation.model.ts b/shared/models/organisation.model.ts
@@ -54,7 +54,7 @@ export const ZOrganisation = z
       .describe("Liste des établissements"),
     _meta: z
       .object({
-        source: z.string().optional(),
+        sources: z.array(z.string()).optional(),
       })
       .nonstrict()
       .optional()

diff --git a/shared/models/person.model.ts b/shared/models/person.model.ts
@@ -37,7 +37,7 @@ export const ZPerson = z
       .describe("Liste de sirets recensé (sécurisation qualité de la donnée)"),
     _meta: z
       .object({
-        source: z.string().optional(),
+        sources: z.array(z.string()).optional(),
       })
       .describe("Métadonnées")
       .nonstrict()