Update to INSEE/Sirene 3.11 (#637)

* [UniteLegale] Remove / Set "CaractereEmployeur" to null * [Etablissement] Add new data & remove sync for null fields * [Insee] Update BASE URL to v3.11 * [SyncInsee] Minimum date for v3.11 * [SyncInsee] Remove min date
Creatiwity · Apr 11, 2024 · 71a2067 · 71a2067
1 parent 8922089
commit 71a2067
Show file tree

Hide file tree

Showing 14 changed files with 109 additions and 60 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,6 +1,10 @@
 {
     "editor.codeActionsOnSave": {
-        "source.fixAll": true,
-        "source.organizeImports": true
+        "source.fixAll": "explicit",
+        "source.organizeImports": "explicit"
+    },
+    "[rust]": {
+        "editor.defaultFormatter": "rust-lang.rust-analyzer",
+        "editor.formatOnSave": true
     }
 }
diff --git a/migrations/2024-02-27-161209_siren_3_11/down.sql b/migrations/2024-02-27-161209_siren_3_11/down.sql
@@ -0,0 +1,13 @@
+ALTER TABLE "public"."etablissement"
+DROP COLUMN IF EXISTS "dernier_numero_voie",
+DROP COLUMN IF EXISTS "indice_repetition_dernier_numero_voie",
+DROP COLUMN IF EXISTS "identifiant_adresse",
+DROP COLUMN IF EXISTS "coordonnee_lambert_x",
+DROP COLUMN IF EXISTS "coordonnee_lambert_y";
+
+ALTER TABLE "public"."etablissement_staging"
+DROP COLUMN IF EXISTS "dernier_numero_voie",
+DROP COLUMN IF EXISTS "indice_repetition_dernier_numero_voie",
+DROP COLUMN IF EXISTS "identifiant_adresse",
+DROP COLUMN IF EXISTS "coordonnee_lambert_x",
+DROP COLUMN IF EXISTS "coordonnee_lambert_y";
diff --git a/migrations/2024-02-27-161209_siren_3_11/up.sql b/migrations/2024-02-27-161209_siren_3_11/up.sql
@@ -0,0 +1,13 @@
+ALTER TABLE "public"."etablissement"
+ADD COLUMN "dernier_numero_voie" text DEFAULT NULL,
+ADD COLUMN "indice_repetition_dernier_numero_voie" text DEFAULT NULL,
+ADD COLUMN "identifiant_adresse" text DEFAULT NULL,
+ADD COLUMN "coordonnee_lambert_x" text DEFAULT NULL,
+ADD COLUMN "coordonnee_lambert_y" text DEFAULT NULL;
+
+ALTER TABLE "public"."etablissement_staging"
+ADD COLUMN "dernier_numero_voie" text DEFAULT NULL,
+ADD COLUMN "indice_repetition_dernier_numero_voie" text DEFAULT NULL,
+ADD COLUMN "identifiant_adresse" text DEFAULT NULL,
+ADD COLUMN "coordonnee_lambert_x" text DEFAULT NULL,
+ADD COLUMN "coordonnee_lambert_y" text DEFAULT NULL;
diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs
@@ -10,7 +10,7 @@ use chrono::NaiveDateTime;
 
 const MAX_CALL: u8 = 20;
 const MAX_DURATION: std::time::Duration = std::time::Duration::from_secs(60);
-const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3";
+const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3.11";
 pub const INITIAL_CURSOR: &str = "*";
 
 struct EndpointConfig {

diff --git a/src/connectors/insee/types/etablissement.rs b/src/connectors/insee/types/etablissement.rs
@@ -32,7 +32,6 @@ pub struct InseeEtablissementInner {
     pub etablissement_siege: bool,
     pub nombre_periodes_etablissement: Option<i32>,
     pub adresse_etablissement: InseeAdresseEtablissement,
-    pub adresse2_etablissement: InseeAdresse2Etablissement,
 }
 
 #[derive(Deserialize, Debug)]
@@ -55,6 +54,8 @@ pub struct InseeAdresseEtablissement {
     pub complement_adresse_etablissement: Option<String>,
     pub numero_voie_etablissement: Option<String>,
     pub indice_repetition_etablissement: Option<String>,
+    pub dernier_numero_voie_etablissement: Option<String>,
+    pub indice_repetition_dernier_numero_voie_etablissement: Option<String>,
     pub type_voie_etablissement: Option<String>,
     pub libelle_voie_etablissement: Option<String>,
     pub code_postal_etablissement: Option<String>,
@@ -66,25 +67,9 @@ pub struct InseeAdresseEtablissement {
     pub libelle_cedex_etablissement: Option<String>,
     pub code_pays_etranger_etablissement: Option<String>,
     pub libelle_pays_etranger_etablissement: Option<String>,
-}
-
-#[derive(Deserialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct InseeAdresse2Etablissement {
-    complement_adresse2_etablissement: Option<String>,
-    numero_voie2_etablissement: Option<String>,
-    indice_repetition2_etablissement: Option<String>,
-    type_voie2_etablissement: Option<String>,
-    libelle_voie2_etablissement: Option<String>,
-    code_postal2_etablissement: Option<String>,
-    libelle_commune2_etablissement: Option<String>,
-    libelle_commune_etranger2_etablissement: Option<String>,
-    distribution_speciale2_etablissement: Option<String>,
-    code_commune2_etablissement: Option<String>,
-    code_cedex2_etablissement: Option<String>,
-    libelle_cedex2_etablissement: Option<String>,
-    code_pays_etranger2_etablissement: Option<String>,
-    libelle_pays_etranger2_etablissement: Option<String>,
+    pub identifiant_adresse_etablissement: Option<String>,
+    pub coordonnee_lambert_abscisse_etablissement: Option<String>,
+    pub coordonnee_lambert_ordonnee_etablissement: Option<String>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -121,7 +106,6 @@ impl From<&InseeEtablissement> for Option<Etablissement> {
 impl From<InseeEtablissementWithPeriode> for Etablissement {
     fn from(e: InseeEtablissementWithPeriode) -> Self {
         let adresse = e.content.adresse_etablissement;
-        let adresse2 = e.content.adresse2_etablissement;
 
         Etablissement {
             siret: e.content.siret,
@@ -151,20 +135,26 @@ impl From<InseeEtablissementWithPeriode> for Etablissement {
             libelle_cedex: adresse.libelle_cedex_etablissement,
             code_pays_etranger: adresse.code_pays_etranger_etablissement,
             libelle_pays_etranger: adresse.libelle_pays_etranger_etablissement,
-            complement_adresse2: adresse2.complement_adresse2_etablissement,
-            numero_voie_2: adresse2.numero_voie2_etablissement,
-            indice_repetition_2: adresse2.indice_repetition2_etablissement,
-            type_voie_2: adresse2.type_voie2_etablissement,
-            libelle_voie_2: adresse2.libelle_voie2_etablissement,
-            code_postal_2: adresse2.code_postal2_etablissement,
-            libelle_commune_2: adresse2.libelle_commune2_etablissement,
-            libelle_commune_etranger_2: adresse2.libelle_commune_etranger2_etablissement,
-            distribution_speciale_2: adresse2.distribution_speciale2_etablissement,
-            code_commune_2: adresse2.code_commune2_etablissement,
-            code_cedex_2: adresse2.code_cedex2_etablissement,
-            libelle_cedex_2: adresse2.libelle_cedex2_etablissement,
-            code_pays_etranger_2: adresse2.code_pays_etranger2_etablissement,
-            libelle_pays_etranger_2: adresse2.libelle_pays_etranger2_etablissement,
+            dernier_numero_voie: adresse.dernier_numero_voie_etablissement,
+            identifiant_adresse: adresse.identifiant_adresse_etablissement,
+            indice_repetition_dernier_numero_voie: adresse
+                .indice_repetition_dernier_numero_voie_etablissement,
+            coordonnee_lambert_x: adresse.coordonnee_lambert_abscisse_etablissement,
+            coordonnee_lambert_y: adresse.coordonnee_lambert_ordonnee_etablissement,
+            complement_adresse2: None,
+            numero_voie_2: None,
+            indice_repetition_2: None,
+            type_voie_2: None,
+            libelle_voie_2: None,
+            code_postal_2: None,
+            libelle_commune_2: None,
+            libelle_commune_etranger_2: None,
+            distribution_speciale_2: None,
+            code_commune_2: None,
+            code_cedex_2: None,
+            libelle_cedex_2: None,
+            code_pays_etranger_2: None,
+            libelle_pays_etranger_2: None,
             date_debut: e.periode.date_debut,
             etat_administratif: e.periode.etat_administratif_etablissement,
             enseigne_1: e.periode.enseigne1_etablissement,

diff --git a/src/connectors/insee/types/unite_legale.rs b/src/connectors/insee/types/unite_legale.rs
@@ -82,7 +82,6 @@ pub struct PeriodeInseeUniteLegale {
     pub nomenclature_activite_principale_unite_legale: Option<String>,
     pub nic_siege_unite_legale: Option<String>,
     pub economie_sociale_solidaire_unite_legale: Option<String>,
-    pub caractere_employeur_unite_legale: Option<String>,
     pub societe_mission_unite_legale: Option<String>,
 }
 
@@ -138,8 +137,8 @@ impl From<InseeUniteLegaleWithPeriode> for UniteLegale {
                 .nomenclature_activite_principale_unite_legale,
             nic_siege: u.periode.nic_siege_unite_legale,
             economie_sociale_solidaire: u.periode.economie_sociale_solidaire_unite_legale,
-            caractere_employeur: u.periode.caractere_employeur_unite_legale,
             societe_mission: u.periode.societe_mission_unite_legale,
+            caractere_employeur: None,
         }
     }
 }

diff --git a/src/models/etablissement/columns.rs b/src/models/etablissement/columns.rs
@@ -13,6 +13,8 @@ nombre_periodes,
 complement_adresse,
 numero_voie,
 indice_repetition,
+dernier_numero_voie,
+indice_repetition_dernier_numero_voie,
 type_voie,
 libelle_voie,
 code_postal,
@@ -24,6 +26,9 @@ code_cedex,
 libelle_cedex,
 code_pays_etranger,
 libelle_pays_etranger,
+identifiant_adresse,
+coordonnee_lambert_x,
+coordonnee_lambert_y,
 complement_adresse2,
 numero_voie_2,
 indice_repetition_2,

diff --git a/src/models/etablissement/common.rs b/src/models/etablissement/common.rs
@@ -1,6 +1,6 @@
-use diesel::prelude::*;
 use super::super::schema::etablissement;
 use chrono::{NaiveDate, NaiveDateTime};
+use diesel::prelude::*;
 use serde::Serialize;
 
 #[derive(Insertable, Queryable, Serialize, Clone, Debug)]
@@ -54,4 +54,9 @@ pub struct Etablissement {
     pub activite_principale: Option<String>,
     pub nomenclature_activite_principale: Option<String>,
     pub caractere_employeur: Option<String>,
+    pub dernier_numero_voie: Option<String>,
+    pub indice_repetition_dernier_numero_voie: Option<String>,
+    pub identifiant_adresse: Option<String>,
+    pub coordonnee_lambert_x: Option<String>,
+    pub coordonnee_lambert_y: Option<String>,
 }
diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs
@@ -20,14 +20,20 @@ pub fn get(connection: &mut Connection, siret: &str) -> Result<Etablissement, Er
         .map_err(|error| error.into())
 }
 
-pub fn get_with_siren(connection: &mut Connection, siren: &str) -> Result<Vec<Etablissement>, Error> {
+pub fn get_with_siren(
+    connection: &mut Connection,
+    siren: &str,
+) -> Result<Vec<Etablissement>, Error> {
     dsl::etablissement
         .filter(dsl::siren.eq(siren))
         .load::<Etablissement>(connection)
         .map_err(|error| error.into())
 }
 
-pub fn get_siege_with_siren(connection: &mut Connection, siren: &str) -> Result<Etablissement, Error> {
+pub fn get_siege_with_siren(
+    connection: &mut Connection,
+    siren: &str,
+) -> Result<Etablissement, Error> {
     dsl::etablissement
         .filter(dsl::siren.eq(siren).and(dsl::etablissement_siege.eq(true)))
         .first::<Etablissement>(connection)
@@ -77,10 +83,8 @@ impl UpdatableModel for EtablissementModel {
     fn swap(&self, connectors: &Connectors) -> Result<(), UpdatableError> {
         let mut connection = connectors.local.pool.get()?;
         connection.build_transaction().read_write().run(|conn| {
-            sql_query("ALTER TABLE etablissement RENAME TO etablissement_temp")
-                .execute(conn)?;
-            sql_query("ALTER TABLE etablissement_staging RENAME TO etablissement")
-                .execute(conn)?;
+            sql_query("ALTER TABLE etablissement RENAME TO etablissement_temp").execute(conn)?;
+            sql_query("ALTER TABLE etablissement_staging RENAME TO etablissement").execute(conn)?;
             sql_query("ALTER TABLE etablissement_temp RENAME TO etablissement_staging")
                 .execute(conn)?;
             sql_query("TRUNCATE etablissement_staging").execute(conn)?;
@@ -203,6 +207,12 @@ impl UpdatableModel for EtablissementModel {
                 dsl::nomenclature_activite_principale
                     .eq(excluded(dsl::nomenclature_activite_principale)),
                 dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)),
+                dsl::dernier_numero_voie.eq(excluded(dsl::dernier_numero_voie)),
+                dsl::indice_repetition_dernier_numero_voie
+                    .eq(excluded(dsl::indice_repetition_dernier_numero_voie)),
+                dsl::identifiant_adresse.eq(excluded(dsl::identifiant_adresse)),
+                dsl::coordonnee_lambert_x.eq(excluded(dsl::coordonnee_lambert_x)),
+                dsl::coordonnee_lambert_y.eq(excluded(dsl::coordonnee_lambert_y)),
             ))
             .execute(&mut connection)?;
 

diff --git a/src/models/schema.rs b/src/models/schema.rs
@@ -55,6 +55,11 @@ diesel::table! {
         activite_principale -> Nullable<Text>,
         nomenclature_activite_principale -> Nullable<Text>,
         caractere_employeur -> Nullable<Text>,
+        dernier_numero_voie -> Nullable<Text>,
+        indice_repetition_dernier_numero_voie -> Nullable<Text>,
+        identifiant_adresse -> Nullable<Text>,
+        coordonnee_lambert_x -> Nullable<Text>,
+        coordonnee_lambert_y -> Nullable<Text>,
     }
 }
 
@@ -113,6 +118,11 @@ diesel::table! {
         activite_principale -> Nullable<Text>,
         nomenclature_activite_principale -> Nullable<Text>,
         caractere_employeur -> Nullable<Text>,
+        dernier_numero_voie -> Nullable<Text>,
+        indice_repetition_dernier_numero_voie -> Nullable<Text>,
+        identifiant_adresse -> Nullable<Text>,
+        coordonnee_lambert_x -> Nullable<Text>,
+        coordonnee_lambert_y -> Nullable<Text>,
     }
 }
 
@@ -173,9 +183,9 @@ diesel::table! {
         nic_siege -> Nullable<Text>,
         economie_sociale_solidaire -> Nullable<Text>,
         #[max_length = 1]
-        caractere_employeur -> Nullable<Varchar>,
-        #[max_length = 1]
         societe_mission -> Nullable<Varchar>,
+        #[max_length = 1]
+        caractere_employeur -> Nullable<Varchar>,
     }
 }
 
@@ -219,9 +229,9 @@ diesel::table! {
         nic_siege -> Nullable<Text>,
         economie_sociale_solidaire -> Nullable<Text>,
         #[max_length = 1]
-        caractere_employeur -> Nullable<Varchar>,
-        #[max_length = 1]
         societe_mission -> Nullable<Varchar>,
+        #[max_length = 1]
+        caractere_employeur -> Nullable<Varchar>,
     }
 }
 

diff --git a/src/models/unite_legale/columns.rs b/src/models/unite_legale/columns.rs
@@ -31,6 +31,6 @@ activite_principale,
 nomenclature_activite_principale,
 nic_siege,
 economie_sociale_solidaire,
-caractere_employeur,
-societe_mission
+societe_mission,
+caractere_employeur
 "#;
diff --git a/src/models/unite_legale/common.rs b/src/models/unite_legale/common.rs
@@ -1,6 +1,6 @@
-use diesel::prelude::*;
 use super::super::schema::unite_legale;
 use chrono::{NaiveDate, NaiveDateTime};
+use diesel::prelude::*;
 use serde::Serialize;
 
 #[derive(Insertable, Queryable, Serialize, Clone, Debug)]
@@ -38,6 +38,6 @@ pub struct UniteLegale {
     pub nomenclature_activite_principale: Option<String>,
     pub nic_siege: Option<String>,
     pub economie_sociale_solidaire: Option<String>,
-    pub caractere_employeur: Option<String>,
     pub societe_mission: Option<String>,
+    pub caractere_employeur: Option<String>,
 }
diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs
@@ -63,10 +63,8 @@ impl UpdatableModel for UniteLegaleModel {
     fn swap(&self, connectors: &Connectors) -> Result<(), UpdatableError> {
         let mut connection = connectors.local.pool.get()?;
         connection.build_transaction().read_write().run(|conn| {
-            sql_query("ALTER TABLE unite_legale RENAME TO unite_legale_temp")
-                .execute(conn)?;
-            sql_query("ALTER TABLE unite_legale_staging RENAME TO unite_legale")
-                .execute(conn)?;
+            sql_query("ALTER TABLE unite_legale RENAME TO unite_legale_temp").execute(conn)?;
+            sql_query("ALTER TABLE unite_legale_staging RENAME TO unite_legale").execute(conn)?;
             sql_query("ALTER TABLE unite_legale_temp RENAME TO unite_legale_staging")
                 .execute(conn)?;
             sql_query("TRUNCATE unite_legale_staging").execute(conn)?;
@@ -172,8 +170,8 @@ impl UpdatableModel for UniteLegaleModel {
                     .eq(excluded(dsl::nomenclature_activite_principale)),
                 dsl::nic_siege.eq(excluded(dsl::nic_siege)),
                 dsl::economie_sociale_solidaire.eq(excluded(dsl::economie_sociale_solidaire)),
-                dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)),
                 dsl::societe_mission.eq(excluded(dsl::societe_mission)),
+                dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)),
             ))
             .execute(&mut connection)?;
 

diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs
@@ -37,6 +37,8 @@ impl Action for SyncInseeAction {
                     planned_count,
                 )?;
 
+                debug!("Syncing {} {:#?}...", planned_count, group_type);
+
                 while let Some(cursor) = current_cursor {
                     let (next_cursor, inserted_count) = model
                         .update_daily_data(connectors, timestamp, cursor)