Keep label treatments when reimport (#112)

* prepare for reimport labeltreatments * wip * working labelTreatments reimportation * update concat and sder version * deploy on preprod - to revert * Oh, enable build * fix branch name for ci * change branch name * refacto preAssignation * small fix on preAssignation refacto * update sder * update sder again * fix concat * update sder for validation schema * remove reimported treatment when its no longer neccesary, not before * really delete reimportedTreatments * update concat order calculation * delete custom labelTreatmentsType * add labelTreatment date * fix gitlabci * docker login onoly for build * fine tune logs * fix tests * fix lint * docker login only for build, not for deploy * clean gitlabci and update sder version * treatment date from timestamp to iso string --------- Co-authored-by: Antoine Jeanneney <antoine.jeanneney@justice.fr>
Cour-de-cassation · Jul 12, 2024 · 3ab5bd6 · 3ab5bd6
1 parent c66b638
commit 3ab5bd6
Show file tree

Hide file tree

Showing 28 changed files with 391 additions and 109 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -43,6 +43,7 @@ build_label_backend:
     HTTP_PROXY: $HTTP_PROXY_DEV
     HTTPS_PROXY: $HTTPS_PROXY_DEV
   script:
+    - docker login -u $PUBLIC_DOCKER_USERNAME -p $PUBLIC_DOCKER_PASSWORD
     - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - docker build
       --build-arg http_proxy=$HTTP_PROXY
@@ -64,6 +65,7 @@ build_label_client:
     HTTP_PROXY: $HTTP_PROXY_DEV
     HTTPS_PROXY: $HTTPS_PROXY_DEV
   script:
+    - docker login -u $PUBLIC_DOCKER_USERNAME -p $PUBLIC_DOCKER_PASSWORD
     - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - export NODE_OPTIONS=--openssl-legacy-provider
     - docker build

diff --git a/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts b/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts
@@ -1,4 +1,4 @@
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 import { documentType, settingsType } from '@label/core';
 
 export type { nlpApiType, nlpResponseType, nlpLossType, nlpVersion };

diff --git a/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts b/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts
@@ -1,4 +1,4 @@
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 import { documentType, settingsType } from '@label/core';
 import { buildNlpApi } from './api';
 import { nlpMapper } from './mapper';

diff --git a/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts b/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts
@@ -1,4 +1,4 @@
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 import { idModule } from '@label/core';
 import { mapCourtDecisionToDocument } from './mapCourtDecisionToDocument';
 

diff --git a/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts b/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts
@@ -5,7 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 
 (async () => {
   const { settings } = await parametersHandler.getParameters();
-  const { documentNumber, source, lowPriority } = parseArgv();
+  const {
+    documentNumber,
+    source,
+    lowPriority,
+    keepLabelTreatments,
+  } = parseArgv();
   const backend = buildBackend(settings);
 
   backend.runScript(
@@ -14,6 +19,8 @@ import { parametersHandler } from '../lib/parametersHandler';
         documentNumber,
         source,
         lowPriority,
+        keepLabelTreatments,
+        settings,
       }),
     {
       shouldLoadDb: true,
@@ -40,6 +47,11 @@ function parseArgv() {
           'source (jurinet, jurica or juritj) of the document you want to import',
         type: 'string',
       },
+      keepLabelTreatments: {
+        demandOption: false,
+        description: 'import labelTreatments from SDER database if exist',
+        type: 'boolean',
+      },
     })
     .help()
     .alias('help', 'h').argv;
@@ -48,5 +60,6 @@ function parseArgv() {
     documentNumber: argv.documentNumber as number,
     lowPriority: !!argv.lowPriority as boolean,
     source: argv.source as string,
+    keepLabelTreatments: !!argv.keepLabelTreatments as boolean,
   };
 }
diff --git a/packages/courDeCassation/src/scripts/resetDocument.ts b/packages/courDeCassation/src/scripts/resetDocument.ts
@@ -9,7 +9,7 @@ import { sderConnector } from '../connector';
   const backend = buildBackend(settings);
 
   backend.runScript(
-    () => sderConnector.resetDocument({ documentNumber, source }),
+    () => sderConnector.resetDocument({ documentNumber, source, settings }),
     {
       shouldLoadDb: true,
     },

diff --git a/packages/courDeCassation/src/sderApi/sderApi.ts b/packages/courDeCassation/src/sderApi/sderApi.ts
@@ -256,6 +256,7 @@ const sderApi: sderApiType = {
     publishStatus,
   }) {
     //TODO : include publishStatus to dbsder api call
+    //TODO : manage labelTreatments before sending to dbsder-api (like in sder updateDecisionPseudonymisation function)
     if (process.env.DBSDER_API_ENABLED === 'true') {
       await fetchApi({
         method: 'put',

diff --git a/packages/courDeCassation/src/sderApi/sderApiType.ts b/packages/courDeCassation/src/sderApi/sderApiType.ts
@@ -1,6 +1,6 @@
 import { decisionType, publishStatusType } from 'sder';
 import { documentType } from '@label/core';
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 
 export type { sderApiType };
 

diff --git a/packages/generic/backend/src/index.ts b/packages/generic/backend/src/index.ts
@@ -1,10 +1,6 @@
 import { buildAnnotator, annotatorConfigType } from './lib/annotator';
 import { buildConnector, connectorConfigType } from './lib/connector';
-import {
-  buildExporter,
-  exporterConfigType,
-  labelTreatmentsType,
-} from './lib/exporter';
+import { buildExporter, exporterConfigType } from './lib/exporter';
 import { settingsLoader } from './lib/settingsLoader';
 import { buildMongo, dependencyManager, fileSystem, logger } from './utils';
 import { buildBackend } from './app';
@@ -25,9 +21,4 @@ export {
   treatmentService,
 };
 
-export type {
-  annotatorConfigType,
-  connectorConfigType,
-  exporterConfigType,
-  labelTreatmentsType,
-};
+export type { annotatorConfigType, connectorConfigType, exporterConfigType };
diff --git a/packages/generic/backend/src/lib/annotator/buildAnnotator.ts b/packages/generic/backend/src/lib/annotator/buildAnnotator.ts
@@ -8,15 +8,13 @@ import {
   settingsType,
   treatmentModule,
   annotationModule,
-  preAssignationType,
 } from '@label/core';
 import { buildAnnotationReportRepository } from '../../modules/annotationReport';
 import { documentService } from '../../modules/document';
 import { treatmentService } from '../../modules/treatment';
 import { logger } from '../../utils';
 import { annotatorConfigType } from './annotatorConfigType';
-import { preAssignationService } from '../../modules/preAssignation';
-import { assignationService } from '../../modules/assignation';
+import { buildPreAssignator } from '../preAssignator';
 
 export { buildAnnotator };
 
@@ -385,31 +383,10 @@ function buildAnnotator(
       route: document.route,
     });
 
-    // Check first pre-assignation by documentNumber and then by appelNumber
-    const preAssignationForDocument =
-      (await preAssignationService.fetchPreAssignationBySourceAndNumber(
-        document.documentNumber.toString(),
-        document.source,
-      )) ||
-      (await preAssignationService.fetchPreAssignationBySourceAndNumber(
-        document.decisionMetadata.appealNumber,
-        document.source,
-      ));
+    const preAssignator = buildPreAssignator();
+    const isPreassignated = await preAssignator.preAssignDocument(document);
 
-    if (
-      nextDocumentStatus === 'free' &&
-      preAssignationForDocument != undefined
-    ) {
-      logger.log({
-        operationName: 'annotateDocument',
-        msg: `Pre-assignation found for document ${formatDocumentInfos(
-          document,
-        )}. Matching pre-assignation number : ${
-          preAssignationForDocument.number
-        }. Creating assignation...`,
-      });
-      await createAssignation(preAssignationForDocument, document);
-    } else {
+    if (!isPreassignated) {
       await documentService.updateDocumentStatus(
         document._id,
         nextDocumentStatus,
@@ -428,21 +405,6 @@ function buildAnnotator(
     });
   }
 
-  async function createAssignation(
-    preAssignation: preAssignationType,
-    document: documentType,
-  ) {
-    await assignationService.createAssignation({
-      documentId: idModule.lib.buildId(document._id),
-      userId: idModule.lib.buildId(preAssignation.userId),
-    });
-    await preAssignationService.deletePreAssignation(preAssignation._id);
-    await documentService.updateDocumentStatus(
-      idModule.lib.buildId(document._id),
-      'saved',
-    );
-  }
-
   async function createAnnotatorTreatment({
     annotations,
     documentId,

diff --git a/packages/generic/backend/src/lib/connector/buildConnector.ts b/packages/generic/backend/src/lib/connector/buildConnector.ts
@@ -1,11 +1,21 @@
-import { dateBuilder, documentType, idModule, timeOperator } from '@label/core';
+import {
+  annotationModule,
+  annotationType,
+  dateBuilder,
+  documentType,
+  idModule,
+  settingsType,
+  timeOperator,
+} from '@label/core';
 import {
   buildDocumentRepository,
   documentService,
 } from '../../modules/document';
 import { logger } from '../../utils';
 import { connectorConfigType } from './connectorConfigType';
 import { decisionType } from 'sder';
+import { treatmentService } from '../../modules/treatment';
+import { buildPreAssignator } from '../preAssignator';
 
 export { buildConnector };
 
@@ -53,14 +63,18 @@ function buildConnector(connectorConfig: connectorConfigType) {
     documentNumber,
     source,
     lowPriority,
+    keepLabelTreatments,
+    settings,
   }: {
     documentNumber: number;
     source: string;
     lowPriority: boolean;
+    keepLabelTreatments: boolean;
+    settings: settingsType;
   }) {
     logger.log({
       operationName: 'importSpecificDocument',
-      msg: `START: ${documentNumber} - ${source}, lowPriority: ${lowPriority}`,
+      msg: `START: ${documentNumber} - ${source}, lowPriority: ${lowPriority}, keepLabelTreatments: ${keepLabelTreatments}`,
     });
 
     try {
@@ -94,6 +108,7 @@ function buildConnector(connectorConfig: connectorConfigType) {
         operationName: 'importSpecificDocument',
         msg: 'Court decision converted. Inserting document into database...',
       });
+
       if (lowPriority) {
         await insertDocument({ ...document });
       } else {
@@ -104,9 +119,64 @@ function buildConnector(connectorConfig: connectorConfigType) {
         msg: 'Insertion done',
       });
 
+      if (keepLabelTreatments) {
+        if (courtDecision.labelTreatments.length == 0) {
+          logger.error({
+            operationName: 'importSpecificDocument',
+            msg:
+              'LabelTreatments not found in court decision, skiping labelTreatments reimport.',
+          });
+        } else {
+          logger.log({
+            operationName: 'importSpecificDocument',
+            msg: 'LabelTreatments found in court decision, importing.',
+          });
+
+          const annotations: annotationType[] = courtDecision.labelTreatments[
+            courtDecision.labelTreatments.length - 1
+          ].annotations.map((annotation) => {
+            return annotationModule.lib.buildAnnotation({
+              category: annotation.category,
+              start: annotation.start,
+              text: annotation.text,
+              certaintyScore: 1,
+              entityId: annotation.entityId,
+            });
+          });
+
+          await treatmentService.createTreatment(
+            {
+              documentId: document._id,
+              previousAnnotations: [],
+              nextAnnotations: annotations,
+              source: 'reimportedTreatment',
+            },
+            settings,
+          );
+          logger.log({
+            operationName: 'importSpecificDocument',
+            msg: 'LabelTreatments reimported, checking for pre-assignation.',
+          });
+          const preAssignator = buildPreAssignator();
+          const isPreassignated = await preAssignator.preAssignDocument(
+            document,
+          );
+          if (!isPreassignated) {
+            logger.log({
+              operationName: 'importSpecificDocument',
+              msg: 'No preAssignation found, setting documentStatus to free.',
+            });
+            await documentService.updateDocumentStatus(
+              idModule.lib.buildId(document._id),
+              'free',
+            );
+          }
+        }
+      }
+
       logger.log({
         operationName: 'importSpecificDocument',
-        msg: 'Send document has been loaded...',
+        msg: 'Selected document has been inserted in label database.',
       });
       await connectorConfig.updateDocumentsLoadedStatus({
         documents: [document],
@@ -758,9 +828,11 @@ function buildConnector(connectorConfig: connectorConfigType) {
   async function resetDocument({
     documentNumber,
     source,
+    settings,
   }: {
     documentNumber: documentType['documentNumber'];
     source: documentType['source'];
+    settings: settingsType;
   }) {
     const documentRepository = buildDocumentRepository();
 
@@ -805,6 +877,8 @@ function buildConnector(connectorConfig: connectorConfigType) {
       documentNumber,
       source,
       lowPriority: true,
+      keepLabelTreatments: false,
+      settings,
     });
   }
 }

diff --git a/packages/generic/backend/src/lib/exporter/buildExporter.spec.ts b/packages/generic/backend/src/lib/exporter/buildExporter.spec.ts
@@ -3,11 +3,13 @@ import {
   documentModule,
   settingsModule,
   treatmentModule,
+  treatmentType,
 } from '@label/core';
 import { buildDocumentRepository } from '../../modules/document';
 import { buildTreatmentRepository } from '../../modules/treatment';
 import { buildExporter } from './buildExporter';
-import { exporterConfigType, labelTreatmentsType } from './exporterConfigType';
+import { exporterConfigType } from './exporterConfigType';
+import { labelTreatmentsType } from 'sder';
 
 /* eslint-disable @typescript-eslint/no-unused-vars */
 describe('buildExporter', () => {
@@ -50,6 +52,8 @@ describe('buildExporter', () => {
           },
           documentId: documents[0]._id,
           order: 0,
+          lastUpdateDate: 1720776507000,
+          source: 'NLP' as treatmentType['source'],
         },
         {
           annotationsDiff: {
@@ -65,7 +69,9 @@ describe('buildExporter', () => {
             ],
           },
           documentId: documents[2]._id,
-          order: 0,
+          order: 1,
+          lastUpdateDate: 1720776599000,
+          source: 'NLP' as treatmentType['source'],
         },
       ].map(treatmentModule.generator.generate);
       await Promise.all(documents.map(documentRepository.insert));
@@ -99,6 +105,8 @@ describe('buildExporter', () => {
               ],
               source: 'NLP',
               order: 1,
+              treatmentDate: '2024-07-12T09:28:27.000Z',
+              version: undefined,
             },
           ],
           [
@@ -114,6 +122,8 @@ describe('buildExporter', () => {
               ],
               source: 'NLP',
               order: 1,
+              treatmentDate: '2024-07-12T09:29:59.000Z',
+              version: undefined,
             },
           ],
         ].sort(),