Merge branch 'dev' into feat/rework-assignation-ordering

Cour-de-cassation · Jul 22, 2024 · 476ea90 · 476ea90
2 parents fdddcc7 + 6861f16
commit 476ea90
Show file tree

Hide file tree

Showing 33 changed files with 415 additions and 130 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -43,6 +43,7 @@ build_label_backend:
     HTTP_PROXY: $HTTP_PROXY_DEV
     HTTPS_PROXY: $HTTPS_PROXY_DEV
   script:
+    - docker login -u $PUBLIC_DOCKER_USERNAME -p $PUBLIC_DOCKER_PASSWORD
     - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - docker build
       --build-arg http_proxy=$HTTP_PROXY
@@ -64,6 +65,7 @@ build_label_client:
     HTTP_PROXY: $HTTP_PROXY_DEV
     HTTPS_PROXY: $HTTPS_PROXY_DEV
   script:
+    - docker login -u $PUBLIC_DOCKER_USERNAME -p $PUBLIC_DOCKER_PASSWORD
     - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - export NODE_OPTIONS=--openssl-legacy-provider
     - docker build

diff --git a/ansible/roles/deploy_backend/defaults/main/defaults.yml b/ansible/roles/deploy_backend/defaults/main/defaults.yml
@@ -8,15 +8,15 @@ jobs:
     parallelism: 1
     active_deadline_seconds: 3600
     command: "dist/scripts/annotateDocumentsWithoutAnnotationsWithNlp.js"
-  - name: "import-j-4-dateDecision"
+  - name: "import-j-4-date-decision"
     schedule: "15,45 6-12 * * *"
     successful_jobs_history_limit: 7
     failed_jobs_history_limit: 7
     backoff_limit: 0
     parallelism: 1
     active_deadline_seconds: 300
     command: "dist/scripts/importAllDocumentsFromSderSinceOrBetween.js --fromDaysAgo 4"
-  - name: "import-j-4-dateCreation"
+  - name: "import-j-4-date-creation"
     schedule: "0,30 6-12 * * *"
     successful_jobs_history_limit: 7
     failed_jobs_history_limit: 7

diff --git a/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts b/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts
@@ -1,4 +1,4 @@
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 import { documentType, settingsType } from '@label/core';
 
 export type { nlpApiType, nlpResponseType, nlpLossType, nlpVersion };

diff --git a/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts b/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts
@@ -1,4 +1,4 @@
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 import { documentType, settingsType } from '@label/core';
 import { buildNlpApi } from './api';
 import { nlpMapper } from './mapper';

diff --git a/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts b/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts
@@ -1,4 +1,4 @@
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 import { idModule } from '@label/core';
 import { mapCourtDecisionToDocument } from './mapCourtDecisionToDocument';
 

diff --git a/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts b/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts
@@ -5,7 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler';
 
 (async () => {
   const { settings } = await parametersHandler.getParameters();
-  const { documentNumber, source, lowPriority } = parseArgv();
+  const {
+    documentNumber,
+    source,
+    lowPriority,
+    keepLabelTreatments,
+  } = parseArgv();
   const backend = buildBackend(settings);
 
   backend.runScript(
@@ -14,6 +19,8 @@ import { parametersHandler } from '../lib/parametersHandler';
         documentNumber,
         source,
         lowPriority,
+        keepLabelTreatments,
+        settings,
       }),
     {
       shouldLoadDb: true,
@@ -40,6 +47,11 @@ function parseArgv() {
           'source (jurinet, jurica or juritj) of the document you want to import',
         type: 'string',
       },
+      keepLabelTreatments: {
+        demandOption: false,
+        description: 'import labelTreatments from SDER database if exist',
+        type: 'boolean',
+      },
     })
     .help()
     .alias('help', 'h').argv;
@@ -48,5 +60,6 @@ function parseArgv() {
     documentNumber: argv.documentNumber as number,
     lowPriority: !!argv.lowPriority as boolean,
     source: argv.source as string,
+    keepLabelTreatments: !!argv.keepLabelTreatments as boolean,
   };
 }
diff --git a/packages/courDeCassation/src/scripts/resetDocument.ts b/packages/courDeCassation/src/scripts/resetDocument.ts
@@ -9,7 +9,7 @@ import { sderConnector } from '../connector';
   const backend = buildBackend(settings);
 
   backend.runScript(
-    () => sderConnector.resetDocument({ documentNumber, source }),
+    () => sderConnector.resetDocument({ documentNumber, source, settings }),
     {
       shouldLoadDb: true,
     },

diff --git a/packages/courDeCassation/src/sderApi/sderApi.ts b/packages/courDeCassation/src/sderApi/sderApi.ts
@@ -256,6 +256,7 @@ const sderApi: sderApiType = {
     publishStatus,
   }) {
     //TODO : include publishStatus to dbsder api call
+    //TODO : manage labelTreatments before sending to dbsder-api (like in sder updateDecisionPseudonymisation function)
     if (process.env.DBSDER_API_ENABLED === 'true') {
       await fetchApi({
         method: 'put',

diff --git a/packages/courDeCassation/src/sderApi/sderApiType.ts b/packages/courDeCassation/src/sderApi/sderApiType.ts
@@ -1,6 +1,6 @@
 import { decisionType, publishStatusType } from 'sder';
 import { documentType } from '@label/core';
-import { labelTreatmentsType } from '@label/backend';
+import { labelTreatmentsType } from 'sder';
 
 export type { sderApiType };
 

diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanFreeDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanFreeDocuments.ts
@@ -16,6 +16,11 @@ async function cleanFreeDocuments() {
     ['free'],
     ['_id'],
   );
+  logger.log({
+    operationName: 'countFreeDocuments',
+    msg: `${freeDocuments.length} free documents found.`,
+    data: { freeDocumentsCount: freeDocuments.length },
+  });
   const freeDocumentIds = freeDocuments.map(({ _id }) => _id);
   logger.log({
     operationName: 'cleanFreeDocuments',

diff --git a/packages/generic/backend/src/index.ts b/packages/generic/backend/src/index.ts
@@ -1,10 +1,6 @@
 import { buildAnnotator, annotatorConfigType } from './lib/annotator';
 import { buildConnector, connectorConfigType } from './lib/connector';
-import {
-  buildExporter,
-  exporterConfigType,
-  labelTreatmentsType,
-} from './lib/exporter';
+import { buildExporter, exporterConfigType } from './lib/exporter';
 import { settingsLoader } from './lib/settingsLoader';
 import { buildMongo, dependencyManager, fileSystem, logger } from './utils';
 import { buildBackend } from './app';
@@ -25,9 +21,4 @@ export {
   treatmentService,
 };
 
-export type {
-  annotatorConfigType,
-  connectorConfigType,
-  exporterConfigType,
-  labelTreatmentsType,
-};
+export type { annotatorConfigType, connectorConfigType, exporterConfigType };
diff --git a/packages/generic/backend/src/lib/annotator/buildAnnotator.spec.ts b/packages/generic/backend/src/lib/annotator/buildAnnotator.spec.ts
@@ -30,22 +30,6 @@ describe('buildAnnotator', () => {
       expect(documentWithoutAnnotations).toEqual(undefined);
     });
   });
-
-  describe('reAnnotateFreeDocuments', () => {
-    it('should re annotate all the documents', async () => {
-      await insertNDocumentsWithoutAnnotationsInDb(5);
-      const fakeAnnotator = buildFakeAnnotatorConfig();
-      const annotator = buildAnnotator(settings, fakeAnnotator);
-      await annotator.annotateDocumentsWithoutAnnotations();
-
-      await annotator.reAnnotateFreeDocuments();
-
-      const documentWithoutAnnotations = await documentService.fetchDocumentWithoutAnnotationsNotIn(
-        [],
-      );
-      expect(documentWithoutAnnotations).toEqual(undefined);
-    });
-  });
 });
 
 async function insertNDocumentsWithoutAnnotationsInDb(n: number) {

diff --git a/packages/generic/backend/src/lib/annotator/buildAnnotator.ts b/packages/generic/backend/src/lib/annotator/buildAnnotator.ts
@@ -8,15 +8,13 @@ import {
   settingsType,
   treatmentModule,
   annotationModule,
-  preAssignationType,
 } from '@label/core';
 import { buildAnnotationReportRepository } from '../../modules/annotationReport';
 import { documentService } from '../../modules/document';
 import { treatmentService } from '../../modules/treatment';
 import { logger } from '../../utils';
 import { annotatorConfigType } from './annotatorConfigType';
-import { preAssignationService } from '../../modules/preAssignation';
-import { assignationService } from '../../modules/assignation';
+import { buildPreAssignator } from '../preAssignator';
 
 export { buildAnnotator };
 
@@ -179,7 +177,8 @@ function buildAnnotator(
       await documentService.updateDocumentStatus(documentId, 'loaded');
     }
 
-    await annotateDocumentsWithoutAnnotations();
+    // Uncomment this line to run the annotation script
+    // await annotateDocumentsWithoutAnnotations();
   }
 
   async function annotateDocument(document: documentType) {
@@ -384,31 +383,10 @@ function buildAnnotator(
       route: document.route,
     });
 
-    // Check first pre-assignation by documentNumber and then by appelNumber
-    const preAssignationForDocument =
-      (await preAssignationService.fetchPreAssignationBySourceAndNumber(
-        document.documentNumber.toString(),
-        document.source,
-      )) ||
-      (await preAssignationService.fetchPreAssignationBySourceAndNumber(
-        document.decisionMetadata.appealNumber,
-        document.source,
-      ));
+    const preAssignator = buildPreAssignator();
+    const isPreassignated = await preAssignator.preAssignDocument(document);
 
-    if (
-      nextDocumentStatus === 'free' &&
-      preAssignationForDocument != undefined
-    ) {
-      logger.log({
-        operationName: 'annotateDocument',
-        msg: `Pre-assignation found for document ${formatDocumentInfos(
-          document,
-        )}. Matching pre-assignation number : ${
-          preAssignationForDocument.number
-        }. Creating assignation...`,
-      });
-      await createAssignation(preAssignationForDocument, document);
-    } else {
+    if (!isPreassignated) {
       await documentService.updateDocumentStatus(
         document._id,
         nextDocumentStatus,
@@ -427,21 +405,6 @@ function buildAnnotator(
     });
   }
 
-  async function createAssignation(
-    preAssignation: preAssignationType,
-    document: documentType,
-  ) {
-    await assignationService.createAssignation({
-      documentId: idModule.lib.buildId(document._id),
-      userId: idModule.lib.buildId(preAssignation.userId),
-    });
-    await preAssignationService.deletePreAssignation(preAssignation._id);
-    await documentService.updateDocumentStatus(
-      idModule.lib.buildId(document._id),
-      'saved',
-    );
-  }
-
   async function createAnnotatorTreatment({
     annotations,
     documentId,