From 3ab5bd6f50f0a076fa400fa46035c4cfde8bda6c Mon Sep 17 00:00:00 2001 From: Antoine Jeanneney <29945628+ajeanneney@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:29:29 +0200 Subject: [PATCH] Keep label treatments when reimport (#112) * prepare for reimport labeltreatments * wip * working labelTreatments reimportation * update concat and sder version * deploy on preprod - to revert * Oh, enable build * fix branch name for ci * change branch name * refacto preAssignation * small fix on preAssignation refacto * update sder * update sder again * fix concat * update sder for validation schema * remove reimported treatment when its no longer neccesary, not before * really delete reimportedTreatments * update concat order calculation * delete custom labelTreatmentsType * add labelTreatment date * fix gitlabci * docker login onoly for build * fine tune logs * fix tests * fix lint * docker login only for build, not for deploy * clean gitlabci and update sder version * treatment date from timestamp to iso string --------- Co-authored-by: Antoine Jeanneney --- .gitlab-ci.yml | 2 + .../src/annotator/fetcher/api/nlpApiType.ts | 2 +- .../src/annotator/fetcher/nlpFetcher.ts | 2 +- .../mapper/mapCourtDecisionToDocument.spec.ts | 2 +- .../scripts/importSpecificDocumentFromSder.ts | 15 ++- .../src/scripts/resetDocument.ts | 2 +- .../courDeCassation/src/sderApi/sderApi.ts | 1 + .../src/sderApi/sderApiType.ts | 2 +- packages/generic/backend/src/index.ts | 13 +-- .../src/lib/annotator/buildAnnotator.ts | 46 +-------- .../src/lib/connector/buildConnector.ts | 80 +++++++++++++++- .../src/lib/exporter/buildExporter.spec.ts | 14 ++- .../backend/src/lib/exporter/buildExporter.ts | 7 +- .../src/lib/exporter/exporterConfigType.ts | 16 +--- .../generic/backend/src/lib/exporter/index.ts | 4 +- .../lib/preAssignator/buildPreAssignator.ts | 77 +++++++++++++++ .../backend/src/lib/preAssignator/index.ts | 3 + .../lib/preAssignator/preAssignator.spec.ts | 48 ++++++++++ .../buildFakePreAssignationRepository.ts | 15 ++- .../buildPreAssignationRepository.ts | 6 +- .../customPreAssignationRepositoryType.ts | 3 +- packages/generic/core/package.json | 2 +- .../lib/assertAnnotationsDiffAreConsistent.ts | 3 + ...assertTreatmentsSourcesFollowRightOrder.ts | 2 +- .../src/modules/treatment/lib/concat.spec.ts | 93 ++++++++++++++++++- .../core/src/modules/treatment/lib/concat.ts | 34 ++++--- .../src/modules/treatment/treatmentType.ts | 2 +- yarn.lock | 4 +- 28 files changed, 391 insertions(+), 109 deletions(-) create mode 100644 packages/generic/backend/src/lib/preAssignator/buildPreAssignator.ts create mode 100644 packages/generic/backend/src/lib/preAssignator/index.ts create mode 100644 packages/generic/backend/src/lib/preAssignator/preAssignator.spec.ts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b93e79528..f989b05be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,6 +43,7 @@ build_label_backend: HTTP_PROXY: $HTTP_PROXY_DEV HTTPS_PROXY: $HTTPS_PROXY_DEV script: + - docker login -u $PUBLIC_DOCKER_USERNAME -p $PUBLIC_DOCKER_PASSWORD - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - docker build --build-arg http_proxy=$HTTP_PROXY @@ -64,6 +65,7 @@ build_label_client: HTTP_PROXY: $HTTP_PROXY_DEV HTTPS_PROXY: $HTTPS_PROXY_DEV script: + - docker login -u $PUBLIC_DOCKER_USERNAME -p $PUBLIC_DOCKER_PASSWORD - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - export NODE_OPTIONS=--openssl-legacy-provider - docker build diff --git a/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts b/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts index 03ee1191b..fa65d1f17 100644 --- a/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts +++ b/packages/courDeCassation/src/annotator/fetcher/api/nlpApiType.ts @@ -1,4 +1,4 @@ -import { labelTreatmentsType } from '@label/backend'; +import { labelTreatmentsType } from 'sder'; import { documentType, settingsType } from '@label/core'; export type { nlpApiType, nlpResponseType, nlpLossType, nlpVersion }; diff --git a/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts b/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts index 0b7214103..623246039 100644 --- a/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts +++ b/packages/courDeCassation/src/annotator/fetcher/nlpFetcher.ts @@ -1,4 +1,4 @@ -import { labelTreatmentsType } from '@label/backend'; +import { labelTreatmentsType } from 'sder'; import { documentType, settingsType } from '@label/core'; import { buildNlpApi } from './api'; import { nlpMapper } from './mapper'; diff --git a/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts b/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts index 5d8721f4b..274720328 100644 --- a/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts +++ b/packages/courDeCassation/src/connector/mapper/mapCourtDecisionToDocument.spec.ts @@ -1,4 +1,4 @@ -import { labelTreatmentsType } from '@label/backend'; +import { labelTreatmentsType } from 'sder'; import { idModule } from '@label/core'; import { mapCourtDecisionToDocument } from './mapCourtDecisionToDocument'; diff --git a/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts b/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts index 7101cc169..70ffe5b47 100644 --- a/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts +++ b/packages/courDeCassation/src/scripts/importSpecificDocumentFromSder.ts @@ -5,7 +5,12 @@ import { parametersHandler } from '../lib/parametersHandler'; (async () => { const { settings } = await parametersHandler.getParameters(); - const { documentNumber, source, lowPriority } = parseArgv(); + const { + documentNumber, + source, + lowPriority, + keepLabelTreatments, + } = parseArgv(); const backend = buildBackend(settings); backend.runScript( @@ -14,6 +19,8 @@ import { parametersHandler } from '../lib/parametersHandler'; documentNumber, source, lowPriority, + keepLabelTreatments, + settings, }), { shouldLoadDb: true, @@ -40,6 +47,11 @@ function parseArgv() { 'source (jurinet, jurica or juritj) of the document you want to import', type: 'string', }, + keepLabelTreatments: { + demandOption: false, + description: 'import labelTreatments from SDER database if exist', + type: 'boolean', + }, }) .help() .alias('help', 'h').argv; @@ -48,5 +60,6 @@ function parseArgv() { documentNumber: argv.documentNumber as number, lowPriority: !!argv.lowPriority as boolean, source: argv.source as string, + keepLabelTreatments: !!argv.keepLabelTreatments as boolean, }; } diff --git a/packages/courDeCassation/src/scripts/resetDocument.ts b/packages/courDeCassation/src/scripts/resetDocument.ts index fa39747ed..e0e7e51e4 100644 --- a/packages/courDeCassation/src/scripts/resetDocument.ts +++ b/packages/courDeCassation/src/scripts/resetDocument.ts @@ -9,7 +9,7 @@ import { sderConnector } from '../connector'; const backend = buildBackend(settings); backend.runScript( - () => sderConnector.resetDocument({ documentNumber, source }), + () => sderConnector.resetDocument({ documentNumber, source, settings }), { shouldLoadDb: true, }, diff --git a/packages/courDeCassation/src/sderApi/sderApi.ts b/packages/courDeCassation/src/sderApi/sderApi.ts index 6d1e694b8..ee799fd74 100644 --- a/packages/courDeCassation/src/sderApi/sderApi.ts +++ b/packages/courDeCassation/src/sderApi/sderApi.ts @@ -256,6 +256,7 @@ const sderApi: sderApiType = { publishStatus, }) { //TODO : include publishStatus to dbsder api call + //TODO : manage labelTreatments before sending to dbsder-api (like in sder updateDecisionPseudonymisation function) if (process.env.DBSDER_API_ENABLED === 'true') { await fetchApi({ method: 'put', diff --git a/packages/courDeCassation/src/sderApi/sderApiType.ts b/packages/courDeCassation/src/sderApi/sderApiType.ts index 1ef88ac75..78ff3a6ee 100644 --- a/packages/courDeCassation/src/sderApi/sderApiType.ts +++ b/packages/courDeCassation/src/sderApi/sderApiType.ts @@ -1,6 +1,6 @@ import { decisionType, publishStatusType } from 'sder'; import { documentType } from '@label/core'; -import { labelTreatmentsType } from '@label/backend'; +import { labelTreatmentsType } from 'sder'; export type { sderApiType }; diff --git a/packages/generic/backend/src/index.ts b/packages/generic/backend/src/index.ts index 9f6713df7..47db6af9b 100644 --- a/packages/generic/backend/src/index.ts +++ b/packages/generic/backend/src/index.ts @@ -1,10 +1,6 @@ import { buildAnnotator, annotatorConfigType } from './lib/annotator'; import { buildConnector, connectorConfigType } from './lib/connector'; -import { - buildExporter, - exporterConfigType, - labelTreatmentsType, -} from './lib/exporter'; +import { buildExporter, exporterConfigType } from './lib/exporter'; import { settingsLoader } from './lib/settingsLoader'; import { buildMongo, dependencyManager, fileSystem, logger } from './utils'; import { buildBackend } from './app'; @@ -25,9 +21,4 @@ export { treatmentService, }; -export type { - annotatorConfigType, - connectorConfigType, - exporterConfigType, - labelTreatmentsType, -}; +export type { annotatorConfigType, connectorConfigType, exporterConfigType }; diff --git a/packages/generic/backend/src/lib/annotator/buildAnnotator.ts b/packages/generic/backend/src/lib/annotator/buildAnnotator.ts index 6463e169a..508083273 100644 --- a/packages/generic/backend/src/lib/annotator/buildAnnotator.ts +++ b/packages/generic/backend/src/lib/annotator/buildAnnotator.ts @@ -8,15 +8,13 @@ import { settingsType, treatmentModule, annotationModule, - preAssignationType, } from '@label/core'; import { buildAnnotationReportRepository } from '../../modules/annotationReport'; import { documentService } from '../../modules/document'; import { treatmentService } from '../../modules/treatment'; import { logger } from '../../utils'; import { annotatorConfigType } from './annotatorConfigType'; -import { preAssignationService } from '../../modules/preAssignation'; -import { assignationService } from '../../modules/assignation'; +import { buildPreAssignator } from '../preAssignator'; export { buildAnnotator }; @@ -385,31 +383,10 @@ function buildAnnotator( route: document.route, }); - // Check first pre-assignation by documentNumber and then by appelNumber - const preAssignationForDocument = - (await preAssignationService.fetchPreAssignationBySourceAndNumber( - document.documentNumber.toString(), - document.source, - )) || - (await preAssignationService.fetchPreAssignationBySourceAndNumber( - document.decisionMetadata.appealNumber, - document.source, - )); + const preAssignator = buildPreAssignator(); + const isPreassignated = await preAssignator.preAssignDocument(document); - if ( - nextDocumentStatus === 'free' && - preAssignationForDocument != undefined - ) { - logger.log({ - operationName: 'annotateDocument', - msg: `Pre-assignation found for document ${formatDocumentInfos( - document, - )}. Matching pre-assignation number : ${ - preAssignationForDocument.number - }. Creating assignation...`, - }); - await createAssignation(preAssignationForDocument, document); - } else { + if (!isPreassignated) { await documentService.updateDocumentStatus( document._id, nextDocumentStatus, @@ -428,21 +405,6 @@ function buildAnnotator( }); } - async function createAssignation( - preAssignation: preAssignationType, - document: documentType, - ) { - await assignationService.createAssignation({ - documentId: idModule.lib.buildId(document._id), - userId: idModule.lib.buildId(preAssignation.userId), - }); - await preAssignationService.deletePreAssignation(preAssignation._id); - await documentService.updateDocumentStatus( - idModule.lib.buildId(document._id), - 'saved', - ); - } - async function createAnnotatorTreatment({ annotations, documentId, diff --git a/packages/generic/backend/src/lib/connector/buildConnector.ts b/packages/generic/backend/src/lib/connector/buildConnector.ts index 723cf47f0..592f95539 100644 --- a/packages/generic/backend/src/lib/connector/buildConnector.ts +++ b/packages/generic/backend/src/lib/connector/buildConnector.ts @@ -1,4 +1,12 @@ -import { dateBuilder, documentType, idModule, timeOperator } from '@label/core'; +import { + annotationModule, + annotationType, + dateBuilder, + documentType, + idModule, + settingsType, + timeOperator, +} from '@label/core'; import { buildDocumentRepository, documentService, @@ -6,6 +14,8 @@ import { import { logger } from '../../utils'; import { connectorConfigType } from './connectorConfigType'; import { decisionType } from 'sder'; +import { treatmentService } from '../../modules/treatment'; +import { buildPreAssignator } from '../preAssignator'; export { buildConnector }; @@ -53,14 +63,18 @@ function buildConnector(connectorConfig: connectorConfigType) { documentNumber, source, lowPriority, + keepLabelTreatments, + settings, }: { documentNumber: number; source: string; lowPriority: boolean; + keepLabelTreatments: boolean; + settings: settingsType; }) { logger.log({ operationName: 'importSpecificDocument', - msg: `START: ${documentNumber} - ${source}, lowPriority: ${lowPriority}`, + msg: `START: ${documentNumber} - ${source}, lowPriority: ${lowPriority}, keepLabelTreatments: ${keepLabelTreatments}`, }); try { @@ -94,6 +108,7 @@ function buildConnector(connectorConfig: connectorConfigType) { operationName: 'importSpecificDocument', msg: 'Court decision converted. Inserting document into database...', }); + if (lowPriority) { await insertDocument({ ...document }); } else { @@ -104,9 +119,64 @@ function buildConnector(connectorConfig: connectorConfigType) { msg: 'Insertion done', }); + if (keepLabelTreatments) { + if (courtDecision.labelTreatments.length == 0) { + logger.error({ + operationName: 'importSpecificDocument', + msg: + 'LabelTreatments not found in court decision, skiping labelTreatments reimport.', + }); + } else { + logger.log({ + operationName: 'importSpecificDocument', + msg: 'LabelTreatments found in court decision, importing.', + }); + + const annotations: annotationType[] = courtDecision.labelTreatments[ + courtDecision.labelTreatments.length - 1 + ].annotations.map((annotation) => { + return annotationModule.lib.buildAnnotation({ + category: annotation.category, + start: annotation.start, + text: annotation.text, + certaintyScore: 1, + entityId: annotation.entityId, + }); + }); + + await treatmentService.createTreatment( + { + documentId: document._id, + previousAnnotations: [], + nextAnnotations: annotations, + source: 'reimportedTreatment', + }, + settings, + ); + logger.log({ + operationName: 'importSpecificDocument', + msg: 'LabelTreatments reimported, checking for pre-assignation.', + }); + const preAssignator = buildPreAssignator(); + const isPreassignated = await preAssignator.preAssignDocument( + document, + ); + if (!isPreassignated) { + logger.log({ + operationName: 'importSpecificDocument', + msg: 'No preAssignation found, setting documentStatus to free.', + }); + await documentService.updateDocumentStatus( + idModule.lib.buildId(document._id), + 'free', + ); + } + } + } + logger.log({ operationName: 'importSpecificDocument', - msg: 'Send document has been loaded...', + msg: 'Selected document has been inserted in label database.', }); await connectorConfig.updateDocumentsLoadedStatus({ documents: [document], @@ -758,9 +828,11 @@ function buildConnector(connectorConfig: connectorConfigType) { async function resetDocument({ documentNumber, source, + settings, }: { documentNumber: documentType['documentNumber']; source: documentType['source']; + settings: settingsType; }) { const documentRepository = buildDocumentRepository(); @@ -805,6 +877,8 @@ function buildConnector(connectorConfig: connectorConfigType) { documentNumber, source, lowPriority: true, + keepLabelTreatments: false, + settings, }); } } diff --git a/packages/generic/backend/src/lib/exporter/buildExporter.spec.ts b/packages/generic/backend/src/lib/exporter/buildExporter.spec.ts index f64d142db..8f3803c8e 100644 --- a/packages/generic/backend/src/lib/exporter/buildExporter.spec.ts +++ b/packages/generic/backend/src/lib/exporter/buildExporter.spec.ts @@ -3,11 +3,13 @@ import { documentModule, settingsModule, treatmentModule, + treatmentType, } from '@label/core'; import { buildDocumentRepository } from '../../modules/document'; import { buildTreatmentRepository } from '../../modules/treatment'; import { buildExporter } from './buildExporter'; -import { exporterConfigType, labelTreatmentsType } from './exporterConfigType'; +import { exporterConfigType } from './exporterConfigType'; +import { labelTreatmentsType } from 'sder'; /* eslint-disable @typescript-eslint/no-unused-vars */ describe('buildExporter', () => { @@ -50,6 +52,8 @@ describe('buildExporter', () => { }, documentId: documents[0]._id, order: 0, + lastUpdateDate: 1720776507000, + source: 'NLP' as treatmentType['source'], }, { annotationsDiff: { @@ -65,7 +69,9 @@ describe('buildExporter', () => { ], }, documentId: documents[2]._id, - order: 0, + order: 1, + lastUpdateDate: 1720776599000, + source: 'NLP' as treatmentType['source'], }, ].map(treatmentModule.generator.generate); await Promise.all(documents.map(documentRepository.insert)); @@ -99,6 +105,8 @@ describe('buildExporter', () => { ], source: 'NLP', order: 1, + treatmentDate: '2024-07-12T09:28:27.000Z', + version: undefined, }, ], [ @@ -114,6 +122,8 @@ describe('buildExporter', () => { ], source: 'NLP', order: 1, + treatmentDate: '2024-07-12T09:29:59.000Z', + version: undefined, }, ], ].sort(), diff --git a/packages/generic/backend/src/lib/exporter/buildExporter.ts b/packages/generic/backend/src/lib/exporter/buildExporter.ts index 1df10bf22..b7a8c3394 100644 --- a/packages/generic/backend/src/lib/exporter/buildExporter.ts +++ b/packages/generic/backend/src/lib/exporter/buildExporter.ts @@ -192,12 +192,13 @@ function buildExporter( const anonymizer = buildAnonymizer(settingsForDocument, annotations, seed); try { - const version = document.nlpVersions; - // check treatments in concat lib source if nlp set nlpVersions in labelTreatments await exporterConfig.sendDocumentPseudonymisationAndTreatments({ externalId: document.externalId, pseudonymizationText: anonymizer.anonymizeDocument(document).text, - labelTreatments: treatmentModule.lib.concat(treatments, version), + labelTreatments: treatmentModule.lib.concat( + treatments, + document.nlpVersions, + ), }); logger.log({ operationName: 'exportDocument', diff --git a/packages/generic/backend/src/lib/exporter/exporterConfigType.ts b/packages/generic/backend/src/lib/exporter/exporterConfigType.ts index 98e7862a4..6b6641e6a 100644 --- a/packages/generic/backend/src/lib/exporter/exporterConfigType.ts +++ b/packages/generic/backend/src/lib/exporter/exporterConfigType.ts @@ -1,7 +1,7 @@ import { documentType } from '@label/core'; -import { publishStatusType } from 'sder'; +import { publishStatusType, labelTreatmentsType } from 'sder'; -export type { exporterConfigType, labelTreatmentsType }; +export type { exporterConfigType }; type exporterConfigType = { name: string; @@ -15,15 +15,3 @@ type exporterConfigType = { externalId: documentType['externalId']; }) => Promise; }; - -type labelTreatmentsType = Array<{ - annotations: Array<{ - category: string; - entityId: string; - start: number; - text: string; - }>; - version?: documentType['nlpVersions']; - source: string; - order: number; -}>; diff --git a/packages/generic/backend/src/lib/exporter/index.ts b/packages/generic/backend/src/lib/exporter/index.ts index 2c1adc66f..dd4f8addf 100644 --- a/packages/generic/backend/src/lib/exporter/index.ts +++ b/packages/generic/backend/src/lib/exporter/index.ts @@ -1,6 +1,6 @@ import { buildExporter } from './buildExporter'; -import { exporterConfigType, labelTreatmentsType } from './exporterConfigType'; +import { exporterConfigType } from './exporterConfigType'; export { buildExporter }; -export type { exporterConfigType, labelTreatmentsType }; +export type { exporterConfigType }; diff --git a/packages/generic/backend/src/lib/preAssignator/buildPreAssignator.ts b/packages/generic/backend/src/lib/preAssignator/buildPreAssignator.ts new file mode 100644 index 000000000..fcdb80195 --- /dev/null +++ b/packages/generic/backend/src/lib/preAssignator/buildPreAssignator.ts @@ -0,0 +1,77 @@ +import { documentType, idModule } from '@label/core'; +import { logger } from '../../utils'; +import { preAssignationService } from '../../modules/preAssignation'; +import { assignationService } from '../../modules/assignation'; +import { documentService } from '../../modules/document'; + +export { buildPreAssignator }; + +function buildPreAssignator() { + return { + preAssignDocument, + }; + + async function preAssignDocument(document: documentType): Promise { + logger.log({ + operationName: 'preAssignation', + msg: `Starting preAssignation for document ${document.source} ${document.documentNumber}`, + }); + + if (document.status === 'nlpAnnotating' || document.status === 'loaded') { + const preAssignationForDocument = + (await preAssignationService.fetchPreAssignationBySourceAndNumber( + document.documentNumber.toString(), + document.source, + )) || + (await preAssignationService.fetchPreAssignationBySourceAndNumber( + document.decisionMetadata.appealNumber, + document.source, + )); + + if (preAssignationForDocument != undefined) { + logger.log({ + operationName: 'preAssignation', + msg: `Pre-assignation found for document ${document.source} ${document.documentNumber}. Matching pre-assignation number : ${preAssignationForDocument.number}. Creating assignation...`, + }); + await assignationService.createAssignation({ + documentId: idModule.lib.buildId(document._id), + userId: idModule.lib.buildId(preAssignationForDocument.userId), + }); + await preAssignationService.deletePreAssignation( + preAssignationForDocument._id, + ); + if (document.route === 'automatic' || document.route === 'simple') { + await documentService.updateDocumentRoute( + idModule.lib.buildId(document._id), + 'exhaustive', + ); + } + await documentService.updateDocumentStatus( + idModule.lib.buildId(document._id), + 'saved', + ); + + logger.log({ + operationName: 'preAssignation', + msg: `Pre-assignation DONE`, + }); + + return true; + } else { + logger.log({ + operationName: 'preAssignation', + msg: `Pre-assignation not found for document ${document.source} ${document.documentNumber}`, + }); + return false; + } + } else { + logger.error({ + operationName: 'preAssignation', + msg: `Document status must be loaded or nlpAnnotating before pre-assign it`, + }); + throw new Error( + 'Document status must be loaded or nlpAnnotating before pre-assign it', + ); + } + } +} diff --git a/packages/generic/backend/src/lib/preAssignator/index.ts b/packages/generic/backend/src/lib/preAssignator/index.ts new file mode 100644 index 000000000..c3baa9f65 --- /dev/null +++ b/packages/generic/backend/src/lib/preAssignator/index.ts @@ -0,0 +1,3 @@ +import { buildPreAssignator } from './buildPreAssignator'; + +export { buildPreAssignator }; diff --git a/packages/generic/backend/src/lib/preAssignator/preAssignator.spec.ts b/packages/generic/backend/src/lib/preAssignator/preAssignator.spec.ts new file mode 100644 index 000000000..8842c5320 --- /dev/null +++ b/packages/generic/backend/src/lib/preAssignator/preAssignator.spec.ts @@ -0,0 +1,48 @@ +import { documentModule, preAssignationModule, userModule } from '@label/core'; +import { buildPreAssignator } from './buildPreAssignator'; +import { buildUserRepository } from '../../modules/user'; +import { buildPreAssignationRepository } from '../../modules/preAssignation'; +import { buildDocumentRepository } from '../../modules/document'; + +describe('buildPreAssignator', () => { + const preAssignator = buildPreAssignator(); + + it('must throw an error if document is not free', () => { + const documentNotFree = documentModule.generator.generate({ + status: 'done', + }); + expect( + preAssignator.preAssignDocument(documentNotFree), + ).rejects.toThrowError( + 'Document status must be loaded or nlpAnnotating before pre-assign it', + ); + }); + + it('must find preAssignation and assign document', async () => { + const userRepository = buildUserRepository(); + const preAssignationRepository = buildPreAssignationRepository(); + const documentRepository = buildDocumentRepository(); + + const user = userModule.generator.generate(); + await userRepository.insert(user); + const documentNumber = 123456; + const source = 'juritest'; + const documentToPreAssign = documentModule.generator.generate({ + status: 'loaded', + source: source, + documentNumber: documentNumber, + }); + await documentRepository.insert(documentToPreAssign); + + const preAssignation = preAssignationModule.generator.generate({ + number: documentNumber.toString(), + userId: user._id, + source: source, + }); + await preAssignationRepository.insert(preAssignation); + + expect(await preAssignator.preAssignDocument(documentToPreAssign)).toEqual( + true, + ); + }); +}); diff --git a/packages/generic/backend/src/modules/preAssignation/repository/buildFakePreAssignationRepository.ts b/packages/generic/backend/src/modules/preAssignation/repository/buildFakePreAssignationRepository.ts index c1caf1f94..a6d25a1b9 100644 --- a/packages/generic/backend/src/modules/preAssignation/repository/buildFakePreAssignationRepository.ts +++ b/packages/generic/backend/src/modules/preAssignation/repository/buildFakePreAssignationRepository.ts @@ -1,5 +1,8 @@ -import { preAssignationType } from '@label/core'; -import { buildFakeRepositoryBuilder } from '../../../repository'; +import { idModule, idType, preAssignationType } from '@label/core'; +import { + buildFakeRepositoryBuilder, + updateFakeCollection, +} from '../../../repository'; import { customPreAssignationRepositoryType } from './customPreAssignationRepositoryType'; export { buildFakePreAssignationRepository }; @@ -16,5 +19,13 @@ const buildFakePreAssignationRepository = buildFakeRepositoryBuilder< preAssignation.source === source && preAssignation.number === number, ); }, + async deleteById(id: idType) { + updateFakeCollection( + collection, + collection.filter( + (preAssignation) => !idModule.lib.equalId(preAssignation._id, id), + ), + ); + }, }), }); diff --git a/packages/generic/backend/src/modules/preAssignation/repository/buildPreAssignationRepository.ts b/packages/generic/backend/src/modules/preAssignation/repository/buildPreAssignationRepository.ts index 7cbb87ddb..908cc2158 100644 --- a/packages/generic/backend/src/modules/preAssignation/repository/buildPreAssignationRepository.ts +++ b/packages/generic/backend/src/modules/preAssignation/repository/buildPreAssignationRepository.ts @@ -1,4 +1,4 @@ -import { preAssignationType } from '@label/core'; +import { idType, preAssignationType } from '@label/core'; import { buildRepositoryBuilder } from '../../../repository'; import { customPreAssignationRepositoryType } from './customPreAssignationRepositoryType'; @@ -23,5 +23,9 @@ const buildPreAssignationRepository = buildRepositoryBuilder< }); return preAssignation || undefined; }, + + async deleteById(id: idType) { + await collection.deleteOne({ _id: id }); + }, }), }); diff --git a/packages/generic/backend/src/modules/preAssignation/repository/customPreAssignationRepositoryType.ts b/packages/generic/backend/src/modules/preAssignation/repository/customPreAssignationRepositoryType.ts index 87aebab61..bed2fc9d4 100644 --- a/packages/generic/backend/src/modules/preAssignation/repository/customPreAssignationRepositoryType.ts +++ b/packages/generic/backend/src/modules/preAssignation/repository/customPreAssignationRepositoryType.ts @@ -1,4 +1,4 @@ -import { preAssignationType } from '@label/core'; +import { idType, preAssignationType } from '@label/core'; export type { customPreAssignationRepositoryType }; @@ -10,4 +10,5 @@ type customPreAssignationRepositoryType = { number: string; source: string; }) => Promise; + deleteById: (id: idType) => Promise; }; diff --git a/packages/generic/core/package.json b/packages/generic/core/package.json index 93f8c8aa7..a68a5fc24 100644 --- a/packages/generic/core/package.json +++ b/packages/generic/core/package.json @@ -36,7 +36,7 @@ "bcryptjs": "^2.4.3", "lodash": "^4.17.21", "mongodb": "^3.6.1", - "sder": "https://github.com/Cour-de-cassation/sder#68f786a46e2e8d2aa45dbf26641e98567c152730", + "sder": "https://github.com/Cour-de-cassation/sder#93c0d7b11610205d80a0927f8965062eae40aa25", "string-template": "^1.0.0", "typescript": "~4.0.0" }, diff --git a/packages/generic/core/src/modules/annotationsDiff/lib/assertAnnotationsDiffAreConsistent.ts b/packages/generic/core/src/modules/annotationsDiff/lib/assertAnnotationsDiffAreConsistent.ts index 414907341..2b81b4570 100644 --- a/packages/generic/core/src/modules/annotationsDiff/lib/assertAnnotationsDiffAreConsistent.ts +++ b/packages/generic/core/src/modules/annotationsDiff/lib/assertAnnotationsDiffAreConsistent.ts @@ -46,6 +46,9 @@ function computeAvailableCategoriesFilter(treatmentSource: treatmentType['source case 'supplementaryAnnotations': canBeAnnotatedBy = 'human'; break; + case 'reimportedTreatment': + canBeAnnotatedBy = 'NLP'; + break; } return { status, canBeAnnotatedBy }; } diff --git a/packages/generic/core/src/modules/treatment/lib/assertTreatmentsSourcesFollowRightOrder.ts b/packages/generic/core/src/modules/treatment/lib/assertTreatmentsSourcesFollowRightOrder.ts index fda1944d6..563bdeb47 100644 --- a/packages/generic/core/src/modules/treatment/lib/assertTreatmentsSourcesFollowRightOrder.ts +++ b/packages/generic/core/src/modules/treatment/lib/assertTreatmentsSourcesFollowRightOrder.ts @@ -6,7 +6,7 @@ export { assertTreatmentsSourcesFollowRightOrder }; type documentStepType = { treatmentSource: treatmentType['source'][]; quantity: '0|1' | '1' | '1+' }; const DOCUMENT_STEPS: documentStepType[] = [ - { treatmentSource: ['NLP'], quantity: '1' }, + { treatmentSource: ['NLP', 'reimportedTreatment'], quantity: '1' }, { treatmentSource: ['supplementaryAnnotations'], quantity: '0|1' }, { treatmentSource: ['postProcess'], quantity: '0|1' }, { treatmentSource: ['admin', 'annotator'], quantity: '1+' }, diff --git a/packages/generic/core/src/modules/treatment/lib/concat.spec.ts b/packages/generic/core/src/modules/treatment/lib/concat.spec.ts index 45c30c67e..17b0c0cd5 100644 --- a/packages/generic/core/src/modules/treatment/lib/concat.spec.ts +++ b/packages/generic/core/src/modules/treatment/lib/concat.spec.ts @@ -2,6 +2,24 @@ import { documentModule, treatmentModule, treatmentType } from '../../'; import { concat } from './concat'; describe('concat', () => { + const nlpVersion = { + juriSpacyTokenizer: { + version: `VERSION_${Math.random()}`, + date: `DATE_${Math.random()}`, + }, + juritools: { + version: `VERSION_${Math.random()}`, + date: `DATE_${Math.random()}`, + }, + pseudonymisationApi: { + version: `VERSION_${Math.random()}`, + date: `DATE_${Math.random()}`, + }, + model: { + name: `MODEL_${Math.random()}`, + }, + }; + it('should return a labelTreatment', () => { const document = documentModule.generator.generate(); const treatments: treatmentType[] = [ @@ -10,17 +28,82 @@ describe('concat', () => { documentId: document._id, order: 2, source: 'annotator' as treatmentType['source'], + lastUpdateDate: 1720776507700, }, - { documentId: document._id, order: 0, source: 'NLP' as treatmentType['source'] }, - { documentId: document._id, order: 1, source: 'postProcess' as treatmentType['source'] }, + { documentId: document._id, order: 0, source: 'NLP' as treatmentType['source'], lastUpdateDate: 1720773507000 }, + { + documentId: document._id, + order: 1, + source: 'postProcess' as treatmentType['source'], + lastUpdateDate: 1720776567000, + }, + { documentId: document._id, order: 3, source: 'admin' as treatmentType['source'], lastUpdateDate: 1720776507123 }, + ].map(treatmentModule.generator.generate); + + const labelTreatments = concat(treatments, nlpVersion); + + expect(labelTreatments).toEqual([ + { annotations: [], order: 1, source: 'NLP', treatmentDate: '2024-07-12T08:38:27.000Z', version: nlpVersion }, + { + annotations: [], + order: 2, + source: 'LABEL_AUTO_TREATMENT', + treatmentDate: '2024-07-12T09:29:27.000Z', + version: undefined, + }, + { + annotations: [], + order: 3, + source: 'LABEL_WORKING_USER_TREATMENT', + treatmentDate: '2024-07-12T09:28:27.700Z', + version: undefined, + }, + { + annotations: [], + order: 4, + source: 'LABEL_WORKING_USER_TREATMENT', + treatmentDate: '2024-07-12T09:28:27.123Z', + version: undefined, + }, + ]); + }); + + it('should return a labelTreatment when there is a reimported treatment', () => { + const document = documentModule.generator.generate(); + const treatments: treatmentType[] = [ + { + subAnnotationsSensitiveCount: 5, + documentId: document._id, + order: 1, + source: 'annotator' as treatmentType['source'], + lastUpdateDate: 1720776507000, + }, + { + documentId: document._id, + order: 0, + source: 'reimportedTreatment' as treatmentType['source'], + lastUpdateDate: 1720776597300, + }, + { documentId: document._id, order: 2, source: 'admin' as treatmentType['source'], lastUpdateDate: 1720776507000 }, ].map(treatmentModule.generator.generate); const labelTreatments = concat(treatments); expect(labelTreatments).toEqual([ - { annotations: [], order: 1, source: 'NLP' }, - { annotations: [], order: 2, source: 'LABEL_AUTO_TREATMENT' }, - { annotations: [], order: 3, source: 'LABEL_WORKING_USER_TREATMENT' }, + { + annotations: [], + order: 1, + source: 'LABEL_WORKING_USER_TREATMENT', + version: undefined, + treatmentDate: '2024-07-12T09:28:27.000Z', + }, + { + annotations: [], + order: 2, + source: 'LABEL_WORKING_USER_TREATMENT', + version: undefined, + treatmentDate: '2024-07-12T09:28:27.000Z', + }, ]); }); }); diff --git a/packages/generic/core/src/modules/treatment/lib/concat.ts b/packages/generic/core/src/modules/treatment/lib/concat.ts index c7ec71a3f..7d455332f 100644 --- a/packages/generic/core/src/modules/treatment/lib/concat.ts +++ b/packages/generic/core/src/modules/treatment/lib/concat.ts @@ -12,26 +12,36 @@ function concat(treatments: treatmentType[], nlpVersions?: documentType['nlpVers while (sortedTreatments.length > 0) { const order = sortedTreatments.length; + const currentTreatment = sortedTreatments[order - 1]; - labelTreatments.unshift({ - annotations: computeAnnotations(sortedTreatments), - source: computeSource(order), - order, - version: computeSource(order) == 'NLP' ? nlpVersions : undefined, - }); + if (currentTreatment.source != 'reimportedTreatment') { + labelTreatments.unshift({ + annotations: computeAnnotations(sortedTreatments), + source: computeSource(currentTreatment.source), + order, + version: currentTreatment.source == 'NLP' ? nlpVersions : undefined, + treatmentDate: new Date(currentTreatment.lastUpdateDate).toISOString(), + }); + } sortedTreatments.pop(); } + // re-write order in case of reimportedTreatment + labelTreatments.forEach((labelTreatment, index) => { + labelTreatment.order = index + 1; + }); + return labelTreatments; - function computeSource(order: number) { - switch (order) { - case 1: + function computeSource(source: treatmentType['source']) { + switch (source) { + case 'NLP': return 'NLP'; - case 2: - return 'LABEL_AUTO_TREATMENT'; - default: + case 'annotator': + case 'admin': return 'LABEL_WORKING_USER_TREATMENT'; + default: + return 'LABEL_AUTO_TREATMENT'; } } } diff --git a/packages/generic/core/src/modules/treatment/treatmentType.ts b/packages/generic/core/src/modules/treatment/treatmentType.ts index 180772725..413259591 100644 --- a/packages/generic/core/src/modules/treatment/treatmentType.ts +++ b/packages/generic/core/src/modules/treatment/treatmentType.ts @@ -29,7 +29,7 @@ const treatmentModel = buildModel({ }, source: { kind: 'constant', - content: ['annotator', 'admin', 'NLP', 'postProcess', 'supplementaryAnnotations'] as const, + content: ['annotator', 'admin', 'NLP', 'postProcess', 'supplementaryAnnotations', 'reimportedTreatment'] as const, }, }, } as const); diff --git a/yarn.lock b/yarn.lock index e57f57d87..235434884 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15032,9 +15032,9 @@ schema-utils@^2.5.0, schema-utils@^2.6.0, schema-utils@^2.6.1, schema-utils@^2.6 jsonwebtoken "^8.5.1" lodash "^4.17.21" -"sder@https://github.com/Cour-de-cassation/sder#5333e7ac962fe80850cddbd3cd83b73fbb4fd656": +"sder@https://github.com/Cour-de-cassation/sder#93c0d7b11610205d80a0927f8965062eae40aa25": version "1.0.0" - resolved "https://github.com/Cour-de-cassation/sder#5333e7ac962fe80850cddbd3cd83b73fbb4fd656" + resolved "https://github.com/Cour-de-cassation/sder#93c0d7b11610205d80a0927f8965062eae40aa25" dependencies: bcryptjs "^2.4.3" body-parser "^1.19.0"