Merge pull request #1744 from Inist-CNRS/feat/precomputed-process

feat: process precomputed data
Inist-CNRS · Oct 25, 2023 · 0a9275c · 0a9275c
2 parents 42bd90c + 49750ac
commit 0a9275c
Show file tree

Hide file tree

Showing 17 changed files with 583 additions and 323 deletions.
diff --git a/.gitignore b/.gitignore
@@ -44,7 +44,8 @@ config/staging.js
 config/test.js
 build
 typings
-/upload/
+/upload/*
+!/upload/.gitkeep
 
 # Cypress
 cypress/videos
@@ -55,3 +56,7 @@ stats.json
 
 ## Webstorm
 .idea
+
+## Folder for webservices gzip files exchanges
+/webservice_temp/*
+!/webservice_temp/.gitkeep
diff --git a/config.json b/config.json
@@ -82,19 +82,13 @@
         "doi-list",
         "tar-gz"
     ],
-    "exporters": [
-        "csv",
-        "tsv",
-        "jsonallvalue"
-    ],
+    "exporters": ["csv", "tsv", "jsonallvalue"],
     "datasetClass": "http://test.fr/datasetClass",
     "schemeForDatasetLink": "http://www.w3.org/2004/02/skos/core#inScheme",
     "exportDataset": "true",
     "collectionClass": "",
     "istexQuery": {
-        "labels": [
-            "Cette ressource dans ISTEX"
-        ],
+        "labels": ["Cette ressource dans ISTEX"],
         "linked": "categories.inist",
         "context": {
             "categories.inist": "istex:subjectInist",
@@ -195,5 +189,7 @@
             "search": 200
         },
         "multilingual": false
-    }
+    },
+    "webServiceBaseURL": "https://data-computer.services.istex.fr/v1/",
+    "ISOLATED_MODE": true
 }
diff --git a/src/api/controller/index.js b/src/api/controller/index.js
@@ -7,6 +7,7 @@ import api from './api';
 import front from './front';
 import embedded from './embedded';
 import customPage from './customPage';
+import webhook from './webhook';
 import rootAdmin from './rootAdmin';
 
 import repositoryMiddleware, {
@@ -72,6 +73,7 @@ app.use(async (ctx, next) => {
 
 app.use(repositoryMiddleware);
 
+app.use(mount('/webhook', webhook));
 app.use(mount('/embedded', embedded));
 app.use(mount('/api', api));
 

diff --git a/src/api/controller/webhook.js b/src/api/controller/webhook.js
@@ -0,0 +1,31 @@
+import Koa from 'koa';
+import route from 'koa-route';
+import bodyParser from 'koa-bodyparser';
+
+import getLogger from '../services/logger';
+import { getComputedFromWebservice } from '../services/precomputed/precomputed';
+
+export const getComputedWebserviceData = async ctx => {
+    const { precomputedId, tenant, jobId } = ctx.request.query;
+    const { identifier, generator, state } = ctx.request.body;
+    const logger = getLogger(ctx.tenant);
+    logger.info(`Precompute webhook call for ${tenant}`);
+    logger.info('Body', ctx.request.body);
+
+    if (!state == 'ready') {
+        return;
+    }
+
+    const callId = JSON.stringify([{ id: generator, value: identifier }]);
+
+    await getComputedFromWebservice(ctx, tenant, precomputedId, callId, jobId);
+
+    ctx.body = 'webhook ok';
+    ctx.status = 200;
+};
+
+const app = new Koa();
+app.use(bodyParser());
+app.use(route.post('/compute_webservice/', getComputedWebserviceData));
+
+export default app;
diff --git a/src/api/models/enrichment.js b/src/api/models/enrichment.js
@@ -1,12 +1,12 @@
-import { ObjectID } from 'mongodb';
+import { ObjectId } from 'mongodb';
 import omit from 'lodash.omit';
 import { castIdsFactory } from './utils';
 
 export default async db => {
     const collection = db.collection('enrichment');
 
     collection.findOneById = async id =>
-        collection.findOne({ $or: [{ _id: new ObjectID(id) }, { _id: id }] });
+        collection.findOne({ $or: [{ _id: new ObjectId(id) }, { _id: id }] });
 
     collection.findAll = async () => collection.find({}).toArray();
 
@@ -16,10 +16,10 @@ export default async db => {
     };
 
     collection.delete = async id =>
-        collection.remove({ $or: [{ _id: new ObjectID(id) }, { _id: id }] });
+        collection.remove({ $or: [{ _id: new ObjectId(id) }, { _id: id }] });
 
     collection.update = async (id, data) => {
-        const objectId = new ObjectID(id);
+        const objectId = new ObjectId(id);
 
         return collection
             .findOneAndUpdate(
@@ -36,6 +36,16 @@ export default async db => {
             .then(result => result.value);
     };
 
+    collection.updateStatus = async (id, status, data = {}) => {
+        const newData = { status, ...data };
+        collection.updateOne(
+            {
+                $or: [{ _id: new ObjectId(id) }, { _id: id }],
+            },
+            { $set: newData },
+        );
+    };
+
     collection.castIds = castIdsFactory(collection);
 
     return collection;

diff --git a/src/api/models/precomputed.js b/src/api/models/precomputed.js
@@ -1,4 +1,4 @@
-import { ObjectID } from 'mongodb';
+import { ObjectId } from 'mongodb';
 import omit from 'lodash.omit';
 import { castIdsFactory } from './utils';
 
@@ -12,7 +12,7 @@ export default async db => {
     const collection = db.collection('precomputed');
 
     collection.findOneById = async id =>
-        collection.findOne({ $or: [{ _id: new ObjectID(id) }, { _id: id }] });
+        collection.findOne({ $or: [{ _id: new ObjectId(id) }, { _id: id }] });
 
     collection.findAll = async () => collection.find({}).toArray();
 
@@ -25,13 +25,13 @@ export default async db => {
     };
 
     collection.delete = async id =>
-        collection.remove({ $or: [{ _id: new ObjectID(id) }, { _id: id }] });
+        collection.remove({ $or: [{ _id: new ObjectId(id) }, { _id: id }] });
 
     collection.update = async (id, data) => {
         if (checkMissingFields(data)) {
             throw new Error('Missing required fields');
         }
-        const objectId = new ObjectID(id);
+        const objectId = new ObjectId(id);
 
         return collection
             .findOneAndUpdate(
@@ -48,6 +48,16 @@ export default async db => {
             .then(result => result.value);
     };
 
+    collection.updateStatus = async (id, status, data = {}) => {
+        const newData = { status, ...data };
+        collection.updateOne(
+            {
+                $or: [{ _id: new ObjectId(id) }, { _id: id }],
+            },
+            { $set: newData },
+        );
+    };
+
     collection.castIds = castIdsFactory(collection);
 
     return collection;

diff --git a/src/api/services/enrichment/enrichment.js b/src/api/services/enrichment/enrichment.js
@@ -212,15 +212,7 @@ const processEzsEnrichment = (entries, commands, ctx, preview = false) => {
 };
 
 export const processEnrichment = async (enrichment, ctx) => {
-    await ctx.enrichment.updateOne(
-        {
-            $or: [
-                { _id: new ObjectId(enrichment._id) },
-                { _id: enrichment._id },
-            ],
-        },
-        { $set: { ['status']: IN_PROGRESS } },
-    );
+    await ctx.enrichment.updateStatus(enrichment._id, IN_PROGRESS);
     let errorCount = 0;
 
     const room = `${ctx.tenant}-enrichment-job-${ctx.job.id}`;
@@ -325,15 +317,7 @@ export const processEnrichment = async (enrichment, ctx) => {
             }
         }
     }
-    await ctx.enrichment.updateOne(
-        {
-            $or: [
-                { _id: new ObjectId(enrichment._id) },
-                { _id: enrichment._id },
-            ],
-        },
-        { $set: { ['status']: FINISHED, ['errorCount']: errorCount } },
-    );
+    await ctx.enrichment.updateStatus(enrichment._id, FINISHED, { errorCount });
     progress.finish(ctx.tenant);
     const logData = JSON.stringify({
         level: 'ok',
@@ -346,12 +330,9 @@ export const processEnrichment = async (enrichment, ctx) => {
 };
 
 export const setEnrichmentJobId = async (ctx, enrichmentID, job) => {
-    await ctx.enrichment.updateOne(
-        {
-            $or: [{ _id: new ObjectId(enrichmentID) }, { _id: enrichmentID }],
-        },
-        { $set: { ['jobId']: job.id, ['status']: ENRICHMENT_PENDING } },
-    );
+    await ctx.enrichment.updateStatus(enrichmentID, ENRICHMENT_PENDING, {
+        jobId: job.id,
+    });
 };
 
 export const startEnrichment = async ctx => {
@@ -382,15 +363,11 @@ export const startEnrichment = async ctx => {
 
 export const setEnrichmentError = async (ctx, err) => {
     const id = ctx.job?.data?.id;
-    await ctx.enrichment.updateOne(
-        {
-            $or: [{ _id: new ObjectId(id) }, { _id: id }],
-        },
+    await ctx.enrichment.updateStatus(
+        id,
+        err instanceof CancelWorkerError ? CANCELED : ERROR,
         {
-            $set: {
-                ['status']: err instanceof CancelWorkerError ? CANCELED : ERROR,
-                ['message']: err?.message,
-            },
+            message: err?.message,
         },
     );
 

diff --git a/src/api/services/enrichment/enrichment.spec.js b/src/api/services/enrichment/enrichment.spec.js
@@ -378,6 +378,7 @@ describe('enrichment', () => {
                 },
                 enrichment: {
                     updateOne: jest.fn(),
+                    updateStatus: jest.fn(),
                 },
                 dataset: {
                     updateOne: jest.fn(),