Feat: build precomputed data screen in admin

Makefile

@@ -156,6 +156,7 @@ clear-database: ## Clear the whole database
 		db.publishedFacet.remove({}); \
 		db.subresource.remove({}); \
 		db.enrichment.remove({}); \
+		db.precomputed.remove({}); \
 	"
 clear-publication: ## Clear the published data, keep uploaded dataset and model
 	docker compose exec mongo mongo lodex --eval " \

packages/transformers/.babelrc

@@ -1,11 +1,10 @@
 {
-	"presets": [
-		"@babel/preset-env"
-	],
-	"plugins": [
-		"@babel/plugin-syntax-dynamic-import",
-		"@babel/plugin-syntax-import-meta",
-		"@babel/plugin-proposal-class-properties",
-		"@babel/plugin-proposal-json-strings"
-	]
+    "presets": ["@babel/preset-env"],
+    "plugins": [
+        "@babel/plugin-syntax-dynamic-import",
+        "@babel/plugin-syntax-import-meta",
+        "@babel/plugin-proposal-class-properties",
+        "@babel/plugin-transform-nullish-coalescing-operator",
+        "@babel/plugin-proposal-json-strings"
+    ]
 }

src/api/controller/api/index.js

@@ -27,6 +27,7 @@ import loader from './loader';
 import translate from './translate';
 import subresource from './subresource';
 import enrichment from './enrichment';
+import precomputed from './precomputed';
 import job from './job';
 import dump from './dump';
 import displayConfig from './displayConfig';
@@ -116,6 +117,7 @@ app.use(mount('/characteristic', characteristic));
 app.use(mount('/field', fieldRoutes));
 app.use(mount('/subresource', subresource));
 app.use(mount('/enrichment', enrichment));
+app.use(mount('/precomputed', precomputed));
 app.use(mount('/job', job));
 app.use(mount('/parsing', parsing));
 app.use(mount('/publish', publish));

src/api/controller/api/precomputed.js

@@ -0,0 +1,153 @@
+import Koa from 'koa';
+import route from 'koa-route';
+import koaBodyParser from 'koa-bodyparser';
+import { v1 as uuid } from 'uuid';
+
+import { PRECOMPUTER } from '../../workers/precomputer';
+import { workerQueues } from '../../workers';
+import {
+    getPrecomputedDataPreview,
+    setPrecomputedJobId,
+} from '../../services/precomputed/precomputed';
+import { cancelJob, getActiveJob } from '../../workers/tools';
+
+export const setup = async (ctx, next) => {
+    try {
+        await next();
+    } catch (error) {
+        ctx.status = 500;
+        ctx.body = { error: error.message };
+    }
+};
+
+export const postPrecomputed = async ctx => {
+    try {
+        const precomputed = ctx.request.body;
+        const result = await ctx.precomputed.create(precomputed);
+
+        if (result) {
+            ctx.body = result;
+            return;
+        }
+
+        ctx.status = 403;
+    } catch (error) {
+        ctx.status = 403;
+        ctx.body = { error: error.message };
+        return;
+    }
+};
+
+export const putPrecomputed = async (ctx, id) => {
+    const newPrecomputed = ctx.request.body;
+
+    try {
+        ctx.body = await ctx.precomputed.update(id, newPrecomputed);
+    } catch (error) {
+        ctx.status = 403;
+        ctx.body = { error: error.message };
+        return;
+    }
+};
+
+export const deletePrecomputed = async (ctx, id) => {
+    try {
+        const activeJob = await getActiveJob(ctx.tenant);
+        if (
+            activeJob?.data?.jobType === PRECOMPUTER &&
+            activeJob?.data?.id === id
+        ) {
+            cancelJob(ctx, PRECOMPUTER);
+        }
+        await ctx.precomputed.delete(id);
+        ctx.status = 200;
+        ctx.body = { message: 'ok' };
+    } catch (error) {
+        ctx.status = 403;
+        ctx.body = { error: error.message };
+        return;
+    }
+};
+
+export const getPrecomputed = async (ctx, id) => {
+    ctx.body = await ctx.precomputed.findOneById(id);
+};
+
+export const getAllPrecomputed = async ctx => {
+    ctx.body = await ctx.precomputed.findAll();
+};
+
+export const precomputedAction = async (ctx, action, id) => {
+    if (!['launch', 'pause', 'relaunch'].includes(action)) {
+        throw new Error(`Invalid action "${action}"`);
+    }
+
+    if (action === 'launch') {
+        await workerQueues[ctx.tenant]
+            .add(
+                PRECOMPUTER, // Name of the job
+                {
+                    id,
+                    jobType: PRECOMPUTER,
+                    tenant: ctx.tenant,
+                },
+                { jobId: uuid() },
+            )
+            .then(job => {
+                setPrecomputedJobId(ctx, id, job);
+            });
+        ctx.body = {
+            status: 'pending',
+        };
+    }
+
+    if (action === 'relaunch') {
+        const precomputed = await ctx.precomputed.findOneById(id);
+        await ctx.dataset.removeAttribute(precomputed.name);
+        await workerQueues[ctx.tenant]
+            .add(
+                PRECOMPUTER, // Name of the job
+                {
+                    id,
+                    jobType: PRECOMPUTER,
+                    tenant: ctx.tenant,
+                },
+                { jobId: uuid() },
+            )
+            .then(job => {
+                setPrecomputedJobId(ctx, id, job);
+            });
+        ctx.body = {
+            status: 'pending',
+        };
+    }
+
+    ctx.status = 200;
+};
+
+export const precomputedDataPreview = async ctx => {
+    try {
+        const result = await getPrecomputedDataPreview(ctx);
+        ctx.status = 200;
+        ctx.body = result;
+    } catch (error) {
+        ctx.status = 403;
+        ctx.body = { error: error.message };
+        return;
+    }
+};
+
+const app = new Koa();
+
+app.use(setup);
+
+app.use(route.get('/', getAllPrecomputed));
+app.use(route.get('/:id', getPrecomputed));
+app.use(koaBodyParser());
+app.use(route.post('/', postPrecomputed));
+app.use(route.put('/:id', putPrecomputed));
+app.use(route.delete('/:id', deletePrecomputed));
+app.use(route.post('/:action/:id', precomputedAction));
+app.use(route.post('/preview', precomputedDataPreview));
+
+export default app;

src/api/controller/api/precomputed.spec.js

@@ -0,0 +1,116 @@
+import {
+    postPrecomputed,
+    putPrecomputed,
+    deletePrecomputed,
+} from './precomputed';
+
+jest.mock('../../workers/tools', () => ({
+    getActiveJob: jest.fn(),
+    cancelJob: jest.fn(),
+}));
+
+describe('Precomputed controller', () => {
+    describe('postPrecomputed', () => {
+        it('should call precomputed create repository method', async () => {
+            const ctx = {
+                request: { body: { name: 'test' } },
+                precomputed: { create: jest.fn() },
+            };
+
+            await postPrecomputed(ctx);
+
+            expect(ctx.precomputed.create).toHaveBeenCalledWith({
+                name: 'test',
+            });
+        });
+
+        it('should return result as body result', async () => {
+            const ctx = {
+                request: { body: { name: 'test' } },
+                precomputed: {
+                    create: jest.fn(() => {
+                        return { name: 'test' };
+                    }),
+                },
+            };
+
+            await postPrecomputed(ctx);
+
+            expect(ctx.body).toStrictEqual({
+                name: 'test',
+            });
+        });
+
+        it("should set status 403 if there's not result", async () => {
+            const ctx = {
+                request: { body: 'my precomputed' },
+                precomputed: {
+                    create: jest.fn(() => null),
+                },
+            };
+
+            await postPrecomputed(ctx);
+
+            expect(ctx.status).toBe(403);
+        });
+    });
+
+    describe('putPrecomputed', () => {
+        it('should delete existing dataset data based on the precomputed name and update it', async () => {
+            const ctx = {
+                request: { body: 'my updated precomputed' },
+                precomputed: {
+                    findOneById: jest.fn(() =>
+                        Promise.resolve({ name: 'NAME' }),
+                    ),
+                    update: jest.fn(() =>
+                        Promise.resolve('updated precomputed'),
+                    ),
+                },
+                dataset: { removeAttribute: jest.fn() },
+            };
+
+            await putPrecomputed(ctx, 42);
+
+            expect(ctx.precomputed.update).toHaveBeenCalledWith(
+                42,
+                'my updated precomputed',
+            );
+            expect(ctx.body).toEqual('updated precomputed');
+            return;
+        });
+
+        it('should return a 403 on error if an error occured', async () => {
+            const ctx = {
+                request: { body: 'my updated precomputed' },
+                precomputed: {
+                    update: async () => {
+                        throw new Error('ERROR!');
+                    },
+                },
+            };
+
+            await putPrecomputed(ctx, 42);
+
+            expect(ctx.status).toBe(403);
+            expect(ctx.body).toEqual({ error: 'ERROR!' });
+        });
+    });
+
+    describe('deletePrecomputed', () => {
+        it('should return a 403 on error if an error occured', async () => {
+            const ctx = {
+                precomputed: {
+                    delete: async () => {
+                        throw new Error('ERROR!');
+                    },
+                },
+            };
+
+            await deletePrecomputed(ctx, 42);
+
+            expect(ctx.status).toBe(403);
+            expect(ctx.body).toEqual({ error: 'ERROR!' });
+        });
+    });
+});

src/api/models/precomputed.js

@@ -0,0 +1,52 @@
+import { ObjectID } from 'mongodb';
+import omit from 'lodash.omit';
+import { castIdsFactory } from './utils';
+
+const checkMissingFields = data =>
+    !data.name ||
+    !data.webServiceUrl ||
+    !data.sourceColumns ||
+    (data.sourceColumns instanceof Array && data.sourceColumns.length === 0);
+
+export default async db => {
+    const collection = db.collection('precomputed');
+
+    collection.findOneById = async id =>
+        collection.findOne({ $or: [{ _id: new ObjectID(id) }, { _id: id }] });
+
+    collection.findAll = async () => collection.find({}).toArray();
+
+    collection.create = async data => {
+        if (checkMissingFields(data)) {
+            throw new Error('Missing required fields');
+        }
+        const { insertedId } = await collection.insertOne(data);
+        return collection.findOne({ _id: insertedId });
+    };
+
+    collection.delete = async id =>
+        collection.remove({ $or: [{ _id: new ObjectID(id) }, { _id: id }] });
+
+    collection.update = async (id, data) => {
+        if (checkMissingFields(data)) {
+            throw new Error('Missing required fields');
+        }
+        const objectId = new ObjectID(id);
+
+        return collection
+            .findOneAndUpdate(
+                {
+                    $or: [{ _id: objectId }, { _id: id }],
+                },
+                omit(data, ['_id']),
+                {
+                    returnOriginal: false,
+                },
+            )
+            .then(result => result.value);
+    };
+
+    collection.castIds = castIdsFactory(collection);
+
+    return collection;
+};

src/api/services/enrichment/enrichment.js

@@ -12,7 +12,7 @@ import {
     FINISHED,
     ERROR,
     CANCELED,
-} from '../../../common/enrichmentStatus';
+} from '../../../common/taskStatus';
 import { ENRICHING, PENDING } from '../../../common/progressStatus';
 import { jobLogger } from '../../workers/tools';
 import { CancelWorkerError } from '../../workers';