diff --git a/src/api/services/precomputed/precomputed.js b/src/api/services/precomputed/precomputed.js
index 2291e0f606..558f3a7c38 100644
--- a/src/api/services/precomputed/precomputed.js
+++ b/src/api/services/precomputed/precomputed.js
@@ -53,7 +53,11 @@ export const getPrecomputedDataPreview = async ctx => {
);
let result = [];
try {
- for (let index = 0; index < excerptLines.length; index += BATCH_SIZE) {
+ for (
+ let index = 0;
+ index < Math.min(excerptLines.length, BATCH_SIZE);
+ index += 1
+ ) {
const entry = {};
sourceColumns.map(column => {
entry[column] = excerptLines[index][column];
diff --git a/src/api/services/precomputed/precomputed.spec.js b/src/api/services/precomputed/precomputed.spec.js
index 094b38133b..2ce4dfc690 100644
--- a/src/api/services/precomputed/precomputed.spec.js
+++ b/src/api/services/precomputed/precomputed.spec.js
@@ -1,21 +1,13 @@
-import {
- getPrecomputedDataPreview,
- getSourceError,
- processPrecomputed,
-} from './precomputed';
-import * as fs from 'fs';
-import path from 'path';
-import { ObjectId } from 'mongodb';
-import progress from '../progress';
+import { getPrecomputedDataPreview } from './precomputed';
describe('precomputed', () => {
- describe('getEnrichmentDataPreview', () => {
- it('with direct path, single value', async () => {
+ describe('getPrecomputedDataPreview', () => {
+ it('single value', async () => {
// GIVEN
const ctx = {
request: {
body: {
- sourceColumn: 'simpleValue',
+ sourceColumns: ['simpleValue'],
},
},
dataset: {
@@ -32,97 +24,22 @@ describe('precomputed', () => {
// WHEN
const results = await getPrecomputedDataPreview(ctx);
- // THEN
- expect(results).toEqual(
- expect.arrayContaining(['plop', 'plip', 'ploup']),
- );
- });
-
- it('with direct path, single value and subpath', async () => {
- // GIVEN
- const ctx = {
- request: {
- body: {
- sourceColumn: 'objectValue',
- subPath: 'subPath',
- },
- },
- dataset: {
- getExcerpt: () => {
- return [
- { uri: '1', objectValue: { subPath: 'plop' } },
- { uri: '2', objectValue: { subPath: 'plip' } },
- { uri: '3', objectValue: { subPath: 'ploup' } },
- ];
- },
- },
- };
-
- // WHEN
- const results = await getPrecomputedDataPreview(ctx);
-
// THEN
expect(results).toEqual(
expect.arrayContaining([
- { subPath: 'plop' },
- { subPath: 'plip' },
- { subPath: 'ploup' },
+ { simpleValue: 'plop' },
+ { simpleValue: 'plip' },
+ { simpleValue: 'ploup' },
]),
);
});
- it('with direct path, single value stringified and subpath', async () => {
- // GIVEN
- const ctx = {
- request: {
- body: {
- sourceColumn: 'objectValue',
- subPath: 'subPath',
- },
- },
- dataset: {
- getExcerpt: () => {
- return [
- {
- uri: '1',
- objectValue: JSON.stringify({
- subPath: 'plop',
- }),
- },
- {
- uri: '2',
- objectValue: JSON.stringify({
- subPath: 'plip',
- }),
- },
- {
- uri: '3',
- objectValue: JSON.stringify({
- subPath: 'ploup',
- }),
- },
- ];
- },
- },
- };
-
- // WHEN
- const results = await getPrecomputedDataPreview(ctx);
- // THEN
- expect(results).toEqual(
- expect.arrayContaining([
- '{"subPath":"plop"}',
- '{"subPath":"plip"}',
- '{"subPath":"ploup"}',
- ]),
- );
- });
- it('with direct path, multiple values', async () => {
+ it('complex values', async () => {
// GIVEN
const ctx = {
request: {
body: {
- sourceColumn: 'arrayValue',
+ sourceColumns: ['arrayValue'],
},
},
dataset: {
@@ -141,68 +58,28 @@ describe('precomputed', () => {
// THEN
expect(results).toEqual(
- expect.arrayContaining([['plop', 'plup'], ['plip'], ['ploup']]),
- );
- });
-
- it('with direct path, multiple values stringified', async () => {
- // GIVEN
- const ctx = {
- request: {
- body: {
- sourceColumn: 'arrayValue',
- },
- },
- dataset: {
- getExcerpt: () => {
- return [
- {
- uri: '1',
- arrayValue: JSON.stringify(['plop', 'plup']),
- },
- { uri: '2', arrayValue: JSON.stringify(['plip']) },
- { uri: '3', arrayValue: JSON.stringify(['ploup']) },
- ];
- },
- },
- };
-
- // WHEN
- const results = await getPrecomputedDataPreview(ctx);
-
- // THEN
- expect(results).toEqual(
- expect.arrayContaining([['plop', 'plup'], ['plip'], ['ploup']]),
+ expect.arrayContaining([
+ { arrayValue: ['plop', 'plup'] },
+ { arrayValue: ['plip'] },
+ { arrayValue: ['ploup'] },
+ ]),
);
});
- it('with direct path, multiple values and subpath', async () => {
+ it('multiple values', async () => {
// GIVEN
const ctx = {
request: {
body: {
- sourceColumn: 'arrayValue',
- subPath: 'subPath',
+ sourceColumns: ['lol', 'simpleValue'],
},
},
dataset: {
getExcerpt: () => {
return [
- {
- uri: '1',
- arrayValue: [
- { subPath: 'plop' },
- { subPath: 'plup' },
- ],
- },
- {
- uri: '2',
- arrayValue: [{ subPath: 'plip' }],
- },
- {
- uri: '3',
- arrayValue: [{ subPath: 'ploup' }],
- },
+ { uri: '1', lol: 'a', simpleValue: 'plop' },
+ { uri: '2', lol: 'b', simpleValue: 'plip' },
+ { uri: '3', lol: 'c', simpleValue: 'ploup' },
];
},
},
@@ -214,243 +91,11 @@ describe('precomputed', () => {
// THEN
expect(results).toEqual(
expect.arrayContaining([
- [{ subPath: 'plop' }, { subPath: 'plup' }],
- [{ subPath: 'plop' }, { subPath: 'plup' }],
- [{ subPath: 'plip' }],
- [{ subPath: 'plip' }],
- [{ subPath: 'ploup' }],
- [{ subPath: 'ploup' }],
+ { lol: 'a', simpleValue: 'plop' },
+ { lol: 'b', simpleValue: 'plip' },
+ { lol: 'c', simpleValue: 'ploup' },
]),
);
});
- // We skip that test because it's a very specific case where we want to get a subpath in an array that is stringified, that may not happen. If the dataset import a string, then, it's a string.
- it.skip('with direct path, multiple values and subpath stringified', async () => {
- // GIVEN
- const ctx = {
- request: {
- body: {
- sourceColumn: 'arrayValue',
- subPath: 'subPath',
- },
- },
- dataset: {
- getExcerpt: () => {
- return [
- {
- uri: '1',
- arrayValue: JSON.stringify([
- { subPath: 'plop' },
- { subPath: 'plup' },
- ]),
- },
- {
- uri: '2',
- arrayValue: JSON.stringify([
- { subPath: 'plip' },
- ]),
- },
- {
- uri: '3',
- arrayValue: JSON.stringify([
- { subPath: 'ploup' },
- ]),
- },
- ];
- },
- },
- };
-
- // WHEN
- const results = await getPrecomputedDataPreview(ctx);
-
- // THEN
- expect(results).toEqual(
- expect.arrayContaining([['plop', 'plup'], ['plip'], ['ploup']]),
- );
- });
- });
-
- describe('processEnrichment', () => {
- it('should log error when ws is out', async () => {
- progress.initialize('lodex_test');
-
- // GIVEN
- const ezsRule = fs
- .readFileSync(
- path.resolve(__dirname, './directPathSingleValue.txt'),
- )
- .toString()
- .replace(/\[\[SOURCE COLUMN\]\]/g, 'name')
- .replace(
- '[[WEB SERVICE URL]]',
- 'http://a-fake-url.to.raise.an.error',
- )
- .replace('retries = 5', 'retries = 1');
- const enrichment = {
- rule: ezsRule,
- };
- const ctx = {
- tenant: 'lodex_test',
- job: {
- id: 1,
- log: jest.fn(),
- isActive: jest.fn().mockReturnValue(true),
- data: {
- tenant: 'lodex_test',
- },
- },
- enrichment: {
- updateOne: jest.fn(),
- },
- dataset: {
- updateOne: jest.fn(),
- count: jest.fn().mockReturnValue(3),
- find: jest.fn().mockReturnValue({
- skip: jest.fn().mockReturnValue({
- limit: jest.fn().mockReturnValue({
- toArray: jest.fn().mockReturnValue([
- {
- _id: new ObjectId(),
- uri: '1',
- name: 'plop',
- },
- {
- _id: new ObjectId(),
- uri: '2',
- name: 'plip',
- },
- {
- _id: new ObjectId(),
- uri: '3',
- name: 'ploup',
- },
- ]),
- }),
- }),
- }),
- },
- };
-
- // WHEN
- await processPrecomputed(enrichment, ctx);
-
- // THEN
- expect(ctx.job.log).toHaveBeenCalledTimes(7);
- expect(ctx.job.log).toHaveBeenNthCalledWith(
- 4,
- expect.stringContaining(
- `request to http://a-fake-url.to.raise.an.error/ failed, reason: getaddrinfo ENOTFOUND a-fake-url.to.raise.an.error`,
- ),
- );
- expect(ctx.job.log).toHaveBeenNthCalledWith(
- 5,
- expect.stringContaining(
- `request to http://a-fake-url.to.raise.an.error/ failed, reason: getaddrinfo ENOTFOUND a-fake-url.to.raise.an.error`,
- ),
- );
- expect(ctx.job.log).toHaveBeenNthCalledWith(
- 6,
- expect.stringContaining(
- `request to http://a-fake-url.to.raise.an.error/ failed, reason: getaddrinfo ENOTFOUND a-fake-url.to.raise.an.error`,
- ),
- );
- }, 60000);
- it.skip('should log error for 2nd line when ws errored for this line', async () => {
- // GIVEN
- const ezsRule = `
- [validate]
- path=value.valid
- rule=required
- [transit]
- `;
- const enrichment = {
- rule: ezsRule,
- };
- const ctx = {
- job: {
- id: 1,
- log: jest.fn(),
- isActive: jest.fn().mockReturnValue(true),
- },
- enrichment: {
- updateOne: jest.fn(),
- },
- dataset: {
- updateOne: jest.fn(),
- count: jest.fn().mockReturnValue(3),
- find: jest.fn().mockReturnValue({
- skip: jest.fn().mockReturnValue({
- limit: jest.fn().mockReturnValue({
- toArray: jest.fn().mockReturnValue([
- {
- _id: new ObjectId(),
- uri: '1',
- name: 'plop',
- valid: true,
- },
- {
- _id: new ObjectId(),
- uri: '2',
- name: 'plip',
- invalid: true,
- },
- {
- _id: new ObjectId(),
- uri: '3',
- name: 'ploup',
- valid: true,
- },
- ]),
- }),
- }),
- }),
- },
- };
-
- // WHEN
- await processPrecomputed(enrichment, ctx);
-
- // THEN
- expect(ctx.job.log).toHaveBeenCalledTimes(7);
- expect(ctx.job.log).toHaveBeenNthCalledWith(
- 4,
- expect.stringContaining(`Finished enriching #1`),
- );
- expect(ctx.job.log).toHaveBeenNthCalledWith(
- 5,
- expect.stringContaining(
- `Error enriching #2: [Error] { 'value.valid': [ 'The value.valid field is required.' ] }`,
- ),
- );
- expect(ctx.job.log).toHaveBeenNthCalledWith(
- 6,
- expect.stringContaining(`Finished enriching #3`),
- );
- });
- });
-});
-
-describe('getSourceError', () => {
- it('should return the deepest sourceError object', () => {
- // GIVEN
- const error = {
- sourceError: {
- sourceError: {
- sourceError: { message: 'Source error' },
- sourceChunk: 'This chunk',
- },
- sourceChunk: 'not that',
- },
- sourceChunk: 'not this',
- };
-
- // WHEN
- const deepestSourceError = getSourceError(error);
-
- // THEN
- expect(deepestSourceError.sourceError).toEqual({
- message: 'Source error',
- });
- expect(deepestSourceError.sourceChunk).toEqual('This chunk');
});
});
diff --git a/src/app/js/admin/precomputed/PrecomputedForm.spec.js b/src/app/js/admin/precomputed/PrecomputedForm.spec.js
index 4e085f7c31..d990f99569 100644
--- a/src/app/js/admin/precomputed/PrecomputedForm.spec.js
+++ b/src/app/js/admin/precomputed/PrecomputedForm.spec.js
@@ -10,6 +10,7 @@ describe('', () => {
const wrapper = shallow(
{} }}
+ handleSubmit={() => {}}
excerptColumns={[]}
excerptLines={EXCERPT_LINES}
/>,
@@ -21,6 +22,7 @@ describe('', () => {
const wrapper = shallow(
{} }}
+ handleSubmit={() => {}}
excerptColumns={[]}
excerptLines={EXCERPT_LINES}
/>,
@@ -30,32 +32,33 @@ describe('', () => {
expect(textField.prop('name')).toEqual('name');
});
- it('should render 2 Fields for precomputed rule', () => {
+ it('should render 2 Fields for precomputed info', () => {
const wrapper = shallow(
{} }}
+ handleSubmit={() => {}}
excerptColumns={[]}
excerptLines={EXCERPT_LINES}
/>,
);
- const webServiceUrl = wrapper.find(Field).at(2);
+ const webServiceUrl = wrapper.find(Field).at(1);
expect(webServiceUrl).toHaveLength(1);
expect(webServiceUrl.prop('name')).toEqual('webServiceUrl');
- const sourceColumns = wrapper.find(Field).at(3);
+ const sourceColumns = wrapper.find(Field).at(2);
expect(sourceColumns).toHaveLength(1);
expect(sourceColumns.prop('name')).toEqual('sourceColumns');
});
it('should render a precomputed logs dialog', () => {
const initialValues = {
- advancedMode: false,
_id: '123',
status: 'IN_PROGRESS',
};
const wrapper = shallow(
{} }}
+ handleSubmit={() => {}}
excerptColumns={[]}
initialValues={initialValues}
isEdit={true}
diff --git a/src/app/js/admin/withInitialData.spec.js b/src/app/js/admin/withInitialData.spec.js
index d31a35abd8..524a788e5c 100644
--- a/src/app/js/admin/withInitialData.spec.js
+++ b/src/app/js/admin/withInitialData.spec.js
@@ -14,6 +14,7 @@ describe('withInitialData HOC', () => {
loadPublication: jest.fn(),
loadSubresources: jest.fn(),
loadEnrichments: jest.fn(),
+ loadPrecomputed: jest.fn(),
p: { t: () => {} },
isLoading: false,
};
@@ -25,6 +26,7 @@ describe('withInitialData HOC', () => {
expect(defaultProps.loadPublication).toHaveBeenCalled();
expect(defaultProps.loadSubresources).toHaveBeenCalled();
expect(defaultProps.loadEnrichments).toHaveBeenCalled();
+ expect(defaultProps.loadPrecomputed).toHaveBeenCalled();
});
it('should render AdminComponent when isLoading is false', () => {