diff --git a/src/reports/family-clinical-data/generateFamilySqon.ts b/src/reports/family-clinical-data/generateFamilySqon.ts deleted file mode 100644 index 70bda9b..0000000 --- a/src/reports/family-clinical-data/generateFamilySqon.ts +++ /dev/null @@ -1,76 +0,0 @@ -import get from 'lodash/get'; -import { buildQuery } from '@arranger/middleware'; - -import { getExtendedConfigs, getNestedFields } from '../../utils/arrangerUtils'; -import { executeSearch } from '../../utils/esUtils'; -import { Client } from '@elastic/elasticsearch'; -import { resolveSetsInSqon } from '../../utils/sqonUtils'; -import { ProjectType } from '../types'; -import { Sqon } from '../../utils/setsTypes'; -import { ES_QUERY_MAX_SIZE } from '../../env'; - -/** - * Generate a sqon from the family_id of all the participants in the given `sqon`. - * @param {object} es - an `elasticsearch.Client` instance. - * @param {string} projectId - the id of the arranger project. - * @param {object} sqon - the sqon used to filter the results. - * @param {object} normalizedConfigs - the normalized report configuration. - * @param {string} userId - the user id. - * @param {string} accessToken - the user access token. - * @param {string} program - the program the report will run on. - * @returns {object} - A sqon of all the `family_id`. - */ -const generateFamilySqon = async ( - es: Client, - projectId: string, - sqon: Sqon, - normalizedConfigs, - userId: string, - accessToken: string, - program: string, - isKfNext: boolean, -): Promise => { - const extendedConfig = await getExtendedConfigs(es, projectId, normalizedConfigs.indexName); - const nestedFields = getNestedFields(extendedConfig); - const newSqon = await resolveSetsInSqon(sqon, userId, accessToken); - const query = buildQuery({ nestedFields, filters: newSqon }); - - const participantIds = - (sqon.content || []).filter(e => (e.content?.field || '') === 'participant_id')[0]?.content.value || []; - - const field = program.toLowerCase() === ProjectType.include || isKfNext ? 'families_id' : 'family_id'; - const esRequest = { - query, - aggs: { - family_id: { - terms: { field, size: ES_QUERY_MAX_SIZE }, - }, - }, - }; - const results = await executeSearch(es, normalizedConfigs.alias, esRequest); - const buckets = get(results, 'body.aggregations.family_id.buckets', []); - const familyIds = buckets.map(b => b.key); - - return { - op: 'or', - content: [ - { - op: 'in', - content: { - field, - value: familyIds, - }, - }, - { - op: 'in', - content: { - field: 'participant_id', - value: participantIds, - }, - }, - ], - }; -}; - - -export default generateFamilySqon; diff --git a/src/reports/family-clinical-data/generatePtSqonWithRelativesIfExist.test.ts b/src/reports/family-clinical-data/generatePtSqonWithRelativesIfExist.test.ts new file mode 100644 index 0000000..55ae40e --- /dev/null +++ b/src/reports/family-clinical-data/generatePtSqonWithRelativesIfExist.test.ts @@ -0,0 +1,50 @@ +import { + extractFieldAggregationIds, + mergeParticipantsWithoutDuplicates, + xIsSubsetOfY, +} from './generatePtSqonWithRelativesIfExist'; + +describe('Sqon generator for selected participants and their relatives', () => { + test('checks if X is a subset of Y', () => { + // + const x = ['p1', 'p2']; + const y = ['p1', 'p2', 'p4']; + expect(xIsSubsetOfY(x, y)).toBeTruthy(); + }); + test('merges participants adequately', () => { + const x = ['p1', 'p2', 'p3']; + const y = ['p2', 'p3', 'p4']; + expect(mergeParticipantsWithoutDuplicates(x, y).every(p => ['p1', 'p2', 'p3', 'p4'].includes(p))).toBeTruthy(); + }); + test('extracts correctly all participants from initial ES response', async () => { + const query = { + bool: { + must: [ + { + terms: { + down_syndrome_status: ['T21'], + boost: 0, + }, + }, + ], + }, + }; + const searchExecutor = async (_: object) => + Promise.resolve({ + query: _, + body: { + aggregations: { + ids: { + buckets: [ + { key: 'p1', count: 1 }, + { key: 'p2', count: 1 }, + { key: 'p2', count: 1 }, + ], + }, + }, + }, + }); + const ps = await extractFieldAggregationIds(query, 'participant_id', searchExecutor); + expect(ps).toEqual(['p1', 'p2']); + }); +}); diff --git a/src/reports/family-clinical-data/generatePtSqonWithRelativesIfExist.ts b/src/reports/family-clinical-data/generatePtSqonWithRelativesIfExist.ts new file mode 100644 index 0000000..d2e5588 --- /dev/null +++ b/src/reports/family-clinical-data/generatePtSqonWithRelativesIfExist.ts @@ -0,0 +1,130 @@ +import { buildQuery } from '@arranger/middleware'; + +import { getExtendedConfigs, getNestedFields } from '../../utils/arrangerUtils'; +import { executeSearch } from '../../utils/esUtils'; +import { Client } from '@elastic/elasticsearch'; +import { resolveSetsInSqon } from '../../utils/sqonUtils'; + +import { Sqon } from '../../utils/setsTypes'; +import { ES_QUERY_MAX_SIZE } from '../../env'; + +type Bucket = { key: string; count: number }; +type AggregationIdsRequest = { + [index: string]: any; + query: object; + body: { + aggregations: { + ids: { + buckets: Bucket[]; + }; + }; + }; +}; +export const extractFieldAggregationIds = async ( + query: object, + field: string, + searchExecutor: (q: object) => Promise, +): Promise => { + const r = await searchExecutor({ + query, + aggs: { + ids: { + terms: { field: field, size: ES_QUERY_MAX_SIZE }, + }, + }, + }); + const rawIds: string[] = (r.body?.aggregations?.ids?.buckets || []).map((bucket: Bucket) => bucket.key); + return [...new Set(rawIds)]; +}; + +export const mergeParticipantsWithoutDuplicates = (x: string[], y: string[]) => [...new Set([...x, ...y])]; + +// extract in a more general file when and if needed. +export const xIsSubsetOfY = (x: string[], y: string[]) => x.every((e: string) => y.includes(e)); +/** + * Generate a sqon from the family_id of all the participants in the given `sqon`. + * @param {object} es - an `elasticsearch.Client` instance. + * @param {string} projectId - the id of the arranger project. + * @param {object} sqon - the sqon used to filter the results. + * @param {object} normalizedConfigs - the normalized report configuration. + * @param {string} userId - the user id. + * @param {string} accessToken - the user access token. + * @returns {object} - A sqon of all the `family_id`. + */ +const generatePtSqonWithRelativesIfExist = async ( + es: Client, + projectId: string, + sqon: Sqon, + normalizedConfigs: { indexName: string; alias: string; [index: string]: any }, + userId: string, + accessToken: string, +): Promise => { + const extendedConfig = await getExtendedConfigs(es, projectId, normalizedConfigs.indexName); + const nestedFields = getNestedFields(extendedConfig); + const newSqon = await resolveSetsInSqon(sqon, userId, accessToken); + + const query = buildQuery({ nestedFields, filters: newSqon }); + const searchExecutor = async (q: object) => await executeSearch(es, normalizedConfigs.alias, q); + + const allSelectedParticipantsIds: string[] = await extractFieldAggregationIds( + query, + 'participant_id', + searchExecutor, + ); + const allFamiliesIdsOfSelectedParticipants: string[] = await extractFieldAggregationIds( + { + bool: { + must: [ + { + terms: { + participant_id: allSelectedParticipantsIds, + }, + }, + ], + }, + }, + 'families_id', + searchExecutor, + ); + const allRelativesIds: string[] = await extractFieldAggregationIds( + { + bool: { + must: [ + { + terms: { + families_id: allFamiliesIdsOfSelectedParticipants, + }, + }, + ], + }, + }, + 'participant_id', + searchExecutor, + ); + const selectedParticipantsIdsPlusRelatives = mergeParticipantsWithoutDuplicates( + allSelectedParticipantsIds, + allRelativesIds, + ); + + console.assert( + selectedParticipantsIdsPlusRelatives.length >= allSelectedParticipantsIds.length && + xIsSubsetOfY(allSelectedParticipantsIds, selectedParticipantsIdsPlusRelatives), + `Family Report (sqon enhancer): The participants ids computed must be equal or greater than the selected participants. + Moreover, selected participants must a subset of the computed ids.`, + ); + + return { + op: 'and', + content: [ + { + op: 'in', + content: { + field: 'participant_id', + value: selectedParticipantsIdsPlusRelatives, + }, + }, + ], + }; +}; + +export default generatePtSqonWithRelativesIfExist; diff --git a/src/reports/family-clinical-data/index.ts b/src/reports/family-clinical-data/index.ts index e43ffe4..4ab1b37 100644 --- a/src/reports/family-clinical-data/index.ts +++ b/src/reports/family-clinical-data/index.ts @@ -6,7 +6,7 @@ import configKf from './configKf'; import configInclude from './configInclude'; import { normalizeConfigs } from '../../utils/configUtils'; -import generateFamilySqon from './generateFamilySqon'; +import generatePtSqonWithRelativesIfExist from './generatePtSqonWithRelativesIfExist'; import { reportGenerationErrorHandler } from '../../errors'; import { PROJECT } from '../../env'; import { ProjectType, ReportConfig } from '../types'; @@ -38,19 +38,10 @@ const clinicalDataReport = async (req: Request, res: Response): Promise => const normalizedConfigs = await normalizeConfigs(esClient, projectId, reportConfig); // generate a new sqon containing the id of all family members for the current sqon - const familySqon = await generateFamilySqon( - esClient, - projectId, - sqon, - normalizedConfigs, - userId, - accessToken, - PROJECT, - isKfNext, - ); + const participantsSqonWithRelatives = await generatePtSqonWithRelativesIfExist(esClient, projectId, sqon, normalizedConfigs, userId, accessToken); // Generate the report - await generateReport(esClient, res, projectId, familySqon, filename, normalizedConfigs, userId, accessToken); + await generateReport(esClient, res, projectId, participantsSqonWithRelatives, filename, normalizedConfigs, userId, accessToken); } catch (err) { reportGenerationErrorHandler(err); }